Merge remote-tracking branch 'origin/master' into lash/verify-cache

bug: False 404 in cic-meta logging

See merge request grassrootseconomics/cic-internal-integration!302

.gitignore

@@ -14,3 +14,4 @@ build/
 **/.venv
 .idea
 **/.vim
+**/*secret.yaml

.gitlab-ci.yml

@@ -1,14 +1,79 @@
 include:
-  - local: 'ci_templates/.cic-template.yml'
-  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
+  # these includes are app specific unit tests
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
+    #- local: 'apps/contract-migration/.gitlab-ci.yml'
+    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
+  - version
   - build
   - test
-  - release
+  - deploy
+
+image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
+
+variables:
+  DOCKER_BUILDKIT: "1"
+  COMPOSE_DOCKER_CLI_BUILD: "1"
+  CI_DEBUG_TRACE: "true"
+  SEMVERBOT_VERSION: "0.2.0"
+
+  #before_script:
+  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+version:
+  #image: python:3.7-stretch
+  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
+  stage: version
+  script:
+    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
+    - export TAG=$(sbot predict version -m auto)
+    - |
+      if [[ -z $TAG ]]
+      then
+        echo "tag could not be set $@"
+        exit 1
+      fi
+    - echo $TAG > version
+    - git tag -a v$TAG -m "ci tagged"
+    - git push origin v$TAG
+  artifacts:
+    paths:
+      - version
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
+
+# runs on protected branches and pushes to repo
+build-push:
+  stage: build
+  tags:
+    - integration
+  #script:
+  #  - TAG=$CI_Cbefore_script:
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  script:
+    - TAG=latest ./scripts/build-push.sh
+    - TAG=$(cat ./version) ./scripts/build-push.sh
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
+
+deploy-dev:
+  stage: deploy
+  trigger: grassrootseconomics/devops
+  when: manual

.semverbot.toml

@@ -0,0 +1,16 @@
+[git]
+
+[git.config]
+email = "semverbot@grassroots.org"
+name = "semvervot"
+
+[git.tags]
+prefix = "v"
+
+[semver]
+mode = "git-commit"
+
+[semver.detection]
+patch = ["fix", "[fix]", "patch", "[patch]"]
+minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
+major = ["BREAKING CHANGE"]

README.md

@@ -2,25 +2,22 @@
 
 ## Getting started 
 
-## Make some keys
+This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
+
 
 ```
-docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
+export COMPOSE_DOCKER_CLI_BUILD=1
+export DOCKER_BUILDKIT=1
 ```
 
-
-### Prepare the repo
-
-This is stuff we need to put in  makefile but for now...
-
-File mounts and permisssions need to be set
+start services, database, redis and local ethereum node
 ```
-chmod -R 755 scripts/initdb apps/cic-meta/scripts/initdb
-````
-
-start cluster
+docker-compose up -d
 ```
-docker-compose up
+
+Run app/contract-migration to deploy contracts
+```
+RUN_MASK=3 docker-compose up contract-migration
 ```
 
 stop cluster
@@ -28,9 +25,9 @@ stop cluster
 docker-compose down
 ```
 
-delete data
+stop cluster and delete data
 ```
-docker-compose down -v
+docker-compose down -v --remove-orphans
 ```
 
 rebuild an images
@@ -38,5 +35,7 @@ rebuild an images
 docker-compose up --build <service_name>
 ```
 
-Deployment variables are writtend to service-configs/.env after everthing is up.
-
+to delete the buildkit cache
+```
+docker builder prune --filter type=exec.cachemount
+```

apps/cic-base-os/Dockerfile

@@ -1,34 +0,0 @@
-# The solc image messes up the alpine environment, so we have to go all over again
-FROM python:3.8.6-slim-buster
-
-LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
-LABEL spdx-license-identifier="GPL-3.0-or-later"
-LABEL description="Base layer for buiding development images for the cic component suite"
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
-
-
-RUN echo installing nodejs tooling
-
-COPY ./dev/nvm.sh /root/
-
-# Install nvm with node and npm
-# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
-ENV NVM_DIR /root/.nvm
-ENV NODE_VERSION 15.3.0
-ENV BANCOR_NODE_VERSION 10.16.0
-
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
-	&& . $NVM_DIR/nvm.sh \
-	&& nvm install $NODE_VERSION \
-	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION \
-# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
-# owner of these files is "1001" by default - wtf
-	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
-ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
-

apps/cic-base-os/README.md

@@ -1 +0,0 @@
-## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-base-os/aux/wait-for-it/.gitignore

@@ -0,0 +1,3 @@
+**/*.pyc
+.pydevproject
+/vendor/

apps/cic-base-os/aux/wait-for-it/.travis.yml

@@ -0,0 +1,7 @@
+language: python
+python:
+  - "2.7"
+
+script:
+  - python test/wait-for-it.py
+

apps/cic-base-os/aux/wait-for-it/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apps/cic-base-os/aux/wait-for-it/README.md

@@ -0,0 +1,75 @@
+# wait-for-it
+
+`wait-for-it.sh` is a pure bash script that will wait on the availability of a
+host and TCP port.  It is useful for synchronizing the spin-up of
+interdependent services, such as linked docker containers.  Since it is a pure
+bash script, it does not have any external dependencies.
+
+## Usage
+
+```text
+wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
+-h HOST | --host=HOST       Host or IP under test
+-p PORT | --port=PORT       TCP port under test
+                            Alternatively, you specify the host and port as host:port
+-s | --strict               Only execute subcommand if the test succeeds
+-q | --quiet                Don't output any status messages
+-t TIMEOUT | --timeout=TIMEOUT
+                            Timeout in seconds, zero for no timeout
+-- COMMAND ARGS             Execute command with args after the test finishes
+```
+
+## Examples
+
+For example, let's test to see if we can access port 80 on `www.google.com`,
+and if it is available, echo the message `google is up`.
+
+```text
+$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+You can set your own timeout with the `-t` or `--timeout=` option.  Setting
+the timeout value to 0 will disable the timeout:
+
+```text
+$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting for www.google.com:80 without a timeout
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+The subcommand will be executed regardless if the service is up or not.  If you
+wish to execute the subcommand only if the service is up, add the `--strict`
+argument. In this example, we will test port 81 on `www.google.com` which will
+fail:
+
+```text
+$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
+wait-for-it.sh: waiting 1 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
+wait-for-it.sh: strict mode, refusing to execute subprocess
+```
+
+If you don't want to execute a subcommand, leave off the `--` argument.  This
+way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
+and determine how to proceed:
+
+```text
+$ ./wait-for-it.sh www.google.com:80
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+$ echo $?
+0
+$ ./wait-for-it.sh www.google.com:81
+wait-for-it.sh: waiting 15 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
+$ echo $?
+124
+```
+
+## Community
+
+*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-base-os/aux/wait-for-it/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-cache/.coveragerc

@@ -4,3 +4,4 @@ omit =
 	scripts/*
 	cic_cache/db/migrations/*
 	cic_cache/version.py
+	cic_cache/cli

apps/cic-cache/.gitlab-ci.yml

@@ -1,52 +1,17 @@
-.cic_cache_variables:
-    variables:
-        APP_NAME: cic-cache
-        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-cache:
-    extends:
-        - .py_build_merge_request
-        - .cic_cache_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-
-test-mr-cic-cache:
-    stage: test
-    extends:
-        - .cic_cache_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-cache"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
-
-build-push-cic-cache:
-    extends:
-        - .py_build_push
-        - .cic_cache_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-        
-
+build-test-cic-cache:
+  stage: test 
+  tags:
+    - integration
+  variables:
+    APP_NAME: cic-cache
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+  script:
+    - cd apps/cic-cache
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+  allow_failure: true
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    changes:
+        - apps/$APP_NAME/**/*
+    when: always

apps/cic-cache/MANIFEST.in

@@ -0,0 +1 @@
+include *requirements.txt cic_cache/data/config/*

apps/cic-cache/README.md

@@ -0,0 +1 @@
+# CIC-CACHE

apps/cic-cache/aux/wait-for-it/.gitignore

@@ -0,0 +1,3 @@
+**/*.pyc
+.pydevproject
+/vendor/

apps/cic-cache/aux/wait-for-it/.travis.yml

@@ -0,0 +1,7 @@
+language: python
+python:
+  - "2.7"
+
+script:
+  - python test/wait-for-it.py
+

apps/cic-cache/aux/wait-for-it/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apps/cic-cache/aux/wait-for-it/README.md

@@ -0,0 +1,75 @@
+# wait-for-it
+
+`wait-for-it.sh` is a pure bash script that will wait on the availability of a
+host and TCP port.  It is useful for synchronizing the spin-up of
+interdependent services, such as linked docker containers.  Since it is a pure
+bash script, it does not have any external dependencies.
+
+## Usage
+
+```text
+wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
+-h HOST | --host=HOST       Host or IP under test
+-p PORT | --port=PORT       TCP port under test
+                            Alternatively, you specify the host and port as host:port
+-s | --strict               Only execute subcommand if the test succeeds
+-q | --quiet                Don't output any status messages
+-t TIMEOUT | --timeout=TIMEOUT
+                            Timeout in seconds, zero for no timeout
+-- COMMAND ARGS             Execute command with args after the test finishes
+```
+
+## Examples
+
+For example, let's test to see if we can access port 80 on `www.google.com`,
+and if it is available, echo the message `google is up`.
+
+```text
+$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+You can set your own timeout with the `-t` or `--timeout=` option.  Setting
+the timeout value to 0 will disable the timeout:
+
+```text
+$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting for www.google.com:80 without a timeout
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+The subcommand will be executed regardless if the service is up or not.  If you
+wish to execute the subcommand only if the service is up, add the `--strict`
+argument. In this example, we will test port 81 on `www.google.com` which will
+fail:
+
+```text
+$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
+wait-for-it.sh: waiting 1 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
+wait-for-it.sh: strict mode, refusing to execute subprocess
+```
+
+If you don't want to execute a subcommand, leave off the `--` argument.  This
+way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
+and determine how to proceed:
+
+```text
+$ ./wait-for-it.sh www.google.com:80
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+$ echo $?
+0
+$ ./wait-for-it.sh www.google.com:81
+wait-for-it.sh: waiting 15 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
+$ echo $?
+124
+```
+
+## Community
+
+*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-cache/aux/wait-for-it/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-cache/cic_cache/api.py

@@ -55,15 +55,37 @@ class Api:
                     queue=callback_queue,
                     )       
 
-    def list(self, offset, limit, address=None):
+    def list(self, offset=0, limit=100, address=None, oldest=False):
         s = celery.signature(
         'cic_cache.tasks.tx.tx_filter',
         [
-            0,
-            100,
+            offset,
+            limit,
             address,
+            oldest,
             ],
-            queue=None
+            queue=self.queue,
+        )
+        if self.callback_param != None:
+            s.link(self.callback_success).on_error(self.callback_error)
+
+        t = s.apply_async()
+
+        return t
+
+
+    def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False):
+        s = celery.signature(
+        'cic_cache.tasks.tx.tx_filter_content',
+        [
+            offset,
+            limit,
+            address,
+            block_offset,
+            block_limit,
+            oldest,
+            ],
+            queue=self.queue,
         )
         if self.callback_param != None:
             s.link(self.callback_success).on_error(self.callback_error)

apps/cic-cache/cic_cache/cache.py

@@ -10,12 +10,16 @@ from cic_cache.db.list import (
         list_transactions_mined,
         list_transactions_account_mined,
         list_transactions_mined_with_data,
+        list_transactions_mined_with_data_index,
+        list_transactions_account_mined_with_data_index,
+        list_transactions_account_mined_with_data,
         )
 
 logg = logging.getLogger()
 
 
 DEFAULT_FILTER_SIZE = 8192 * 8
+DEFAULT_LIMIT = 100
 
 class Cache:
 
@@ -32,7 +36,7 @@ class BloomCache(Cache):
         return n
 
 
-    def load_transactions(self, offset, limit):
+    def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
         """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
 
         Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
@@ -49,7 +53,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_mined(self.session, offset, limit) 
+        rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -58,7 +62,12 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-            lowest_block = r[0]
+                lowest_block = r[0]
+            else:
+                if oldest:
+                    highest_block = r[0]
+                else:
+                    lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -67,7 +76,7 @@ class BloomCache(Cache):
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
 
 
-    def load_transactions_account(self, address, offset, limit):
+    def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
         """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
 
         :param address: Address to retrieve transactions for.
@@ -79,7 +88,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_account_mined(self.session, address, offset, limit) 
+        rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -88,7 +97,12 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-            lowest_block = r[0]
+                lowest_block = r[0]
+            else:
+                if oldest:
+                    highest_block = r[0]
+                else:
+                    lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -99,8 +113,21 @@ class BloomCache(Cache):
 
 class DataCache(Cache):
 
-    def load_transactions_with_data(self, offset, end):
-        rows = list_transactions_mined_with_data(self.session, offset, end) 
+    def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
+        if limit == 0:
+            limit = DEFAULT_LIMIT
+        rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) 
+        return self.__process_rows(rows, oldest)
+
+
+    def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
+        if limit == 0:
+            limit = DEFAULT_LIMIT
+        rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) 
+        return self.__process_rows(rows, oldest)
+
+
+    def __process_rows(self, rows, oldest):
         tx_cache = []
         highest_block = -1;
         lowest_block = -1;
@@ -108,7 +135,12 @@ class DataCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r['block_number']
-            lowest_block = r['block_number']
+                lowest_block = r['block_number']
+            else:
+                if oldest:
+                    highest_block = r['block_number']
+                else:
+                    lowest_block = r['block_number']
             tx_type = 'unknown'
 
             if r['value'] != None:

apps/cic-cache/cic_cache/cli/__init__.py

@@ -0,0 +1,15 @@
+# local imports
+from .base import *
+from .chain import (
+        EthChainInterface,
+        chain_interface,
+        )
+from .rpc import RPC
+from .arg import ArgumentParser
+from .config import Config
+from .celery import CeleryApp
+from .registry import (
+        connect_registry,
+        connect_token_registry,
+        connect_declarator,
+        )

apps/cic-cache/cic_cache/cli/arg.py

@@ -0,0 +1,20 @@
+# external imports
+from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
+
+# local imports
+from .base import (
+        CICFlag,
+        Flag,
+        )
+
+
+class ArgumentParser(BaseArgumentParser):
+
+    def process_local_flags(self, local_arg_flags):
+        if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
+        if local_arg_flags & CICFlag.SYNCER:
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+        if local_arg_flags & CICFlag.CHAIN:
+            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/cli/base.py

@@ -0,0 +1,31 @@
+# standard imports
+import enum
+
+# external imports
+from chainlib.eth.cli import (
+    argflag_std_read,
+    argflag_std_write,
+    argflag_std_base,
+    Flag,
+    )
+
+class CICFlag(enum.IntEnum):
+   
+    # celery - nibble 1
+    CELERY = 1
+
+    # redis - nibble 2
+#    REDIS = 16
+#    REDIS_CALLBACK = 32
+
+    # chain - nibble 3
+    CHAIN = 256
+
+    # sync - nibble 4
+    SYNCER = 4096
+
+
+argflag_local_task = CICFlag.CELERY 
+#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
+argflag_local_chain = CICFlag.CHAIN
+argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-cache/cic_cache/cli/celery.py

@@ -0,0 +1,24 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
+logg = logging.getLogger(__name__)
+
+
+class CeleryApp:
+   
+    @classmethod
+    def from_config(cls, config):
+        backend_url = config.get('CELERY_RESULT_URL')
+        broker_url = config.get('CELERY_BROKER_URL')
+        celery_app = None
+        if backend_url != None:
+            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
+            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
+        else:
+            celery_app = celery.Celery(broker=broker_url)
+            logg.info('creating celery app without results backend on {}'.format(broker_url))
+
+        return celery_app

apps/cic-cache/cic_cache/cli/chain.py

@@ -0,0 +1,21 @@
+# external imports
+from chainlib.eth.block import (
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
+
+
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()

apps/cic-cache/cic_cache/cli/config.py

@@ -0,0 +1,63 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.cli import (
+        Config as BaseConfig,
+        Flag,
+        )
+
+# local imports
+from .base import CICFlag
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+logg = logging.getLogger(__name__)
+
+
+class Config(BaseConfig):
+
+    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
+
+    @classmethod
+    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
+        expanded_base_config_dir = [cls.local_base_config_dir]
+        if base_config_dir != None:
+            if isinstance(base_config_dir, str):
+                base_config_dir = [base_config_dir]
+            for d in base_config_dir:
+                expanded_base_config_dir.append(d)
+        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
+
+        local_args_override = {}
+#        if local_arg_flags & CICFlag.REDIS:
+#            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
+#            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
+#            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
+#            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
+
+        if local_arg_flags & CICFlag.CHAIN:
+            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
+
+        if local_arg_flags & CICFlag.CELERY:
+            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
+
+        if local_arg_flags & CICFlag.SYNCER:
+            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
+            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
+
+        config.dict_override(local_args_override, 'local cli args')
+
+#        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+#            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
+#            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
+
+        if local_arg_flags & CICFlag.CELERY:
+            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
+
+        logg.debug('config loaded:\n{}'.format(config))
+
+        return config
+
+

apps/cic-cache/cic_cache/cli/registry.py

@@ -0,0 +1,33 @@
+# standard imports
+import logging
+
+# external imports
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
+from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
+from chainlib.eth.constant import ZERO_ADDRESS
+
+logg = logging.getLogger()
+
+
+def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS):
+    registry = CICRegistry(chain_spec, conn)
+    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
+    logg.debug('using token registry address {}'.format(token_registry_address))
+    lookup = TokenIndexLookup(chain_spec, token_registry_address)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
+    registry = CICRegistry(chain_spec, conn)
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+    logg.debug('using declarator address {}'.format(declarator_address))
+    lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
+    CICRegistry.address = registry_address
+    registry = CICRegistry(chain_spec, conn)
+    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
+    return registry

apps/cic-cache/cic_cache/cli/rpc.py

@@ -0,0 +1,43 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import (
+        RPCConnection,
+        ConnType,
+        )
+from chainlib.eth.connection import EthUnixSignerConnection
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+
+class RPC:
+
+    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
+        self.chain_spec = chain_spec
+        self.rpc_provider = rpc_provider
+        self.signer_provider = signer_provider
+
+
+    def get_default(self):
+        return RPCConnection.connect(self.chain_spec, 'default')
+
+
+    @staticmethod
+    def from_config(config):
+        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
+        if config.get('SIGNER_PROVIDER'):
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
+        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        logg.info('set up rpc: {}'.format(rpc))
+        return rpc
+
+
+    def __str__(self):
+        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
+
+
+

apps/cic-cache/cic_cache/data/config/celery.ini

@@ -0,0 +1,5 @@
+[celery]
+broker_url = redis://localhost:6379
+result_url = 
+queue = cic-cache
+debug = 0

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -0,0 +1,4 @@
+[cic]
+registry_address = 
+trust_address =
+health_modules =

apps/cic-cache/cic_cache/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine =
+driver =
+host =
+port =
+name = cic-cache
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-cache/cic_cache/data/config/signer.ini

@@ -0,0 +1,2 @@
+[signer]
+provider =

apps/cic-cache/cic_cache/data/config/syncer.ini

@@ -1,3 +1,4 @@
 [syncer]
 loop_interval = 1
-history_start = 0
+offset = 0
+no_history = 0

apps/cic-cache/cic_cache/db/list.py

@@ -13,6 +13,9 @@ def list_transactions_mined(
         session,
         offset,
         limit,
+        block_offset,
+        block_limit,
+        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -23,15 +26,62 @@ def list_transactions_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
     r = session.execute(s)
     return r
 
 
 def list_transactions_mined_with_data(
+        session,
+        offset,
+        limit,
+        block_offset,
+        block_limit,
+        oldest=False,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit.
+
+    :param block_offset: First block to include in search
+    :type block_offset: int
+    :param block_limit: Last block to include in search
+    :type block_limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
+
+
+    r = session.execute(s)
+    return r
+
+
+def list_transactions_mined_with_data_index(
         session,
         offset,
         end,
+        block_offset,
+        block_limit,
+        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -42,7 +92,87 @@ def list_transactions_mined_with_data(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end)
+
+    r = session.execute(s)
+    return r
+
+
+def list_transactions_account_mined_with_data_index(
+        session,
+        address,
+        offset,
+        limit,
+        block_offset,
+        block_limit,
+        oldest=False,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address
+
+    :param offset: Offset in data set to return transactions from
+    :type offset: int
+    :param limit: Max number of transactions to retrieve
+    :type limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
+
+    r = session.execute(s)
+    return r
+
+def list_transactions_account_mined_with_data(
+        session,
+        address,
+        offset,
+        limit,
+        block_offset,
+        block_limit,
+        oldest=False,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit.
+
+    :param block_offset: First block to include in search
+    :type block_offset: int
+    :param block_limit: Last block to include in search
+    :type block_limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
 
     r = session.execute(s)
     return r
@@ -53,6 +183,9 @@ def list_transactions_account_mined(
         address,
         offset,
         limit,
+        block_offset,
+        block_limit,
+        oldest=False,
         ):
     """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
 
@@ -65,7 +198,20 @@ def list_transactions_account_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
+
+    else:
+        s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
+
     r = session.execute(s)
     return r
 

apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py

@@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243
 """
 from alembic import op
 import sqlalchemy as sa
-from chainsyncer.db.migrations.sqlalchemy import (
+from chainsyncer.db.migrations.default.export import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,3 +100,4 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -5,7 +5,10 @@ import re
 import base64
 
 # external imports
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
 from cic_cache.cache import (
@@ -19,6 +22,7 @@ logg = logging.getLogger(__name__)
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
 re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
 re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
+re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
 
 DEFAULT_LIMIT = 100
 
@@ -28,9 +32,7 @@ def process_transactions_account_bloom(session, env):
     if not r:
         return None
 
-    address = r[1]
-    if r[2] == None:
-        address = add_0x(address)
+    address = strip_0x(r[1])
     offset = 0
     if r.lastindex > 2:
         offset = r[4]
@@ -91,13 +93,14 @@ def process_transactions_all_data(session, env):
     if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
         return None
 
-    offset = r[1]
-    end = r[2]
+    logg.debug('got data request {}'.format(env))
+    block_offset = r[1]
+    block_end = r[2]
     if int(r[2]) < int(r[1]):
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
 
     for r in tx_cache:
         r['date_block'] = r['date_block'].timestamp()
@@ -112,3 +115,38 @@ def process_transactions_all_data(session, env):
     j = json.dumps(o)
 
     return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_account_data(session, env):
+    r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
+    if not r:
+        return None
+    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+        return None
+
+    logg.debug('got data request {}'.format(env))
+    address = strip_0x(r[1])
+    #if r[2] == None:
+    #    address = add_0x(address)
+    offset = 0
+    if r.lastindex > 2:
+        offset = r[4]
+    limit = DEFAULT_LIMIT
+    if r.lastindex > 4:
+        limit = r[6]
+
+    c = DataCache(session)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
+
+    for r in tx_cache:
+        r['date_block'] = r['date_block'].timestamp()
+
+    o = {
+        'low': lowest_block,
+        'high': highest_block,
+        'data': tx_cache,
+    }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -8,41 +8,31 @@ import base64
 import confini
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.runnable.daemons.query import (
         process_transactions_account_bloom,
+        process_transactions_account_data,
         process_transactions_all_bloom,
         process_transactions_all_data,
         )
+import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
+arg_flags = cic_cache.cli.argflag_std_read
+local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task 
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
+# connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 
@@ -58,6 +48,7 @@ def application(env, start_response):
             process_transactions_all_data,
             process_transactions_all_bloom,
             process_transactions_account_bloom,
+            process_transactions_account_data,
             ]:
         r = None
         try:

apps/cic-cache/cic_cache/runnable/daemons/tasker.py

@@ -9,6 +9,7 @@ import celery
 import confini
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.tasks.tx import *
@@ -16,35 +17,20 @@ from cic_cache.tasks.tx import *
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-
+# process args
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_task
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn)
 
-# verify database connection with minimal sanity query
-#session = SessionBase.create_session()
-#session.execute('select version_num from alembic_version')
-#session.close()
-
 # set up celery
 current_app = celery.Celery(__name__)
 
@@ -87,9 +73,9 @@ def main():
     elif args.v:
         argv.append('--loglevel=INFO')
     argv.append('-Q')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
     argv.append('-n')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
 
     current_app.worker_main(argv)
 

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -8,15 +8,7 @@ import sys
 import re
 
 # external imports
-import confini
-import celery
 import sqlalchemy
-import rlp
-import cic_base.config
-import cic_base.log
-import cic_base.argparse
-import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -34,6 +26,7 @@ from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import (
         dsn_from_config,
         add_tag,
@@ -43,32 +36,36 @@ from cic_cache.runnable.daemons.filters import (
         FaucetFilter,
         )
 
-script_dir = os.path.realpath(os.path.dirname(__file__))
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
 
-def add_block_args(argparser):
-    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
-    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-    return argparser
+# process args
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_sync
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
+args = argparser.parse_args()
 
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
-logg = cic_base.log.create()
-argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
-args = cic_base.argparse.parse(argparser, logg)
-config = cic_base.config.create(args.c, args, args.env_prefix)
-
-config.add(args.history_start, 'SYNCER_HISTORY_START', True)
-config.add(args.no_history, '_NO_HISTORY', True)
-
-cic_base.config.log(config)
-
+# connect to database
 dsn = dsn_from_config(config)
-
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+# set up rpc
+rpc = cic_cache.cli.RPC.from_config(config)
+conn = rpc.get_default()
 
-cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
+# set up chain provisions
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+registry = None
+try:
+    registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+except UnknownContractError as e:
+    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+    sys.exit(1)
+logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 
 def register_filter_tags(filters, session):
@@ -95,14 +92,12 @@ def main():
 
     syncers = []
 
-    #if SQLBackend.first(chain_spec):
-    #    backend = SQLBackend.initial(chain_spec, block_offset)
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_HISTORY_START')
+        initial_block_start = config.get('SYNCER_OFFSET')
         initial_block_offset = block_offset
-        if config.get('_NO_HISTORY'):
+        if config.get('SYNCER_NO_HISTORY'):
             initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
@@ -112,10 +107,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, chain_interface))
+        syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, chain_interface))
+    syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/cic_cache/tasks/tx.py

@@ -2,14 +2,17 @@
 import celery
 
 # local imports
-from cic_cache.cache import BloomCache
+from cic_cache.cache import (
+        BloomCache,
+        DataCache,
+        )
 from cic_cache.db.models.base import SessionBase
 
 celery_app = celery.current_app
 
 
 @celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, encoding='hex'):
+def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     queue = self.request.delivery_info.get('routing_key')
 
     session = SessionBase.create_session()
@@ -17,9 +20,9 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'):
     c = BloomCache(session)
     b = None
     if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest)
     else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest)
 
     session.close()
 
@@ -35,4 +38,17 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'):
     return o
 
 
+@celery_app.task(bind=True)
+def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'):
+    session = SessionBase.create_session()
 
+    c = DataCache(session)
+    b = None
+    if address == None:
+        (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
+    else:
+        (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit)
+
+    session.close()
+
+    return (lowest_block, highest_block, tx_cache,)

apps/cic-cache/cic_cache/version.py

@@ -4,7 +4,7 @@ import semver
 version = (
         0,
         2,
-        0,
+        1,
         'alpha.2',
         )
 

apps/cic-cache/config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis:///
-result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-chain_spec =
-trust_address = 

apps/cic-cache/config/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/bancor.ini

@@ -1,3 +0,0 @@
-[bancor]
-registry_address = 
-dir = /usr/local/share/bancor

apps/cic-cache/config/docker/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-chain_spec =
-registry_address =  
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=grassroots
-PASSWORD=
-HOST=localhost
-PORT=63432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = ws://localhost:8545

apps/cic-cache/config/syncer.ini

@@ -1,3 +0,0 @@
-[syncer]
-loop_interval = 5
-history_start = 0

apps/cic-cache/docker/Dockerfile

@@ -1,31 +1,25 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 
 COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_ARGS=""
+ARG PIP_INDEX_URL="https://pypi.org/simple"
+
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt
 
 COPY . . 
 
 RUN python setup.py install
 
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
 # for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY ./aux/wait-for-it/wait-for-it.sh ./
 COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
 COPY /docker/start_tracker.sh ./start_tracker.sh

apps/cic-cache/docker/Dockerfile_ci

@@ -1,37 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+

apps/cic-cache/requirements.txt

@@ -1,13 +1,15 @@
-cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
-confini~=0.3.6rc3
+confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
-moolb~=0.1.0
-cic-eth-registry~=0.5.6a1
+moolb~=0.1.1b2
+cic-eth-registry~=0.6.1a5
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a1
+chainsyncer[sql]>=0.0.6a3,<0.1.0
+erc20-faucet>=0.3.2a2, <0.4.0
+chainlib-eth>=0.0.9a14,<0.1.0
+eth-address-index>=0.2.3a4,<0.3.0
+okota>=0.2.4a6,<0.3.0

apps/cic-cache/scripts/migrate.py

@@ -1,14 +1,19 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+
+# standard imports
 import os
 import argparse
 import logging
 import re
 
+# external imports
 import alembic
 from alembic.config import Config as AlembicConfig
 import confini
 
+# local imports
 from cic_cache.db import dsn_from_config
+import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -17,31 +22,29 @@ logg = logging.getLogger()
 rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_cache', 'db')
 migrationsdir = os.path.join(dbdir, 'migrations')
+configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
+#config_dir = os.path.join('/usr/local/etc/cic-cache')
 
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_sync
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
+argparser.add_argument('-f', '--force', action='store_true', help='force action')
+argparser.add_argument('--migrations-dir', dest='migrations_dir', type=str, help='migrations directory')
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
+extra_args = {
+    'reset': None,
+    'force': None,
+    'migrations_dir': None,
+        }
 
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
-migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
+migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE'))
 if not os.path.isdir(migrations_dir):
     logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
     migrations_dir = os.path.join(args.migrations_dir, 'default')

apps/cic-cache/setup.cfg

@@ -23,11 +23,13 @@ licence_files =
 
 [options]
 python_requires = >= 3.6
+include_package_data = True
 packages = 
 	cic_cache
 	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
+	cic_cache.cli
 	cic_cache.runnable
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters
@@ -39,3 +41,4 @@ console_scripts =
 	cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
 	cic-cache-serverd = cic_cache.runnable.daemons.server:main
 	cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main
+	cic-cache-list = cic_cache.runable.list:main

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-cic_base[full]==0.1.3a3+build.984b5cff
-sarafu-faucet~=0.0.4a1
+sarafu-faucet~=0.0.7a1
+erc20-transfer-authorization>=0.3.5a1,<0.4.0

apps/cic-cache/tests/cli/test_cli_args.py

@@ -0,0 +1,40 @@
+# standard imports
+import os
+
+# external imports
+import chainlib.cli 
+
+# local imports
+import cic_cache.cli
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+config_dir = os.path.join(script_dir, '..', 'testdata', 'config')
+
+
+def test_argumentparserto_config():
+
+    argparser = cic_cache.cli.ArgumentParser()
+    
+    local_flags = 0xffff
+    argparser.process_local_flags(local_flags) 
+    argparser.add_argument('--foo', type=str)
+    args = argparser.parse_args([
+        '-q', 'baz',
+        '--offset', '13',
+        '--no-history',
+        '-r','0xdeadbeef',
+        '-vv',
+        '--foo', 'bar',
+        ])
+
+    extra_args = {
+            'foo': '_BARBARBAR',
+            }
+    config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
+
+    assert config.get('_BARBARBAR') == 'bar'
+    assert config.get('CELERY_QUEUE') == 'baz'
+    assert config.get('SYNCER_NO_HISTORY') == True
+    assert config.get('SYNCER_OFFSET') == 13
+    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
+

apps/cic-cache/tests/cli/test_cli_celery.py

@@ -0,0 +1,17 @@
+# standard imports
+import tempfile
+
+# local imports
+import cic_cache.cli
+
+
+def test_cli_celery():
+    cf = tempfile.mkdtemp()
+
+    config = {
+            'CELERY_RESULT_URL': 'filesystem://' + cf,
+            }
+    cic_cache.cli.CeleryApp.from_config(config)
+
+    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
+    cic_cache.cli.CeleryApp.from_config(config)

apps/cic-cache/tests/cli/test_cli_chain.py

@@ -0,0 +1,68 @@
+# external imports
+import pytest
+from chainlib.eth.gas import (
+        Gas,
+        RPCGasOracle,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.block import (
+        block_latest,
+        Block,
+        )
+from chainlib.eth.pytest.fixtures_chain import default_chain_spec
+from chainlib.eth.pytest.fixtures_ethtester import *
+from cic_eth_registry.pytest.fixtures_contracts import *
+from hexathon import add_0x
+
+# local imports
+import cic_cache.cli
+
+
+@pytest.mark.xfail()
+def test_cli_rpc(
+        eth_rpc,
+        eth_signer,
+        default_chain_spec,
+        ):
+    config = {
+        'CHAIN_SPEC': str(default_chain_spec),
+        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
+            }
+    rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') 
+    conn = rpc.get_by_label('foo')
+    #o = block_latest()
+    #conn.do(o)
+
+
+def test_cli_chain(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        contract_roles,
+        ):
+    ifc = cic_cache.cli.EthChainInterface()
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
+    gas_oracle = RPCGasOracle(conn=eth_rpc)
+    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
+    recipient = add_0x(os.urandom(20).hex())
+    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024)
+    r = eth_rpc.do(o)
+
+    o = ifc.tx_receipt(r)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    o = ifc.block_by_number(1)
+    block_src = eth_rpc.do(o)
+    block = ifc.block_from_src(block_src)
+    assert block.number == 1
+
+    with pytest.raises(KeyError):
+        assert block_src['gasUsed'] == 21000
+        assert block_src['gas_used'] == 21000
+
+    block_src = ifc.src_normalize(block_src)
+    assert block_src['gasUsed'] == 21000
+    assert block_src['gas_used'] == 21000
+

apps/cic-cache/tests/conftest.py

@@ -64,7 +64,6 @@ def txs(
         dt.timestamp(),
             )
 
-
     tx_number = 42
     tx_hash_second = '0x' + os.urandom(32).hex()
     tx_signed_second = '0x' + os.urandom(128).hex()
@@ -93,6 +92,44 @@ def txs(
             ]
 
 
+@pytest.fixture(scope='function')
+def more_txs(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        ):
+
+    session = init_database
+
+    tx_number = 666
+    tx_hash = '0x' + os.urandom(32).hex()
+    tx_signed = '0x' + os.urandom(128).hex()
+    nonce = 3
+
+    dt = datetime.datetime.utcnow()
+    dt += datetime.timedelta(hours=1)
+    db.add_transaction(
+        session,
+        tx_hash,
+        list_defaults['block']+2,
+        tx_number,
+        list_actors['alice'],
+        list_actors['diane'],
+        list_tokens['bar'],
+        list_tokens['bar'],
+        2048,
+        4096,
+        False,
+        dt.timestamp(),
+        )
+
+    session.commit()
+
+    return [tx_hash] + txs
+
+
 @pytest.fixture(scope='function')
 def tag_txs(
         init_database,

apps/cic-cache/tests/test_cache.py

@@ -8,6 +8,7 @@ import json
 import pytest
 
 # local imports
+from cic_cache import db
 from cic_cache import BloomCache
 from cic_cache.cache import DataCache
 
@@ -18,7 +19,6 @@ def test_cache(
         init_database,
         list_defaults,
         list_actors,
-        list_tokens,
         txs,
         ):
 
@@ -37,9 +37,6 @@ def test_cache(
 
 def test_cache_data(
         init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
         txs,
         tag_txs,
         ):
@@ -47,10 +44,209 @@ def test_cache_data(
     session = init_database
 
     c = DataCache(session)
-    b = c.load_transactions_with_data(410000, 420000)
+    b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True)
 
     assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == txs[1]
-    assert b[2][1]['tx_type'] == 'unknown'
-    assert b[2][0]['tx_type'] == 'test.taag'
+    assert b[2][0]['tx_hash'] == txs[0]
+    assert b[2][0]['tx_type'] == 'unknown'
+    assert b[2][1]['tx_type'] == 'test.taag'
+
+
+def test_cache_ranges(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        more_txs,
+        ):
+
+    session = init_database
+       
+    oldest = list_defaults['block'] - 1
+    mid = list_defaults['block']
+    newest = list_defaults['block'] + 2
+
+    c = BloomCache(session)
+    b = c.load_transactions(0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    b = c.load_transactions(1, 2)
+    assert b[0] == oldest
+    assert b[1] == mid
+
+    b = c.load_transactions(0, 2)
+    assert b[0] == mid
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 1)
+    assert b[0] == newest
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 100, oldest=True)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == mid
+
+    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
+    assert b[0] == oldest
+    assert b[1] == mid
+
+    # now check when supplying account
+    b = c.load_transactions_account(list_actors['alice'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    b = c.load_transactions_account(list_actors['bob'], 0, 100)
+    assert b[0] == mid
+    assert b[1] == mid
+
+    b = c.load_transactions_account(list_actors['diane'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    # add block filter to the mix
+    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
     
+    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+
+    b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == mid
+
+    b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == oldest
+
+
+def test_cache_ranges_data( 
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        more_txs,
+        ):
+
+    session = init_database
+       
+    oldest = list_defaults['block'] - 1
+    mid = list_defaults['block']
+    newest = list_defaults['block'] + 2
+
+    c = DataCache(session)
+
+    b = c.load_transactions_with_data(0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 3
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][2]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_with_data(1, 2)
+    assert b[0] == oldest
+    assert b[1] == mid
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[1]
+    assert b[2][1]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_with_data(0, 2)
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_with_data(0, 1)
+    assert b[0] == newest
+    assert b[1] == newest
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[0]
+
+    b = c.load_transactions_with_data(0, 100, oldest=True)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 3
+    assert b[2][0]['tx_hash'] == more_txs[2]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+    assert b[2][2]['tx_hash'] == more_txs[0]
+
+    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == mid
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[1]
+    assert b[2][1]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
+    assert b[0] == oldest
+    assert b[1] == mid
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[2]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    # now check when supplying account
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 3
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+    assert b[2][2]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
+    assert b[0] == mid
+    assert b[1] == mid
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[2]
+
+    # add block filter to the mix
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == mid
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == oldest
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[2]

apps/cic-cache/tests/testdata/config/test.ini

@@ -0,0 +1,2 @@
+[foo]
+bar_baz = xyzzy

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py

@@ -46,7 +46,7 @@ def get_adjusted_balance(self, token_symbol, amount, timestamp):
 
 
 def aux_setup(rpc, config, sender_address=ZERO_ADDRESS):
-    chain_spec_str = config.get('CIC_CHAIN_SPEC')
+    chain_spec_str = config.get('CHAIN_SPEC')
     chain_spec = ChainSpec.from_chain_str(chain_spec_str)
     token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,5 +1,5 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.2a3
-cic-eth-registry~=0.5.6a1
-chainlib~=0.0.5a1
-cic_eth~=0.12.0a2
+erc20-demurrage-token~=0.0.5a3
+cic-eth-registry~=0.6.1a6
+chainlib~=0.0.9rc1
+cic_eth~=0.12.4a11

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.2a4
+version = 0.0.2a7
 description = cic-eth tasks supporting erc20 demurrage token
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt

@@ -5,8 +5,7 @@ pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
 SQLAlchemy==1.3.20
-cic-eth~=0.12.0a1
 liveness~=0.0.1a7
-eth-accounts-index==0.0.12a1
-eth-contract-registry==0.5.6a1
-eth-address-index==0.1.2a1
+eth-accounts-index==0.1.1a1
+eth-contract-registry==0.5.8a1
+eth-address-index==0.2.1a1

apps/cic-eth/.coveragerc

@@ -6,4 +6,5 @@ omit =
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
 	cic_eth/queue/state.py
+	cic_eth/cli
 	*redis*.py

apps/cic-eth/.gitlab-ci.yml

@@ -1,52 +1,16 @@
-.cic_eth_variables:
-    variables:
-        APP_NAME: cic-eth
-        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-eth:
-    extends:
-        - .cic_eth_variables
-        - .py_build_target_dev
-    rules:
+build-test-cic-eth:
+  stage: test
+  tags:
+    - integration
+  variables:
+    APP_NAME: cic-eth
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+  script:
+    - cd apps/cic-eth
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+  rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-          - apps/cic-eth/**/*
-      when: always
-
-test-mr-cic-eth:
-    stage: test
-    extends:
-        - .cic_eth_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r admin_requirements.txt
-          -r services_requirements.txt
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-eth"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
-
-build-push-cic-eth:
-    extends:
-        - .py_build_push
-        - .cic_eth_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-eth/**/*
+          - apps/$APP_NAME/**/*
       when: always

apps/cic-eth/MANIFEST.in

@@ -1,2 +1,2 @@
-include *requirements.txt config/test/*
+include *requirements.txt config/test/* cic_eth/data/config/*
 

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry~=0.5.6a1
-hexathon~=0.0.1a7
-chainqueue~=0.0.2b5
-eth-erc20==0.0.10a2
+cic-eth-registry>=0.6.1a6,<0.7.0
+hexathon~=0.0.1a8
+chainqueue>=0.0.4a6,<0.1.0
+eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -4,8 +4,12 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
+from hexathon import (
+        add_0x,
+        strip_0x,
+        uniform as hex_uniform,
+        )
 
 # local imports
 from cic_eth.db.enum import LockEnum
@@ -15,12 +19,17 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         )
 from cic_eth.error import LockedError
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -32,6 +41,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -41,7 +51,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -53,6 +63,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -62,7 +73,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -72,6 +83,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -79,7 +91,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -89,6 +101,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -96,7 +109,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -106,6 +119,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -113,7 +127,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -123,6 +137,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -131,11 +146,13 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
+    if address != None:
+        address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
     if address != None:
         r |= Lock.check(chain_str, lock_flags, address=address, session=session)
     if r > 0:

apps/cic-eth/cic_eth/admin/nonce.py

@@ -14,7 +14,11 @@ from chainqueue.sql.query import get_tx
 from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        uniform as hex_uniform,
+        )
 from potaahto.symbols import snake_and_camel
 
 # local imports
@@ -29,6 +33,7 @@ from cic_eth.admin.ctrl import (
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
 from cic_eth.task import BaseTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -69,15 +74,17 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
+    query_address = tx_normalize.wallet_address(address)
     q = session.query(Otx)
     q = q.join(TxCache)
-    q = q.filter(TxCache.sender==address)
+    q = q.filter(TxCache.sender==query_address)
     q = q.filter(Otx.nonce>=nonce+delta)
     q = q.order_by(Otx.nonce.asc())
     otxs = q.all()
 
     tx_hashes = []
     txs = []
+    gas_total = 0
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
@@ -89,8 +96,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         tx_new['gas_price'] += 1
         tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
+        gas_total += tx_new['gas_price'] * tx_new['gas']
 
         logg.debug('tx_new {}'.format(tx_new))
+        logg.debug('gas running total {}'.format(gas_total))
 
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
@@ -122,8 +131,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     s = create_check_gas_task(
          txs, 
          chain_spec,
-         tx_new['from'],
-         gas=tx_new['gas'],
+         #tx_new['from'],
+         address,
+         #gas=tx_new['gas'],
+         gas=gas_total,
          tx_hashes_hex=tx_hashes, 
          queue=queue,
         )
@@ -132,7 +143,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_send',
         [
             chain_spec.asdict(),
-            tx_new['from'],
+            address,
+            #tx_new['from'],
             ],
         queue=queue,
         )
@@ -140,7 +152,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_queue',
         [
             chain_spec.asdict(),
-            tx_new['from'],
+            address,
+            #tx_new['from'],
             ],
         queue=queue,
         )

apps/cic-eth/cic_eth/admin/token.py

@@ -1,21 +1,2 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
 # local imports
-from cic_eth.task import BaseTask
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-            'symbol': self.default_token_symbol,
-            'address': self.default_token_address,
-            'name': self.default_token_name,
-            'decimals': self.default_token_decimals,
-            }
+from cic_eth.eth.erc20 import default_token

apps/cic-eth/cic_eth/api/admin.py

@@ -21,6 +21,7 @@ from chainlib.hash import keccak256_hex_to_hex
 from hexathon import (
         strip_0x,
         add_0x,
+        uniform as hex_uniform,
         )
 from chainlib.eth.gas import balance
 from chainqueue.db.enum import (
@@ -31,7 +32,6 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
 from eth_erc20 import ERC20
 
 # local imports
@@ -39,6 +39,7 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
+from cic_eth.queue.query import get_tx_local
 
 app = celery.current_app
 
@@ -283,7 +284,7 @@ class AdminApi:
         tx_hash_hex = None
         session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
+            tx_dict = get_tx_local(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
         session.close()
@@ -307,6 +308,8 @@ class AdminApi:
         :param address: Ethereum address to return transactions for
         :type address: str, 0x-hex
         """
+
+        address = add_0x(hex_uniform(strip_0x(address)))
         last_nonce = -1
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',

apps/cic-eth/cic_eth/api/api_task.py

@@ -9,6 +9,7 @@ import logging
 # external imports 
 import celery
 from chainlib.chain import ChainSpec
+from hexathon import strip_0x
 
 # local imports
 from cic_eth.api.base import ApiBase
@@ -16,15 +17,50 @@ from cic_eth.enum import LockEnum
 
 app = celery.current_app
 
-logg = logging.getLogger(__name__)
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 class Api(ApiBase):
-    
+
+    @staticmethod
+    def to_v_list(v, n):
+        """Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
+
+        :param v: Arguments
+        :type v: str or list
+        :param n: Number of elements to generate arguments for
+        :type n: int
+        :rtype: list
+        :returns: list of assembled arguments
+        """
+        if isinstance(v, str):
+            vv = v
+            v = []
+            for i in range(n):
+                v.append([vv])
+        elif not isinstance(v, list):
+            raise ValueError('argument must be single string, or list or strings or lists')
+        else:
+            if len(v) != n:
+                raise ValueError('v argument count must match integer n')
+            for i in range(n):
+                if isinstance(v[i], str):
+                    v[i] = [v[i]]
+                elif not isinstance(v, list):
+                    raise ValueError('proof argument must be single string, or list or strings or lists')
+
+        return v
+   
 
     def default_token(self):
+        """Retrieves the default fallback token of the custodial network.
+
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
         s_token = celery.signature(
-                'cic_eth.admin.token.default_token',
+                'cic_eth.eth.erc20.default_token',
                 [],
                 queue=self.queue,
                 )
@@ -34,6 +70,97 @@ class Api(ApiBase):
         return s_token.apply_async()
 
 
+    def token(self, token_symbol, proof=None):
+        """Single-token alias for tokens method.
+
+        See tokens method for details.
+
+        :param token_symbol: Token symbol to look up
+        :type token_symbol: str
+        :param proof: Proofs to add to signature verification for the token
+        :type proof: str or list
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        if not isinstance(token_symbol, str):
+            raise ValueError('token symbol must be string')
+
+        return self.tokens([token_symbol], proof=proof)
+
+
+    def tokens(self, token_symbols, proof=None):
+        """Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
+
+        Token symbols are always strings, and should be specified using uppercase letters.
+
+        If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
+
+        The proof argument may be specified in a number of ways:
+
+        - as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
+        - as a single string, where the same proof is used for each token lookup
+        - as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
+        - as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
+
+        The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
+
+        This method is not intended to be used synchronously. Do so at your peril.
+
+        :param token_symbols: Token symbol strings to look up
+        :type token_symbol: list
+        :param proof: Proof(s) to verify tokens against
+        :type proof: None, str or list
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        if not isinstance(token_symbols, list):
+            raise ValueError('token symbols argument must be list')
+
+        if proof == None:
+            logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
+            proof = ''
+
+        logg.debug('proof is {}'.format(proof))
+        l = len(token_symbols)
+        if len(proof) == 0:
+            l = 0 
+        proof = Api.to_v_list(proof, l)
+
+        chain_spec_dict = self.chain_spec.asdict()
+
+        s_token_resolve = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    token_symbols,
+                    chain_spec_dict,
+                    ],
+                queue=self.queue,
+                )
+
+        s_token_info = celery.signature(
+                'cic_eth.eth.erc20.token_info',
+                [
+                    chain_spec_dict,
+                    proof,
+                    ],
+                queue=self.queue,
+                )
+
+        s_token_verify = celery.signature(
+                    'cic_eth.eth.erc20.verify_token_info',
+                    [
+                        chain_spec_dict,
+                        self.callback_success,
+                        self.callback_error,
+                        ],
+                    queue=self.queue,
+                    )
+
+        s_token_info.link(s_token_verify)
+        s_token_resolve.link(s_token_info)
+        return s_token_resolve.apply_async()
+
+
 #    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
 #        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 #
@@ -254,6 +381,8 @@ class Api(ApiBase):
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        #from_address = strip_0x(from_address)
+        #to_address = strip_0x(to_address)
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -520,9 +649,9 @@ class Api(ApiBase):
             s_external_get = celery.signature(
                 external_task,
                 [
-                    address,
                     offset,
                     limit,
+                    address,
                     ],
                 queue=external_queue,
                 )
@@ -554,3 +683,4 @@ class Api(ApiBase):
 
         t = self.callback_success.apply_async([r])
         return t
+

apps/cic-eth/cic_eth/callbacks/noop.py

@@ -1,7 +1,10 @@
+import logging
+
 import celery
 
 celery_app = celery.current_app
-logg = celery_app.log.get_default_logger()
+#logg = celery_app.log.get_default_logger()
+logg = logging.getLogger()
 
 
 @celery_app.task(bind=True)

apps/cic-eth/cic_eth/check/gas.py

@@ -21,7 +21,7 @@ def health(*args, **kwargs):
     session = SessionBase.create_session()
 
     config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
     logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
 
     try:

apps/cic-eth/cic_eth/check/signer.py

@@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__)
 def health(*args, **kwargs):
     blocked = True
     max_attempts = 5
-    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
+    conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer')
     for i in range(max_attempts):
         idx = i + 1
         logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))

apps/cic-eth/cic_eth/cli/__init__.py

@@ -0,0 +1,10 @@
+# local imports
+from .base import *
+from .chain import (
+        EthChainInterface,
+        chain_interface,
+        )
+from .rpc import RPC
+from .arg import ArgumentParser
+from .config import Config
+from .celery import CeleryApp

apps/cic-eth/cic_eth/cli/arg.py

@@ -0,0 +1,31 @@
+# external imports
+from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
+
+# local imports
+from .base import (
+        CICFlag,
+        Flag,
+        )
+
+
+class ArgumentParser(BaseArgumentParser):
+
+    def process_local_flags(self, local_arg_flags):
+        if local_arg_flags & CICFlag.REDIS:
+            self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+            self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+            self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+            self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
+        if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
+        if local_arg_flags & CICFlag.SYNCER:
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+        if local_arg_flags & CICFlag.CHAIN:
+            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
+
+
+

apps/cic-eth/cic_eth/cli/base.py

@@ -0,0 +1,31 @@
+# standard imports
+import enum
+
+# external imports
+from chainlib.eth.cli import (
+    argflag_std_read,
+    argflag_std_write,
+    argflag_std_base,
+    Flag,
+    )
+
+class CICFlag(enum.IntEnum):
+   
+    # celery - nibble 1
+    CELERY = 1
+
+    # redis - nibble 2
+    REDIS = 16
+    REDIS_CALLBACK = 32
+
+    # chain - nibble 3
+    CHAIN = 256
+
+    # sync - nibble 4
+    SYNCER = 4096
+
+
+argflag_local_task = CICFlag.CELERY 
+argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
+argflag_local_chain = CICFlag.CHAIN
+argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-eth/cic_eth/cli/celery.py

@@ -0,0 +1,24 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
+logg = logging.getLogger(__name__)
+
+
+class CeleryApp:
+   
+    @classmethod
+    def from_config(cls, config):
+        backend_url = config.get('CELERY_RESULT_URL')
+        broker_url = config.get('CELERY_BROKER_URL')
+        celery_app = None
+        if backend_url != None:
+            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
+            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
+        else:
+            celery_app = celery.Celery(broker=broker_url)
+            logg.info('creating celery app without results backend on {}'.format(broker_url))
+
+        return celery_app

apps/cic-eth/cic_eth/cli/chain.py

@@ -0,0 +1,21 @@
+# external imports
+from chainlib.eth.block import (
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
+
+
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()

apps/cic-eth/cic_eth/cli/config.py

@@ -0,0 +1,63 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.cli import (
+        Config as BaseConfig,
+        Flag,
+        )
+
+# local imports
+from .base import CICFlag
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+logg = logging.getLogger(__name__)
+
+
+class Config(BaseConfig):
+
+    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
+
+    @classmethod
+    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
+        expanded_base_config_dir = [cls.local_base_config_dir]
+        if base_config_dir != None:
+            if isinstance(base_config_dir, str):
+                base_config_dir = [base_config_dir]
+            for d in base_config_dir:
+                expanded_base_config_dir.append(d)
+        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
+
+        local_args_override = {}
+        if local_arg_flags & CICFlag.REDIS:
+            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
+            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
+            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
+            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
+
+        if local_arg_flags & CICFlag.CHAIN:
+            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
+
+        if local_arg_flags & CICFlag.CELERY:
+            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
+
+        if local_arg_flags & CICFlag.SYNCER:
+            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
+            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
+
+        config.dict_override(local_args_override, 'local cli args')
+
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
+            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
+
+        if local_arg_flags & CICFlag.CELERY:
+            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
+
+        logg.debug('config loaded:\n{}'.format(config))
+
+        return config
+
+

apps/cic-eth/cic_eth/cli/rpc.py

@@ -0,0 +1,90 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import (
+        RPCConnection,
+        ConnType,
+        )
+from chainlib.eth.connection import (
+        EthUnixSignerConnection,
+        EthHTTPSignerConnection,
+        )
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+
+class RPC:
+
+    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
+        self.chain_spec = chain_spec
+        self.rpc_provider = rpc_provider
+        self.signer_provider = signer_provider
+
+
+    def get_default(self):
+        return self.get_by_label('default')
+
+
+    def get_by_label(self, label):
+        return RPCConnection.connect(self.chain_spec, label)
+
+
+    @staticmethod
+    def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
+        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
+        if use_signer:
+
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
+            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
+            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
+        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        logg.info('set up rpc: {}'.format(rpc))
+        return rpc
+
+
+    def __str__(self):
+        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
+
+
+# TOOD: re-implement file backend option from omittec code:
+#broker = config.get('CELERY_BROKER_URL')
+#if broker[:4] == 'file':
+#    bq = tempfile.mkdtemp()
+#    bp = tempfile.mkdtemp()
+#    conf_update = {
+#            'broker_url': broker,
+#            'broker_transport_options': {
+#                'data_folder_in': bq,
+#                'data_folder_out': bq,
+#                'data_folder_processed': bp,
+#            },
+#            }
+#    if config.true('CELERY_DEBUG'):
+#        conf_update['result_extended'] = True
+#    current_app.conf.update(conf_update)
+#    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
+#else:
+#    conf_update = {
+#            'broker_url': broker,
+#            }
+#    if config.true('CELERY_DEBUG'):
+#        conf_update['result_extended'] = True
+#    current_app.conf.update(conf_update)
+#
+#result = config.get('CELERY_RESULT_URL')
+#if result[:4] == 'file':
+#    rq = tempfile.mkdtemp()
+#    current_app.conf.update({
+#        'result_backend': 'file://{}'.format(rq),
+#        })
+#    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
+#else:
+#    current_app.conf.update({
+#        'result_backend': result,
+#        })
+#

apps/cic-eth/cic_eth/data/config/celery.ini

@@ -0,0 +1,5 @@
+[celery]
+broker_url = redis://localhost:6379
+result_url = 
+queue = cic-eth
+debug = 0

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -1,8 +1,6 @@
 [cic]
 registry_address = 
-chain_spec = evm:bloxberg:8996
-tx_retry_delay = 
 trust_address =
-default_token_symbol = GFT
+default_token_symbol =
 health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
 run_dir = /run

apps/cic-eth/cic_eth/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine =
+driver =
+host =
+port =
+name =
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-eth/cic_eth/data/config/dispatcher.ini

@@ -1,2 +1,2 @@
-[SYNCER]
+[dispatcher]
 loop_interval = 1

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,3 +1,2 @@
 [eth]
-provider = http://localhost:8545
 gas_gifter_minimum_balance = 10000000000000000000000

apps/cic-eth/cic_eth/data/config/redis.ini

@@ -0,0 +1,5 @@
+[redis]
+host = localhost
+port = 6379
+db = 0
+timeout = 20.0

apps/cic-eth/cic_eth/data/config/retry.ini

@@ -0,0 +1,3 @@
+[retry]
+delay =
+batch_size =

apps/cic-eth/cic_eth/data/config/signer.ini

@@ -0,0 +1,2 @@
+[signer]
+provider =

apps/cic-eth/cic_eth/data/config/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+offset = 0
+no_history = 0