improvement: updated folder structure

.gitignore

@@ -15,3 +15,4 @@ build/
 .idea
 **/.vim
 **/*secret.yaml
+.env

.gitlab-ci.yml

@@ -10,7 +10,6 @@ include:
     #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
-  - version
   - build
   - test
   - deploy
@@ -21,41 +20,9 @@ variables:
   DOCKER_BUILDKIT: "1"
   COMPOSE_DOCKER_CLI_BUILD: "1"
   CI_DEBUG_TRACE: "true"
-  SEMVERBOT_VERSION: "0.2.0"
 
-  #before_script:
+before_script:
-  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-
-version:
-  #image: python:3.7-stretch
-  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
-  stage: version
-  tags:
-    - integration
-  script:
-    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
-    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
-    - eval $(ssh-agent -s)
-    - ssh-add <(echo "$SSH_PRIVATE_KEY")
-    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
-    - export TAG=$(sbot predict version -m auto)
-    - |
-      if [[ -z $TAG ]]
-      then
-        echo "tag could not be set $@"
-        exit 1
-      fi
-    - echo $TAG > version
-    - git tag -a v$TAG -m "ci tagged"
-    - git push origin v$TAG
-  artifacts:
-    paths:
-      - version
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: always
 
 # runs on protected branches and pushes to repo
 build-push:
@@ -63,17 +30,12 @@ build-push:
   tags:
     - integration
   #script:
-  #  - TAG=$CI_Cbefore_script:
+  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - TAG=latest ./scripts/build-push.sh
+    - TAG=latest sh ./scripts/build-push.sh
-    - TAG=$(cat ./version) ./scripts/build-push.sh
   rules:
   - if: $CI_COMMIT_REF_PROTECTED == "true"
     when: always
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: always
 
 deploy-dev:
   stage: deploy

.gitlab/issue_templates/bug.md

@@ -1,44 +0,0 @@
-<!---
-Please read this!
-
-Before opening a new issue, make sure to search for keywords in the issues
-filtered by the "bug" label:
-
-- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
-
-and verify the issue you're about to submit isn't a duplicate.
---->
-
-### Summary
-
-<!-- Summarize the bug encountered concisely. -->
-
-### Steps to reproduce
-
-<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
-
-### Example Project
-
-<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic 
-behavior, and link to it here in the bug report. If you are using an older version of GitLab, this 
-will also determine whether the bug is fixed in a more recent version. -->
-
-### What is the current *bug* behavior?
-
-<!-- Describe what actually happens. -->
-
-### What is the expected *correct* behavior?
-
-<!-- Describe what you should see instead. -->
-
-### Relevant logs and/or screenshots
-
-<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
- as it's tough to read otherwise. -->
-
-
-### Possible fixes
-
-<!-- If you can, link to the line of code that might be responsible for the problem. -->
-
-/label ~"bug"

.semverbot.toml

@@ -1,16 +0,0 @@
-[git]
-
-[git.config]
-email = "semverbot@grassroots.org"
-name = "semvervot"
-
-[git.tags]
-prefix = "v"
-
-[semver]
-mode = "git-commit"
-
-[semver.detection]
-patch = ["fix", "[fix]", "patch", "[patch]"]
-minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
-major = ["BREAKING CHANGE"]

CODE_OF_CONDUCT.md

@@ -1,83 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
-
-
-
-
-...Try to keep in mind the immortal
-words of Bill and Ted, "Be excellent to each other."
-

CONTRIBUTING.md

@@ -1,16 +0,0 @@
-Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider. 
-
-__To request a change to the code please fork this repository and sumbit a merge request.__
-
-__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
-
-__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
-
-__Ask a question in our dev chat:__
-
-[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
-
-[Discord](https://discord.gg/XWunwAsX)
-
-[Matrix, IRC soon?]
-

CONTRIBUTING_CORE.md

@@ -1,117 +0,0 @@
-# CORE TEAM CONTRIBUTION GUIDE
-
-# 1. Transparency
-
-1.1 Use work logs for reflection of work done, aswell as telling your peers about changes that may affect their own tasks
-
-1.2 A work log SHOULD be submitted after a "unit of work" is complete.
-
-1.2.1 A "unit of work" should not span more than one full day's worth of work.
-
-1.2.2 A "unit of work" should be small enough that the log entries give useful insight.
-
-1.3 Individual logs are reviewed in weekly meetings
-
-<!--1.4 Bullet point list of topics and one or more sub-points describing each item in short sentences, eg;
-
-```
-- Core
-	* fixed foo
-	* fixed bar
-- Frontend
-	* connected bar to baz
-
-```-->
-
-1.4 Work log format is defined in []()
-
-1.5 Link to issue/MR in bullet point where appropriate
-
-1.6 
-
-
-# 2. Code hygiene
-
-2.1 Keep function names and variable names short
-
-2.2 Keep code files, functions and test fixtures short
-
-2.3 The less magic the better. Recombinable and replaceable is king
-
-2.4 Group imports by `standard`, `external`, `local`, `test` - in that order
-
-2.5 Only auto-import when necessary, and always with a minimum of side-effects
-
-2.6 Use custom errors. Let them bubble up
-
-2.7 No logs in tight loops
-
-2.8 Keep executable main routine minimal. Pass variables (do not use globals) in main business logic function
-
-2.9 Test coverage MUST be kept higher than 90% after changes
-
-2.10 Docstrings. Always. Always!
-
-
-# 3. Versioning
-
-3.1 Use [Semantic Versioning](https://semver.org/)
-
-3.2 When merging code, explicit dependencies SHOULD NOT use pre-release version
-
-
-# 4. Issues
-
-4.1 Issue title should use [Convention Commit structure](https://www.conventionalcommits.org/en/v1.0.0-beta.2/)
-
-4.2 Issues need proper problem statement
-
-4.2.1. What is the current state
-
-4.2.2. If current state is not behaving as expected, what was the expected state
-
-4.2.3. What is the desired new state.
-
-4.3  Issues need proper resolution statement
-
-4.3.1. Bullet point list of short sentences describing practical steps to reach desired state
-		
-4.3.2. Builet point list of external resources informing the issue and resolution
-
-4.4 Tasks needs to be appropriately labelled using GROUP labels.
-
-
-# 5. Code submission
-
-5.1 A branch and new MR is always created BEFORE THE WORK STARTS
-
-5.2 An MR should solve ONE SINGLE PART of a problem
-
-5.3 Every MR should have at least ONE ISSUE associated with it. Ideally issue can be closed when MR is merged
-
-5.4 MRs should not be open for more than one week (during normal operation periods)
-
-5.5 MR should ideally not be longer than 400 lines of changes of logic
-
-5.6 MRs that MOVE or DELETE code should not CHANGE that same code in a single MR. Scope MOVEs and DELETEs in separate commits (or even better, separate MRs) for transparency
-
-
-# 6. Code reviews
-
-6.1 At least one peer review before merge
-
-6.2 If MR is too long, evaluate whether this affects the quality of the review negatively. If it does, expect to be asked to split it up
-
-6.3 Evaluate changes against associated issues' problem statement and proposed resolution steps. If there is a mismatch, either MR needs to change or issue needs to be amended accordingly
-
-6.4 Make sure all technical debt introduced by MR is documented in issues. Add them according to criteria in section ISSUES if not
-
-6.5 If CI is not working, reviewer MUST make sure code builds and runs
-
-6.6 Behave!
-
-6.6.1 Don't be a jerk
-
-6.6.2 Don't block needlessly
-
-6.6.3 Say please

README.md

@@ -1,9 +1,11 @@
-# Community Inclusion Currency Stack (CIC Stack) 
+# cic-internal-integration
-
-A custodial evm wallet for executing transactions via USSD
 
 ## Getting started 
 
+### Preperation
+
+Copy the .env_sample file to .env and populate with appropriate env vars
+
 This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
 ```
@@ -11,9 +13,32 @@ export COMPOSE_DOCKER_CLI_BUILD=1
 export DOCKER_BUILDKIT=1
 ```
 
-To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
+start services, database, redis and local ethereum node
+```
+docker-compose up -d
+```
 
-## Documentation 
+Run app/contract-migration to deploy contracts
+```
+RUN_MASK=3 docker-compose up contract-migration
+```
 
-[https://docs.grassecon.org/software/](https://docs.grassecon.org/software/)
+stop cluster
+```
+docker-compose down
+```
 
+stop cluster and delete data
+```
+docker-compose down -v --remove-orphans
+```
+
+rebuild an images
+```
+docker-compose up --build <service_name>
+```
+
+to delete the buildkit cache
+```
+docker builder prune --filter type=exec.cachemount
+```

apps/cic-base-os/aux/wait-for-it/.gitignore

@@ -1,3 +0,0 @@
-**/*.pyc
-.pydevproject
-/vendor/

apps/cic-base-os/aux/wait-for-it/.travis.yml

@@ -1,7 +0,0 @@
-language: python
-python:
-  - "2.7"
-
-script:
-  - python test/wait-for-it.py
-

apps/cic-base-os/aux/wait-for-it/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-Copyright (c) 2016 Giles Hall
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

apps/cic-base-os/aux/wait-for-it/README.md

@@ -1,75 +0,0 @@
-# wait-for-it
-
-`wait-for-it.sh` is a pure bash script that will wait on the availability of a
-host and TCP port.  It is useful for synchronizing the spin-up of
-interdependent services, such as linked docker containers.  Since it is a pure
-bash script, it does not have any external dependencies.
-
-## Usage
-
-```text
-wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
--h HOST | --host=HOST       Host or IP under test
--p PORT | --port=PORT       TCP port under test
-                            Alternatively, you specify the host and port as host:port
--s | --strict               Only execute subcommand if the test succeeds
--q | --quiet                Don't output any status messages
--t TIMEOUT | --timeout=TIMEOUT
-                            Timeout in seconds, zero for no timeout
--- COMMAND ARGS             Execute command with args after the test finishes
-```
-
-## Examples
-
-For example, let's test to see if we can access port 80 on `www.google.com`,
-and if it is available, echo the message `google is up`.
-
-```text
-$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-You can set your own timeout with the `-t` or `--timeout=` option.  Setting
-the timeout value to 0 will disable the timeout:
-
-```text
-$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting for www.google.com:80 without a timeout
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-The subcommand will be executed regardless if the service is up or not.  If you
-wish to execute the subcommand only if the service is up, add the `--strict`
-argument. In this example, we will test port 81 on `www.google.com` which will
-fail:
-
-```text
-$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
-wait-for-it.sh: waiting 1 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
-wait-for-it.sh: strict mode, refusing to execute subprocess
-```
-
-If you don't want to execute a subcommand, leave off the `--` argument.  This
-way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
-and determine how to proceed:
-
-```text
-$ ./wait-for-it.sh www.google.com:80
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-$ echo $?
-0
-$ ./wait-for-it.sh www.google.com:81
-wait-for-it.sh: waiting 15 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
-$ echo $?
-124
-```
-
-## Community
-
-*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-base-os/aux/wait-for-it/wait-for-it.sh

@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-# Use this script to test if a given TCP host/port are available
-
-WAITFORIT_cmdname=${0##*/}
-
-echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
-
-usage()
-{
-    cat << USAGE >&2
-Usage:
-    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-    -h HOST | --host=HOST       Host or IP under test
-    -p PORT | --port=PORT       TCP port under test
-                                Alternatively, you specify the host and port as host:port
-    -s | --strict               Only execute subcommand if the test succeeds
-    -q | --quiet                Don't output any status messages
-    -t TIMEOUT | --timeout=TIMEOUT
-                                Timeout in seconds, zero for no timeout
-    -- COMMAND ARGS             Execute command with args after the test finishes
-USAGE
-    exit 1
-}
-
-wait_for()
-{
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    else
-        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
-    fi
-    WAITFORIT_start_ts=$(date +%s)
-    while :
-    do
-        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
-            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
-            WAITFORIT_result=$?
-        else
-            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
-            WAITFORIT_result=$?
-        fi
-        if [[ $WAITFORIT_result -eq 0 ]]; then
-            WAITFORIT_end_ts=$(date +%s)
-            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
-            break
-        fi
-        sleep 1
-    done
-    return $WAITFORIT_result
-}
-
-wait_for_wrapper()
-{
-    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
-    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    else
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    fi
-    WAITFORIT_PID=$!
-    trap "kill -INT -$WAITFORIT_PID" INT
-    wait $WAITFORIT_PID
-    WAITFORIT_RESULT=$?
-    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    fi
-    return $WAITFORIT_RESULT
-}
-
-# process arguments
-while [[ $# -gt 0 ]]
-do
-    case "$1" in
-        *:* )
-        WAITFORIT_hostport=(${1//:/ })
-        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
-        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
-        shift 1
-        ;;
-        --child)
-        WAITFORIT_CHILD=1
-        shift 1
-        ;;
-        -q | --quiet)
-        WAITFORIT_QUIET=1
-        shift 1
-        ;;
-        -s | --strict)
-        WAITFORIT_STRICT=1
-        shift 1
-        ;;
-        -h)
-        WAITFORIT_HOST="$2"
-        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
-        shift 2
-        ;;
-        --host=*)
-        WAITFORIT_HOST="${1#*=}"
-        shift 1
-        ;;
-        -p)
-        WAITFORIT_PORT="$2"
-        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --port=*)
-        WAITFORIT_PORT="${1#*=}"
-        shift 1
-        ;;
-        -t)
-        WAITFORIT_TIMEOUT="$2"
-        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --timeout=*)
-        WAITFORIT_TIMEOUT="${1#*=}"
-        shift 1
-        ;;
-        --)
-        shift
-        WAITFORIT_CLI=("$@")
-        break
-        ;;
-        --help)
-        usage
-        ;;
-        *)
-        echoerr "Unknown argument: $1"
-        usage
-        ;;
-    esac
-done
-
-if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
-    echoerr "Error: you need to provide a host and port to test."
-    usage
-fi
-
-WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
-WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
-WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
-WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
-
-# Check to see if timeout is from busybox?
-WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
-WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
-
-WAITFORIT_BUSYTIMEFLAG=""
-if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
-    WAITFORIT_ISBUSY=1
-    # Check if busybox timeout uses -t flag
-    # (recent Alpine versions don't support -t anymore)
-    if timeout &>/dev/stdout | grep -q -e '-t '; then
-        WAITFORIT_BUSYTIMEFLAG="-t"
-    fi
-else
-    WAITFORIT_ISBUSY=0
-fi
-
-if [[ $WAITFORIT_CHILD -gt 0 ]]; then
-    wait_for
-    WAITFORIT_RESULT=$?
-    exit $WAITFORIT_RESULT
-else
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        wait_for_wrapper
-        WAITFORIT_RESULT=$?
-    else
-        wait_for
-        WAITFORIT_RESULT=$?
-    fi
-fi
-
-if [[ $WAITFORIT_CLI != "" ]]; then
-    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
-        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
-        exit $WAITFORIT_RESULT
-    fi
-    exec "${WAITFORIT_CLI[@]}"
-else
-    exit $WAITFORIT_RESULT
-fi

apps/cic-cache/MANIFEST.in

@@ -1 +1 @@
-include *requirements.txt cic_cache/data/config/* cic_cache/db/migrations/default/* cic_cache/db/migrations/default/versions/*
+include *requirements.txt cic_cache/data/config/*

apps/cic-cache/aux/wait-for-it/.gitignore

@@ -1,3 +0,0 @@
-**/*.pyc
-.pydevproject
-/vendor/

apps/cic-cache/aux/wait-for-it/.travis.yml

@@ -1,7 +0,0 @@
-language: python
-python:
-  - "2.7"
-
-script:
-  - python test/wait-for-it.py
-

apps/cic-cache/aux/wait-for-it/README.md

@@ -1,75 +0,0 @@
-# wait-for-it
-
-`wait-for-it.sh` is a pure bash script that will wait on the availability of a
-host and TCP port.  It is useful for synchronizing the spin-up of
-interdependent services, such as linked docker containers.  Since it is a pure
-bash script, it does not have any external dependencies.
-
-## Usage
-
-```text
-wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
--h HOST | --host=HOST       Host or IP under test
--p PORT | --port=PORT       TCP port under test
-                            Alternatively, you specify the host and port as host:port
--s | --strict               Only execute subcommand if the test succeeds
--q | --quiet                Don't output any status messages
--t TIMEOUT | --timeout=TIMEOUT
-                            Timeout in seconds, zero for no timeout
--- COMMAND ARGS             Execute command with args after the test finishes
-```
-
-## Examples
-
-For example, let's test to see if we can access port 80 on `www.google.com`,
-and if it is available, echo the message `google is up`.
-
-```text
-$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-You can set your own timeout with the `-t` or `--timeout=` option.  Setting
-the timeout value to 0 will disable the timeout:
-
-```text
-$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting for www.google.com:80 without a timeout
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-The subcommand will be executed regardless if the service is up or not.  If you
-wish to execute the subcommand only if the service is up, add the `--strict`
-argument. In this example, we will test port 81 on `www.google.com` which will
-fail:
-
-```text
-$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
-wait-for-it.sh: waiting 1 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
-wait-for-it.sh: strict mode, refusing to execute subprocess
-```
-
-If you don't want to execute a subcommand, leave off the `--` argument.  This
-way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
-and determine how to proceed:
-
-```text
-$ ./wait-for-it.sh www.google.com:80
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-$ echo $?
-0
-$ ./wait-for-it.sh www.google.com:81
-wait-for-it.sh: waiting 15 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
-$ echo $?
-124
-```
-
-## Community
-
-*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-cache/cic_cache/cli/arg.py

@@ -14,7 +14,7 @@ class ArgumentParser(BaseArgumentParser):
         if local_arg_flags & CICFlag.CELERY:
             self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, help='Start block height for initial history sync')
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
         if local_arg_flags & CICFlag.CHAIN:
             self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -1,4 +1,4 @@
 [cic]
 registry_address = 
 trust_address =
-health_modules =
+health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas

apps/cic-cache/cic_cache/data/config/database.ini

@@ -3,8 +3,7 @@ engine =
 driver =
 host =
 port =
-#name = cic-cache
+name = cic-cache
-prefix =
 user =
 password =
 debug = 0

apps/cic-cache/cic_cache/db/__init__.py

@@ -9,26 +9,21 @@ from .list import (
         tag_transaction,
         add_tag,
     )
-from cic_cache.db.models.base import SessionBase
 
 
 logg = logging.getLogger()
 
 
-def dsn_from_config(config, name):
+def dsn_from_config(config):
     scheme = config.get('DATABASE_ENGINE')
     if config.get('DATABASE_DRIVER') != None:
         scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
 
-    database_name = name
-    if config.get('DATABASE_PREFIX'):
-        database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
     dsn = ''
     if config.get('DATABASE_ENGINE') == 'sqlite':
-        SessionBase.poolable = False
         dsn = '{}:///{}'.format(
                 scheme,
-                database_name,
+                config.get('DATABASE_NAME'),    
             )
 
     else:
@@ -38,7 +33,7 @@ def dsn_from_config(config, name):
                 config.get('DATABASE_PASSWORD'),
                 config.get('DATABASE_HOST'),
                 config.get('DATABASE_PORT'),
-                database_name,
+                config.get('DATABASE_NAME'),    
             )
     logg.debug('parsed dsn from config: {}'.format(dsn))
     return dsn

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -5,11 +5,7 @@ import re
 import base64
 
 # external imports
-from hexathon import (
+from hexathon import add_0x
-        add_0x,
-        strip_0x,
-        )
-from chainlib.encode import TxHexNormalizer
 
 # local imports
 from cic_cache.cache import (
@@ -20,72 +16,27 @@ from cic_cache.cache import (
 logg = logging.getLogger(__name__)
 #logg = logging.getLogger()
 
-re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
+re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
 re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
-re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
+re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
-re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
-re_default_limit = r'/defaultlimit/?'
 
 DEFAULT_LIMIT = 100
 
-tx_normalize = TxHexNormalizer()
-
-def parse_query_account(r):
-    address = strip_0x(r[1])
-    #address = tx_normalize.wallet_address(address)
-    limit = DEFAULT_LIMIT
-    g = r.groups()
-    if len(g) > 3:
-        limit = int(r[4])
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-    offset = 0
-    if len(g) > 4:
-        offset = int(r[6])
-
-    logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
-
-    return (address, offset, limit,)
-
-
-# r is an re.Match
-def parse_query_any(r):
-    limit = DEFAULT_LIMIT
-    offset = 0
-    block_offset = None
-    block_end = None
-    if r.lastindex != None:
-        if r.lastindex > 0:
-            limit = int(r[1])
-        if r.lastindex > 1:
-            offset = int(r[2])
-        if r.lastindex > 2:
-            block_offset = int(r[3])
-        if r.lastindex > 3:
-            block_end = int(r[4])
-            if block_end < block_offset:
-                raise ValueError('cart before the horse, dude')
-
-    logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
-
-    return (offset, limit, block_offset, block_end,)
-
-
-def process_default_limit(session, env):
-    r = re.match(re_default_limit, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
-
 
 def process_transactions_account_bloom(session, env):
     r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
     if not r:
         return None
-    logg.debug('match account bloom')
 
-    (address, offset, limit,) = parse_query_account(r)
+    address = r[1]
+    if r[2] == None:
+        address = add_0x(address)
+    offset = 0
+    if r.lastindex > 2:
+        offset = r[4]
+    limit = DEFAULT_LIMIT
+    if r.lastindex > 4:
+        limit = r[6]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
@@ -108,9 +59,13 @@ def process_transactions_all_bloom(session, env):
     r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
     if not r:
         return None
-    logg.debug('match all bloom')
 
-    (limit, offset, block_offset, block_end,) = parse_query_any(r)
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 0:
+        offset = r[1]
+    limit = 0
+    if r.lastindex > 1:
+        limit = r[2]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
@@ -133,16 +88,17 @@ def process_transactions_all_data(session, env):
     r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
     if not r:
         return None
-    #if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-    #    return None
+        return None
-    logg.debug('match all data')
 
     logg.debug('got data request {}'.format(env))
-
+    block_offset = r[1]
-    (offset, limit, block_offset, block_end) = parse_query_any(r)
+    block_end = r[2]
+    if int(r[2]) < int(r[1]):
+        raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
 
     for r in tx_cache:
         r['date_block'] = r['date_block'].timestamp()
@@ -157,30 +113,3 @@ def process_transactions_all_data(session, env):
     j = json.dumps(o)
 
     return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_account_data(session, env):
-    r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
-    if not r:
-        return None
-    logg.debug('match account data')
-    #if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-    #    return None
-
-    (address, offset, limit,) = parse_query_account(r)
-
-    c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
-
-    for r in tx_cache:
-        r['date_block'] = r['date_block'].timestamp()
-
-    o = {
-        'low': lowest_block,
-        'high': highest_block,
-        'data': tx_cache,
-    }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -8,33 +8,42 @@ import base64
 import confini
 
 # local imports
-import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.runnable.daemons.query import (
-        process_default_limit,
         process_transactions_account_bloom,
-        process_transactions_account_data,
         process_transactions_all_bloom,
         process_transactions_all_data,
         )
-import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+dbdir = os.path.join(rootdir, 'cic_cache', 'db')
+migrationsdir = os.path.join(dbdir, 'migrations')
 
-arg_flags = cic_cache.cli.argflag_std_read
+config_dir = os.path.join('/usr/local/etc/cic-cache')
-local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task 
+
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser = argparse.ArgumentParser()
-argparser.process_local_flags(local_arg_flags)
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-# process config
+if args.vv:
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
 
-# connect to database
+config = confini.Config(args.c, args.env_prefix)
-dsn = dsn_from_config(config, 'cic_cache')
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
+
+dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 
 
@@ -46,11 +55,9 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
-            process_transactions_account_data,
-            process_transactions_account_bloom,
             process_transactions_all_data,
             process_transactions_all_bloom,
-            process_default_limit,
+            process_transactions_account_bloom,
             ]:
         r = None
         try:

apps/cic-cache/cic_cache/runnable/daemons/tasker.py

@@ -3,14 +3,12 @@ import logging
 import os
 import sys
 import argparse
-import tempfile
 
 # third-party imports
 import celery
 import confini
 
 # local imports
-import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.tasks.tx import *
@@ -18,20 +16,35 @@ from cic_cache.tasks.tx import *
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-# process args
+config_dir = os.path.join('/usr/local/etc/cic-cache')
-arg_flags = cic_cache.cli.argflag_std_base
+
-local_arg_flags = cic_cache.cli.argflag_local_task
+
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser = argparse.ArgumentParser()
-argparser.process_local_flags(local_arg_flags)
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+
 args = argparser.parse_args()
 
-# process config
+if args.vv:
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
 
 # connect to database
-dsn = dsn_from_config(config, 'cic_cache')
+dsn = dsn_from_config(config)
 SessionBase.connect(dsn)
 
+# verify database connection with minimal sanity query
+#session = SessionBase.create_session()
+#session.execute('select version_num from alembic_version')
+#session.close()
+
 # set up celery
 current_app = celery.Celery(__name__)
 
@@ -74,9 +87,9 @@ def main():
     elif args.v:
         argv.append('--loglevel=INFO')
     argv.append('-Q')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)
     argv.append('-n')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)
 
     current_app.worker_main(argv)
 

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -40,7 +40,7 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 # process args
-arg_flags = cic_cache.cli.argflag_std_base
+arg_flags = cic_cache.cli.argflag_std_read
 local_arg_flags = cic_cache.cli.argflag_local_sync
 argparser = cic_cache.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)
@@ -50,7 +50,7 @@ args = argparser.parse_args()
 config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
 # connect to database
-dsn = dsn_from_config(config, 'cic_cache')
+dsn = dsn_from_config(config)
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
 # set up rpc
@@ -95,10 +95,10 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = int(config.get('SYNCER_OFFSET'))
+        initial_block_start = config.get('SYNCER_OFFSET')
-        initial_block_offset = int(block_offset)
+        initial_block_offset = block_offset
         if config.get('SYNCER_NO_HISTORY'):
-            initial_block_start = initial_block_offset
+            initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
         logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))

apps/cic-cache/cic_cache/version.py

@@ -5,7 +5,7 @@ version = (
         0,
         2,
         1,
-        'alpha.3',
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-cache/config/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis:///
+result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -0,0 +1,3 @@
+[cic]
+registry_address = 
+trust_address = 

apps/cic-cache/config/database.ini

@@ -0,0 +1,9 @@
+[database]
+NAME=cic_cache
+USER=postgres
+PASSWORD=
+HOST=localhost
+PORT=5432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=0

apps/cic-cache/config/docker/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis://localhost:63379
+result_url = redis://localhost:63379

apps/cic-cache/config/docker/cic.ini

@@ -0,0 +1,3 @@
+[cic]
+registry_address =  
+trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -0,0 +1,9 @@
+[database]
+NAME=cic_cache
+USER=grassroots
+PASSWORD=
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=0

apps/cic-cache/config/docker/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+offset = 0
+no_history = 0

apps/cic-cache/config/test/bancor.ini

@@ -0,0 +1,2 @@
+[bancor]
+dir =

apps/cic-cache/config/test/cic.ini

@@ -1,3 +1,4 @@
 [cic]
 registry_address = 
+chain_spec = 
 trust_address =

apps/cic-cache/config/test/database.ini

@@ -1,5 +1,5 @@
 [database]
-PREFIX=cic-cache-test
+NAME=cic-cache-test
 USER=postgres
 PASSWORD=
 HOST=localhost

apps/cic-cache/config/test/eth.ini

@@ -0,0 +1,5 @@
+[eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
+provider = http://localhost:8545
+#chain_id = 

apps/cic-cache/doc/openapi/server.yml

@@ -1,4 +1,4 @@
-openapi: "3.0.2"
+openapi: "3.0.3"
 info:
   title: Grassroots Economics CIC Cache
   description: Cache of processed transaction data from Ethereum blockchain and worker queues
@@ -9,34 +9,17 @@ info:
     email: will@grassecon.org
   license:
     name: GPLv3
-  version: 0.2.0
+  version: 0.1.0
 
 paths:
-  /defaultlimit:
+  /tx/{offset}/{limit}:
-    summary: The default limit value of result sets.
+    description: Bloom filter for batch of latest transactions
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve default limit
-      operationId: limit.default
-      responses:
-        200:
-          description: Limit query successful
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Limit"
-
-  /tx:
-    summary: Bloom filter for batch of latest transactions
-    description: Generate a bloom filter of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
     get:
       tags:
         - transactions
       description:
         Retrieve transactions
-      operationId: tx.get.latest
+      operationId: tx.get
       responses:
         200:
           description: Transaction query successful.
@@ -46,109 +29,27 @@ paths:
                 $ref: "#/components/schemas/BlocksBloom"
 
 
-  /tx/{limit}:
-    summary: Bloom filter for batch of latest transactions
-    description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.latest.limit
-      responses:
-        200:
-          description: Transaction query successful. Results are ordered from newest to oldest.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
     parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{limit}/{offset}:
-    summary: Bloom filter for batch of latest transactions
-    description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.latest.range
-      responses:
-        200:
-          description: Transaction query successful. Results are ordered from newest to oldest.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
     - name: offset
       in: path
-      required: true
       schema:
         type: integer
         format: int32
-
-
-  /tx/{limit}/{offset}/{block_offset}:
-    summary: Bloom filter for batch of transactions since a particular block.
-    description: Generate a bloom filter of the latest transactions since a particular block in the cache. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.latest.range.block.offset
-      responses:
-        200:
-          description: Transaction query successful. Results are ordered from oldest to newest.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-    parameters:
     - name: limit
       in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
       schema:
         type: integer
         format: int32
 
 
-  /tx/{limit}/{offset}/{block_offset}/{block_end}:
+  /tx/{address}/{offset}/{limit}:
-    summary: Bloom filter for batch of transactions within a particular block range.
+    description: Bloom filter for batch of latest transactions by account
-    description: Generate a bloom filter of the latest transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
     get:
       tags:
         - transactions
       description:
         Retrieve transactions
-      operationId: tx.get.latest.range.block.range
+      operationId: tx.get
       responses:
         200:
           description: Transaction query successful. 
@@ -157,49 +58,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/BlocksBloom"
 
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_end
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{address}:
-    summary: Bloom filter for batch of latest transactions by account.
-    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.user
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
 
     parameters:
     - name: address
@@ -207,342 +65,26 @@ paths:
       required: true
       schema:
         type: string
-
-
-  /tx/{address}/{limit}:
-    summary: Bloom filter for batch of latest transactions by account.
-    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.user.limit
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{address}/{limit}/{offset}:
-    summary: Bloom filter for batch of latest transactions by account
-    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.user.range
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
     - name: offset
       in: path
-      required: true
       schema:
         type: integer
         format: int32
-
-
-  /txa:
-    summary: Cached data for latest transactions.
-    description: Return data entries of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    
-  /txa/{limit}:
-    summary: Cached data for latest transactions.
-    description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.limit
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
     - name: limit
       in: path
-      required: true
       schema:
         type: integer
         format: int32
 
-
-  /txa/{limit}/{offset}:
-    summary: Cached data for latest transactions.
-    description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.range
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /txa/{limit}/{offset}/{block_offset}:
-    summary: Cached data for transactions since a particular block.
-    description: Return cached data entries of transactions since a particular block. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.range.block.offset
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-  /txa/{limit}/{offset}/{block_offset}/{block_end}:
-    summary: Cached data for transactions within a particular block range.
-    description: Return cached data entries of transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.range.block.range
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_end
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /txa/{address}:
-    summary: Cached data for batch of latest transactions by account.
-    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.user
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-
-
-  /txa/{address}/{limit}:
-    summary: Cached data for batch of latest transactions by account.
-    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.user.limit
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /txa/{address}/{limit}/{offset}:
-    summary: Cached data for batch of latest transactions by account.
-    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.user.range
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
 components:
   schemas:
-    Limit:
-      type: integer
-      format: int32
     BlocksBloom:
       type: object
       properties:
         low:
-          type: integer
+          type: int
           format: int32
           description: The lowest block number included in the filter
-        high:
-          type: integer
-          format: int32
-          description: The highest block number included in the filter
         block_filter:
           type: string
           format: byte
@@ -555,89 +97,6 @@ components:
           type: string
           description: Hashing algorithm (currently only using sha256)
         filter_rounds:
-          type: integer
+          type: int
           format: int32
           description: Number of hash rounds used to create the filter
-    TransactionList:
-      type: object
-      properties:
-        low:
-          type: integer
-          format: int32
-          description: The lowest block number included in the result set
-        high:
-          type: integer
-          format: int32
-          description: The highest block number included in the filter
-        data:
-          type: array
-          description: Cached transaction data
-          items:
-            $ref: "#/components/schemas/Transaction"
-    Transaction:
-      type: object
-      properties:
-        block_number:
-          type: integer
-          format: int64
-          description: Block number transaction was included in.
-        tx_hash:
-          type: string
-          description: Transaction hash, in hex.
-        date_block:
-          type: integer
-          format: int32
-          description: Block timestamp.
-        sender:
-          type: string
-          description: Spender address, in hex.
-        recipient:
-          type: string
-          description: Beneficiary address, in hex.
-        from_value:
-          type: integer
-          format: int64
-          description: Value deducted from spender's balance.
-        to_value:
-          type: integer
-          format: int64
-          description: Value added to beneficiary's balance.
-        source_token:
-          type: string
-          description: Network address of token in which `from_value` is denominated.
-        destination_token:
-          type: string
-          description: Network address of token in which `to_value` is denominated.
-        success:
-          type: boolean
-          description: Network consensus state on whether the transaction was successful or not.
-        tx_type:
-          type: string
-          enum:
-            - erc20.faucet
-            - faucet.give_to
-
-  examples:
-    data_last:
-      summary: Get the latest cached transactions, using the server's default limit. 
-      value: "/txa"
-
-    data_limit:
-      summary: Get the last 42 cached transactions. 
-      value: "/txa/42"
-
-    data_range:
-      summary: Get the next 42 cached transactions, starting from the 13th (zero-indexed).
-      value: "/txa/42/13"
-
-    data_range_block_offset:
-      summary: Get the next 42 cached transactions, starting from block 1337 (inclusive).
-      value: "/txa/42/0/1337"
-
-    data_range_block_offset:
-      summary: Get the next 42 cached transactions within blocks 1337 and 1453 (inclusive).
-      value: "/txa/42/0/1337/1453"
-
-    data_range_block_range:
-      summary: Get the next 42 cached transactions after the 13th, within blocks 1337 and 1453 (inclusive).
-      value: "/txa/42/13/1337/1453"

apps/cic-cache/docker/Dockerfile

@@ -1,25 +1,32 @@
-ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
  
-FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
 COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
 
-ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 ARG EXTRA_PIP_ARGS=""
-ARG PIP_INDEX_URL=https://pypi.org/simple
-
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url $PIP_INDEX_URL \
+    pip install --index-url https://pypi.org/simple  \
-    --pre \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt
 
-
 COPY . . 
-RUN pip install .  --extra-index-url $EXTRA_PIP_INDEX_URL
+
+RUN python setup.py install
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-cache/
 
 # for db migrations
-COPY ./aux/wait-for-it/wait-for-it.sh ./
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
 COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
 COPY /docker/start_tracker.sh ./start_tracker.sh

apps/cic-cache/docker/db.sh

@@ -2,5 +2,5 @@
 
 set -e
 >&2 echo executing database migration
-python scripts/migrate_cic_cache.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
+python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
 set +e

apps/cic-cache/docker/run_tests.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-pip install --extra-index-url https://pip.grassrootseconomics.net \
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
 -r test_requirements.txt 
 

apps/cic-cache/requirements.txt

@@ -1,15 +1,14 @@
 alembic==1.4.2
-confini~=0.5.3
+confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
-moolb~=0.2.0
+moolb~=0.1.1b2
-cic-eth-registry~=0.6.6
+cic-eth-registry~=0.6.1a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.7
+chainsyncer[sql]>=0.0.6a3,<0.1.0
-erc20-faucet~=0.3.2
+erc20-faucet>=0.3.2a2, <0.4.0
-chainlib-eth~=0.0.15
+chainlib-eth>=0.0.9a14,<0.1.0
-eth-address-index~=0.2.4
+eth-address-index>=0.2.3a4,<0.3.0
-okota~=0.2.5

apps/cic-cache/scripts/migrate.py

@@ -1,55 +1,53 @@
-#!/usr/bin/python3
+#!/usr/bin/python
-
-# standard imports
 import os
 import argparse
 import logging
 import re
 
-# external imports
 import alembic
 from alembic.config import Config as AlembicConfig
 import confini
 
-# local imports
 from cic_cache.db import dsn_from_config
-import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 # BUG: the dbdir doesn't work after script install
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(cic_cache.__file__)))
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-default_migrations_dir = os.path.join(dbdir, 'migrations')
+migrationsdir = os.path.join(dbdir, 'migrations')
-configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
 
-#config_dir = os.path.join('/usr/local/etc/cic-cache')
+config_dir = os.path.join('/usr/local/etc/cic-cache')
 
-arg_flags = cic_cache.cli.argflag_std_base
+argparser = argparse.ArgumentParser()
-local_arg_flags = cic_cache.cli.argflag_local_sync
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.process_local_flags(local_arg_flags)
+argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', '--force', action='store_true', help='force action')
+argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='migrations directory')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-extra_args = {
+if args.vv:
-    'reset': None,
+    logging.getLogger().setLevel(logging.DEBUG)
-    'force': None,
+elif args.v:
-    'migrations_dir': None,
+    logging.getLogger().setLevel(logging.INFO)
-        }
-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
-migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE', 'default'))
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
+
+migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
 if not os.path.isdir(migrations_dir):
     logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
     migrations_dir = os.path.join(args.migrations_dir, 'default')
 
 # connect to database
-dsn = dsn_from_config(config, 'cic_cache')
+dsn = dsn_from_config(config)
 
 
 logg.info('using migrations dir {}'.format(migrations_dir))

apps/cic-cache/setup.cfg

@@ -1,7 +1,6 @@
 [metadata]
 name = cic-cache
 description = CIC Cache API and server
-version = 0.3.0a2
 author = Louis Holbrook
 author_email = dev@holbrook.no
 url = https://gitlab.com/grassrootseconomics/cic-eth
@@ -35,7 +34,7 @@ packages =
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters
 scripts =
-	./scripts/migrate_cic_cache.py
+	./scripts/migrate.py
 
 [options.entry_points]
 console_scripts =

apps/cic-cache/setup.py

@@ -1,39 +1,38 @@
 from setuptools import setup
 
-# import configparser
+import configparser
 import os
+import time
 
-# import time
+from cic_cache.version import (
+        version_object,
+        version_string
+        )
 
-# from cic_cache.version import (
+class PleaseCommitFirstError(Exception):
-#        version_object,
+    pass
-#        version_string
+
-#        )
+def git_hash():
-#
+    import subprocess
-# class PleaseCommitFirstError(Exception):
+    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
-#    pass
+    if len(git_diff.stdout) > 0:
-#
+        raise PleaseCommitFirstError()
-# def git_hash():
+    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-#    import subprocess
+    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-#    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
+    return git_hash_brief
-#    if len(git_diff.stdout) > 0:
+
-#        raise PleaseCommitFirstError()
+version_string = str(version_object)
-#    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+
-#    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+try:
-#    return git_hash_brief
+    version_git = git_hash()
-#
+    version_string += '+build.{}'.format(version_git)
-# version_string = str(version_object)
+except FileNotFoundError:
-#
+    time_string_pair = str(time.time()).split('.')
-# try:
+    version_string += '+build.{}{:<09d}'.format(
-#    version_git = git_hash()
+            time_string_pair[0],
-#    version_string += '+build.{}'.format(version_git)
+            int(time_string_pair[1]),
-# except FileNotFoundError:
+            )
-#    time_string_pair = str(time.time()).split('.')
+print('final version string will be {}'.format(version_string))
-#    version_string += '+build.{}{:<09d}'.format(
-#            time_string_pair[0],
-#            int(time_string_pair[1]),
-#            )
-# print('final version string will be {}'.format(version_string))
 
 requirements = []
 f = open('requirements.txt', 'r')
@@ -53,8 +52,9 @@ while True:
     test_requirements.append(l.rstrip())
 f.close()
 
+
 setup(
-    #    version=version_string,
+    version=version_string,
     install_requires=requirements,
     tests_require=test_requirements,
         )

apps/cic-cache/test_requirements.txt

@@ -7,4 +7,4 @@ pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
 sarafu-faucet~=0.0.7a1
-erc20-transfer-authorization~=0.3.6
+erc20-transfer-authorization>=0.3.5a1,<0.4.0

apps/cic-cache/tests/conftest.py

@@ -6,7 +6,6 @@ import datetime
 # external imports
 import pytest
 import moolb
-from chainlib.encode import TxHexNormalizer
 
 # local imports
 from cic_cache import db
@@ -43,8 +42,6 @@ def txs(
         list_tokens,
         ):
 
-    tx_normalize = TxHexNormalizer()
-
     session = init_database
 
     tx_number = 13 
@@ -57,10 +54,10 @@ def txs(
         tx_hash_first,
         list_defaults['block'],
         tx_number,
-        tx_normalize.wallet_address(list_actors['alice']),
+        list_actors['alice'],
-        tx_normalize.wallet_address(list_actors['bob']),
+        list_actors['bob'],
-        tx_normalize.executable_address(list_tokens['foo']),
+        list_tokens['foo'],
-        tx_normalize.executable_address(list_tokens['foo']),
+        list_tokens['foo'],
         1024,
         2048,
         True,
@@ -77,10 +74,10 @@ def txs(
         tx_hash_second,
         list_defaults['block']-1,
         tx_number,
-        tx_normalize.wallet_address(list_actors['diane']),
+        list_actors['diane'],
-        tx_normalize.wallet_address(list_actors['alice']),
+        list_actors['alice'],
-        tx_normalize.executable_address(list_tokens['foo']),
+        list_tokens['foo'],
-        tx_normalize.wallet_address(list_tokens['foo']),
+        list_tokens['foo'],
         1024,
         2048,
         False,
@@ -106,8 +103,6 @@ def more_txs(
 
     session = init_database
 
-    tx_normalize = TxHexNormalizer()
-
     tx_number = 666
     tx_hash = '0x' + os.urandom(32).hex()
     tx_signed = '0x' + os.urandom(128).hex()
@@ -120,10 +115,10 @@ def more_txs(
         tx_hash,
         list_defaults['block']+2,
         tx_number,
-        tx_normalize.wallet_address(list_actors['alice']),
+        list_actors['alice'],
-        tx_normalize.wallet_address(list_actors['diane']),
+        list_actors['diane'],
-        tx_normalize.executable_address(list_tokens['bar']),
+        list_tokens['bar'],
-        tx_normalize.executable_address(list_tokens['bar']),
+        list_tokens['bar'],
         2048,
         4096,
         False,

apps/cic-cache/tests/fixtures_config.py

@@ -14,8 +14,7 @@ logg = logging.getLogger(__file__)
 @pytest.fixture(scope='session')
 def load_config():
     config_dir = os.path.join(root_dir, 'config/test')
-    schema_config_dir = os.path.join(root_dir, 'cic_cache', 'data', 'config')
+    conf = confini.Config(config_dir, 'CICTEST')
-    conf = confini.Config(schema_config_dir, 'CICTEST', override_dirs=config_dir)
     conf.process()
     logg.debug('config {}'.format(conf))
     return conf

apps/cic-cache/tests/fixtures_database.py

@@ -24,15 +24,11 @@ def database_engine(
     if load_config.get('DATABASE_ENGINE') == 'sqlite':
         SessionBase.transactional = False
         SessionBase.poolable = False
-        name = 'cic_cache'
-        database_name = name
-        if load_config.get('DATABASE_PREFIX'):
-            database_name = '{}_{}'.format(load_config.get('DATABASE_PREFIX'), database_name)
         try:
-            os.unlink(database_name)
+            os.unlink(load_config.get('DATABASE_NAME'))
         except FileNotFoundError:
             pass
-    dsn = dsn_from_config(load_config, name)
+    dsn = dsn_from_config(load_config)
     SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
     return dsn
 

apps/cic-cache/tests/test_api.py

@@ -14,7 +14,7 @@ def test_api_all_data(
         ):
 
     env = {
-        'PATH_INFO': '/txa/100/0/410000/420000',
+        'PATH_INFO': '/txa/410000/420000',
         'HTTP_X_CIC_CACHE_MODE': 'all',
             }
     j = process_transactions_all_data(init_database, env)
@@ -23,7 +23,7 @@ def test_api_all_data(
     assert len(o['data']) == 2
 
     env = {
-        'PATH_INFO': '/txa/100/0/420000/410000',
+        'PATH_INFO': '/txa/420000/410000',
         'HTTP_X_CIC_CACHE_MODE': 'all',
             }
    

apps/cic-cache/tests/test_cache.py

@@ -6,7 +6,6 @@ import json
 
 # external imports
 import pytest
-from chainlib.encode import TxHexNormalizer
 
 # local imports
 from cic_cache import db
@@ -63,8 +62,6 @@ def test_cache_ranges(
 
     session = init_database
        
-    tx_normalize = TxHexNormalizer()
-
     oldest = list_defaults['block'] - 1
     mid = list_defaults['block']
     newest = list_defaults['block'] + 2
@@ -103,39 +100,32 @@ def test_cache_ranges(
     assert b[1] == mid
 
     # now check when supplying account
-    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account(list_actors['alice'], 0, 100)
-    b = c.load_transactions_account(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
 
-    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account(list_actors['bob'], 0, 100)
-    b = c.load_transactions_account(account, 0, 100)
     assert b[0] == mid
     assert b[1] == mid
 
-    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account(list_actors['diane'], 0, 100)
-    b = c.load_transactions_account(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
 
     # add block filter to the mix
-    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
     
-    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
 
-    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == mid
 
-    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == oldest
     assert b[1] == oldest
 
@@ -150,8 +140,6 @@ def test_cache_ranges_data(
 
     session = init_database
        
-    tx_normalize = TxHexNormalizer()
-
     oldest = list_defaults['block'] - 1
     mid = list_defaults['block']
     newest = list_defaults['block'] + 2
@@ -215,8 +203,7 @@ def test_cache_ranges_data(
     assert b[2][1]['tx_hash'] == more_txs[1]
 
     # now check when supplying account
-    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
-    b = c.load_transactions_account_with_data(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
     assert len(b[2]) == 3
@@ -224,15 +211,13 @@ def test_cache_ranges_data(
     assert b[2][1]['tx_hash'] == more_txs[1]
     assert b[2][2]['tx_hash'] == more_txs[2]
 
-    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
-    b = c.load_transactions_account_with_data(account, 0, 100)
     assert b[0] == mid
     assert b[1] == mid
     assert len(b[2]) == 1
     assert b[2][0]['tx_hash'] == more_txs[1]
 
-    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
-    b = c.load_transactions_account_with_data(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
     assert len(b[2]) == 2
@@ -240,31 +225,27 @@ def test_cache_ranges_data(
     assert b[2][1]['tx_hash'] == more_txs[2]
 
     # add block filter to the mix
-    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
     assert len(b[2]) == 2
     assert b[2][0]['tx_hash'] == more_txs[0]
     assert b[2][1]['tx_hash'] == more_txs[1]
 
-    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
     assert len(b[2]) == 2
     assert b[2][0]['tx_hash'] == more_txs[0]
     assert b[2][1]['tx_hash'] == more_txs[1]
 
-    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == mid
     assert len(b[2]) == 1
     assert b[2][0]['tx_hash'] == more_txs[1]
 
-    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == oldest
     assert b[1] == oldest
     assert len(b[2]) == 1

apps/cic-cache/tests/test_query.py

@@ -82,7 +82,7 @@ def test_query_regex(
         [ 
             ('alice', None, None, [(420000, 13), (419999, 42)]),
             ('alice', None, 1, [(420000, 13)]),
-            ('alice', 1, 1, [(419999, 42)]), # 420000 == list_defaults['block']
+            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
             ('alice', 2, None, []), # 420000 == list_defaults['block']
             ],
         )
@@ -107,11 +107,10 @@ def test_query_process_txs_account(
     path_info = '/tx/user/0x' + strip_0x(actor)
     if query_offset != None:
         path_info += '/' + str(query_offset)
-    if query_limit == None:
+    if query_limit != None:
-        query_limit = 100
+        if query_offset == None:
-    path_info += '/' + str(query_limit)
+            path_info += '/0'
-    if query_offset == None:
+        path_info += '/' + str(query_limit)
-        path_info += '/0'
     env = {
             'PATH_INFO': path_info,
             }
@@ -193,7 +192,7 @@ def test_query_process_txs_bloom(
 @pytest.mark.parametrize(
         'query_block_start, query_block_end, query_match_count',
         [ 
-            (1, 42, 0),
+            (None, 42, 0),
             (420000, 420001, 1),
             (419999, 419999, 1), # matches are inclusive
             (419999, 420000, 2),
@@ -212,7 +211,7 @@ def test_query_process_txs_data(
         query_match_count,
         ):
 
-    path_info = '/txa/100/0'
+    path_info = '/txa'
     if query_block_start != None:
         path_info += '/' + str(query_block_start)
     if query_block_end != None:
@@ -228,5 +227,4 @@ def test_query_process_txs_data(
     assert r != None
 
     o = json.loads(r[1])
-    logg.debug('oo {}'.format(o))
     assert len(o['data']) == query_match_count

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,5 +1,4 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.6
+erc20-demurrage-token~=0.0.3a1
-cic-eth-registry~=0.6.3
+cic-eth-registry>=0.6.1a2,<0.7.0
-chainlib~=0.0.14
+cic-eth[services]~=0.12.4a8
-cic_eth~=0.12.6

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.3
+version = 0.0.2a6
 description = cic-eth tasks supporting erc20 demurrage token
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth/admin_requirements.txt

@@ -1,4 +1,5 @@
 SQLAlchemy==1.3.20
-hexathon~=0.1.0
+cic-eth-registry>=0.6.1a3,<0.7.0
-chainqueue~=0.0.6a4
+hexathon~=0.0.1a8
-eth-erc20~=0.1.5
+chainqueue>=0.0.4a6,<0.1.0
+eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/admin/token.py

@@ -1,2 +1,21 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
 # local imports
-from cic_eth.eth.erc20 import default_token
+from cic_eth.task import BaseTask
+
+celery_app = celery.current_app
+logg = logging.getLogger()
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def default_token(self):
+    return {
+            'symbol': self.default_token_symbol,
+            'address': self.default_token_address,
+            'name': self.default_token_name,
+            'decimals': self.default_token_decimals,
+            }

apps/cic-eth/cic_eth/api/admin.py

@@ -123,7 +123,7 @@ class AdminApi:
         return s_lock.apply_async()
 
 
-    def tag_account(self, chain_spec, tag, address):
+    def tag_account(self, tag, address_hex, chain_spec):
         """Persistently associate an address with a plaintext tag.
 
         Some tags are known by the system and is used to resolve addresses to use for certain transactions. 
@@ -138,7 +138,7 @@ class AdminApi:
             'cic_eth.eth.account.set_role',
             [
                 tag,
-                address,
+                address_hex,
                 chain_spec.asdict(),
                 ],
             queue=self.queue,
@@ -146,30 +146,6 @@ class AdminApi:
         return s_tag.apply_async()
 
 
-    def get_tag_account(self, chain_spec, tag=None, address=None):
-        if address != None:
-            s_tag = celery.signature(
-                'cic_eth.eth.account.role',
-                [
-                    address,
-                    chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-
-        else:
-            s_tag = celery.signature(
-                'cic_eth.eth.account.role_account',
-                [
-                    tag,
-                    chain_spec.asdict(),
-                ],
-                queue=self.queue,
-                )
-
-        return s_tag.apply_async()
-
-
     def have_account(self, address_hex, chain_spec):
         s_have = celery.signature(
             'cic_eth.eth.account.have',
@@ -527,7 +503,7 @@ class AdminApi:
                     queue=self.queue,
                     )
                 t = s.apply_async()
-                role = t.get()[0][1]
+                role = t.get()
                 if role != None:
                     tx['sender_description'] = role
 
@@ -580,7 +556,7 @@ class AdminApi:
                     queue=self.queue,
                     )
                 t = s.apply_async()
-                role = t.get()[0][1]
+                role = t.get()
                 if role != None:
                     tx['recipient_description'] = role
 

apps/cic-eth/cic_eth/api/api_task.py

@@ -17,50 +17,15 @@ from cic_eth.enum import LockEnum
 
 app = celery.current_app
 
-#logg = logging.getLogger(__name__)
+logg = logging.getLogger(__name__)
-logg = logging.getLogger()
 
 
 class Api(ApiBase):
     
-    @staticmethod
-    def to_v_list(v, n):
-        """Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
-
-        :param v: Arguments
-        :type v: str or list
-        :param n: Number of elements to generate arguments for
-        :type n: int
-        :rtype: list
-        :returns: list of assembled arguments
-        """
-        if isinstance(v, str):
-            vv = v
-            v = []
-            for i in range(n):
-                v.append([vv])
-        elif not isinstance(v, list):
-            raise ValueError('argument must be single string, or list or strings or lists')
-        else:
-            if len(v) != n:
-                raise ValueError('v argument count must match integer n')
-            for i in range(n):
-                if isinstance(v[i], str):
-                    v[i] = [v[i]]
-                elif not isinstance(v, list):
-                    raise ValueError('proof argument must be single string, or list or strings or lists')
-
-        return v
-   
 
     def default_token(self):
-        """Retrieves the default fallback token of the custodial network.
-
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
         s_token = celery.signature(
-                'cic_eth.eth.erc20.default_token',
+                'cic_eth.admin.token.default_token',
                 [],
                 queue=self.queue,
                 )
@@ -70,97 +35,6 @@ class Api(ApiBase):
         return s_token.apply_async()
 
 
-    def token(self, token_symbol, proof=None):
-        """Single-token alias for tokens method.
-
-        See tokens method for details.
-
-        :param token_symbol: Token symbol to look up
-        :type token_symbol: str
-        :param proof: Proofs to add to signature verification for the token
-        :type proof: str or list
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        if not isinstance(token_symbol, str):
-            raise ValueError('token symbol must be string')
-
-        return self.tokens([token_symbol], proof=proof)
-
-
-    def tokens(self, token_symbols, proof=None):
-        """Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
-
-        Token symbols are always strings, and should be specified using uppercase letters.
-
-        If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
-
-        The proof argument may be specified in a number of ways:
-
-        - as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
-        - as a single string, where the same proof is used for each token lookup
-        - as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
-        - as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
-
-        The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
-
-        This method is not intended to be used synchronously. Do so at your peril.
-
-        :param token_symbols: Token symbol strings to look up
-        :type token_symbol: list
-        :param proof: Proof(s) to verify tokens against
-        :type proof: None, str or list
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        if not isinstance(token_symbols, list):
-            raise ValueError('token symbols argument must be list')
-
-        if proof == None:
-            logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
-            proof = ''
-
-        logg.debug('proof is {}'.format(proof))
-        l = len(token_symbols)
-        if len(proof) == 0:
-            l = 0 
-        proof = Api.to_v_list(proof, l)
-
-        chain_spec_dict = self.chain_spec.asdict()
-
-        s_token_resolve = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    token_symbols,
-                    chain_spec_dict,
-                    ],
-                queue=self.queue,
-                )
-
-        s_token_info = celery.signature(
-                'cic_eth.eth.erc20.token_info',
-                [
-                    chain_spec_dict,
-                    proof,
-                    ],
-                queue=self.queue,
-                )
-
-        s_token_verify = celery.signature(
-                    'cic_eth.eth.erc20.verify_token_info',
-                    [
-                        chain_spec_dict,
-                        self.callback_success,
-                        self.callback_error,
-                        ],
-                    queue=self.queue,
-                    )
-
-        s_token_info.link(s_token_verify)
-        s_token_resolve.link(s_token_info)
-        return s_token_resolve.apply_async()
-
-
 #    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
 #        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 #
@@ -515,7 +389,7 @@ class Api(ApiBase):
         :param password: Password to encode the password with in the backend (careful, you will have to remember it)
         :type password: str
         :param register: Register the new account in accounts index backend
-        :type register: bool
+        :type password: bool
         :returns: uuid of root task
         :rtype: celery.Task
         """
@@ -683,4 +557,3 @@ class Api(ApiBase):
 
         t = self.callback_success.apply_async([r])
         return t
-

apps/cic-eth/cic_eth/callbacks/noop.py

@@ -1,10 +1,7 @@
-import logging
-
 import celery
 
 celery_app = celery.current_app
-#logg = celery_app.log.get_default_logger()
+logg = celery_app.log.get_default_logger()
-logg = logging.getLogger()
 
 
 @celery_app.task(bind=True)

apps/cic-eth/cic_eth/check/gas.py

@@ -12,9 +12,8 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import LockedError
 from cic_eth.admin.ctrl import check_lock
-from cic_eth.eth.gas import have_gas_minimum
 
-logg = logging.getLogger(__name__)
+logg = logging.getLogger().getChild(__name__)
 
 
 def health(*args, **kwargs):
@@ -32,15 +31,18 @@ def health(*args, **kwargs):
         return True
 
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
-    min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
-    if config.get('ETH_MIN_FEE_PRICE'):
-        min_gas *= int(config.get('ETH_MIN_FEE_PRICE'))
-
-    r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session)
-
     session.close()
 
-    if not r:
+    rpc = RPCConnection.connect(chain_spec, 'default')
-        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas))
+    o = balance(gas_provider)
+    r = rpc.do(o)
+    try: 
+        r = int(r, 16)
+    except TypeError:
+        r = int(r)
+    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
+    if r < gas_min:
+        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
+        return False
 
-    return r
+    return True

apps/cic-eth/cic_eth/check/start.py

@@ -1,18 +0,0 @@
-# external imports
-from chainlib.chain import ChainSpec
-
-# local imports
-from cic_eth.admin.ctrl import check_lock
-from cic_eth.enum import LockEnum
-from cic_eth.error import LockedError
-
-
-def health(*args, **kwargs):
-    config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-
-    try:
-        check_lock(None, chain_spec.asdict(), LockEnum.START)
-    except LockedError as e:
-        return False
-    return True

apps/cic-eth/cic_eth/cli/arg.py

@@ -16,22 +16,16 @@ class ArgumentParser(BaseArgumentParser):
             self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
             self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
         if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
+            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-            self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
+            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
             self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
         if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
-            self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
-            self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
-            self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
-            self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
-            self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
-            self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
-            self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
-            self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
             self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, help='Start block height for initial history sync')
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
         if local_arg_flags & CICFlag.CHAIN:
             self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
+
+
+

apps/cic-eth/cic_eth/cli/base.py

@@ -24,7 +24,7 @@ class CICFlag(enum.IntEnum):
     # sync - nibble 4
     SYNCER = 4096
 
-argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
+
 argflag_local_task = CICFlag.CELERY 
 argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
 argflag_local_chain = CICFlag.CHAIN

apps/cic-eth/cic_eth/cli/config.py

@@ -1,18 +1,12 @@
 # standard imports
 import os
 import logging
-import urllib.parse
-import copy
 
 # external imports
 from chainlib.eth.cli import (
         Config as BaseConfig,
         Flag,
         )
-from urlybird.merge import (
-        urlhostmerge,
-        urlmerge,
-        )
 
 # local imports
 from .base import CICFlag
@@ -46,7 +40,6 @@ class Config(BaseConfig):
         if local_arg_flags & CICFlag.CHAIN:
             local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
 
-        
         if local_arg_flags & CICFlag.CELERY:
             local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
 
@@ -56,71 +49,15 @@ class Config(BaseConfig):
 
         config.dict_override(local_args_override, 'local cli args')
 
-        local_celery_args_override = {}
-        if local_arg_flags & CICFlag.CELERY:
-            hostport = urlhostmerge(
-                    None,
-                    config.get('REDIS_HOST'),
-                    config.get('REDIS_PORT'),
-                    )
-            db = getattr(args, 'redis_db', None)
-            if db != None:
-                db = str(db)
-
-            redis_url = (
-                    'redis',
-                    hostport,
-                    db,
-                    )
-
-
-            celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
-            hostport = urlhostmerge(
-                    celery_config_url[1],
-                    getattr(args, 'celery_host', None),
-                    getattr(args, 'celery_port', None),
-                    )
-            db = getattr(args, 'redis_db', None)
-            if db != None:
-                db = str(db)
-            celery_arg_url = (
-                    getattr(args, 'celery_scheme', None),
-                    hostport,
-                    db,
-                    )
-
-            celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
-            celery_url_string = urllib.parse.urlunsplit(celery_url)
-            local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
-            if not getattr(args, 'celery_no_result'):
-                local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
-                if local_celery_args_override['CELERY_RESULT_URL'] == None:
-                    local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
-                celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
-                hostport = urlhostmerge(
-                        celery_config_url[1],
-                        getattr(args, 'celery_result_host', None),
-                        getattr(args, 'celery_result_port', None),
-                        )
-                celery_arg_url = (
-                        getattr(args, 'celery_result_scheme', None),
-                        hostport,
-                        getattr(args, 'celery_result_db', None),
-                        )
-                celery_url = urlmerge(celery_config_url, celery_arg_url)
-                logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
-                celery_url_string = urllib.parse.urlunsplit(celery_url)
-                local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
-            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
-
-        config.dict_override(local_celery_args_override, 'local celery cli args')
-
         if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
+            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-            redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
+            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-            config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
+
-            config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
+        if local_arg_flags & CICFlag.CELERY:
+            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
 
         logg.debug('config loaded:\n{}'.format(config))
 
         return config
+
+

apps/cic-eth/cic_eth/data/config/celery.ini

@@ -1,5 +1,5 @@
 [celery]
-broker_url = 
+broker_url = redis://localhost:6379
 result_url = 
 queue = cic-eth
 debug = 0

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -2,5 +2,5 @@
 registry_address = 
 trust_address =
 default_token_symbol =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
+health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
 run_dir = /run

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,6 +1,2 @@
 [eth]
-gas_holder_minimum_units = 180000
+gas_gifter_minimum_balance = 10000000000000000000000
-gas_holder_refill_units = 15
-gas_holder_refill_threshold = 3
-gas_gifter_refill_buffer = 3
-min_fee_price = 1

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -23,7 +23,7 @@ def upgrade():
     op.create_table(
             'lock',
             sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column("address", sa.String, nullable=True),
+            sa.Column("address", sa.String(42), nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
             sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),

apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py

@@ -1,31 +0,0 @@
-"""Add gas cache
-
-Revision ID: c91cafc3e0c1
-Revises: aee12aeb47ec
-Create Date: 2021-10-28 20:45:34.239865
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'c91cafc3e0c1'
-down_revision = 'aee12aeb47ec'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'gas_cache',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column("address", sa.String, nullable=False),
-            sa.Column("tx_hash", sa.String, nullable=True),
-            sa.Column("method", sa.String, nullable=True),
-            sa.Column("value", sa.BIGINT(), nullable=False),
-            )
-
-
-def downgrade():
-    op.drop_table('gas_cache')

apps/cic-eth/cic_eth/db/models/gas_cache.py

@@ -1,27 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from sqlalchemy import Column, String, NUMERIC
-
-# local imports
-from .base import SessionBase
-
-logg = logging.getLogger(__name__)
-
-
-class GasCache(SessionBase):
-    """Provides gas budget cache for token operations
-    """
-    __tablename__ = 'gas_cache'
-
-    address = Column(String())
-    tx_hash = Column(String())
-    method = Column(String())
-    value = Column(NUMERIC())
-
-    def __init__(self, address, method, value, tx_hash):
-        self.address = address
-        self.tx_hash = tx_hash
-        self.method = method
-        self.value = value

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -12,7 +12,7 @@ from cic_eth.error import (
         IntegrityError,
         )
 
-logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 class Nonce(SessionBase):
@@ -21,7 +21,7 @@ class Nonce(SessionBase):
     __tablename__ = 'nonce'
 
     nonce = Column(Integer)
-    address_hex = Column(String(40))
+    address_hex = Column(String(42))
 
 
     @staticmethod

apps/cic-eth/cic_eth/db/models/role.py

@@ -25,21 +25,7 @@ class AccountRole(SessionBase):
     address_hex = Column(String(42))
 
     
-    @staticmethod
+    # TODO: 
-    def all(session=None):
-        session = SessionBase.bind_session(session)
-
-        pairs = []
-
-        q = session.query(AccountRole.tag, AccountRole.address_hex)
-        for r in q.all():
-            pairs.append((r[1], r[0]),)
-
-        SessionBase.release_session(session)
-        
-        return pairs
-
-
     @staticmethod
     def get_address(tag, session):
         """Get Ethereum address matching the given tag

apps/cic-eth/cic_eth/enum.py

@@ -69,12 +69,9 @@ class StatusEnum(enum.IntEnum):
 class LockEnum(enum.IntEnum):
     """
     STICKY: When set, reset is not possible
-    INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance)
-    START: When set, startup is not possible, regardless of state
     CREATE: Disable creation of accounts
     SEND: Disable sending to network
     QUEUE: Disable queueing new or modified transactions
-    QUERY: Disable all queue state and transaction queries
     """
     STICKY=1
     INIT=2
@@ -82,8 +79,7 @@ class LockEnum(enum.IntEnum):
     SEND=8
     QUEUE=16
     QUERY=32
-    START=int(0x80000000)
+    ALL=int(0xfffffffffffffffe)
-    ALL=int(0x7ffffffe)
 
 
 def status_str(v, bits_only=False):

apps/cic-eth/cic_eth/error.py

@@ -48,6 +48,8 @@ class RoleMissingError(Exception):
     pass
 
 
+
+
 class IntegrityError(Exception):
     """Exception raised to signal irregularities with deduplication and ordering of tasks
 
@@ -64,10 +66,8 @@ class LockedError(Exception):
 
 class SeppukuError(Exception):
     """Exception base class for all errors that should cause system shutdown
+
     """
-    def __init__(self, message, lockdown=False):
-        self.message = message
-        self.lockdown = lockdown
 
 
 class SignerError(SeppukuError):
@@ -85,8 +85,3 @@ class RoleAgencyError(SeppukuError):
 class YouAreBrokeError(Exception):
     """Exception raised when a value transfer is attempted without access to sufficient funds
     """
-
-
-class TrustError(Exception):
-    """Exception raised when required trust proofs are missing for a request
-    """

apps/cic-eth/cic_eth/eth/account.py

@@ -136,7 +136,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, code_callback=AccountRegistry.gas)
+    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
     account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -192,7 +192,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, code_callback=MinterFaucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -266,46 +266,19 @@ def set_role(self, tag, address, chain_spec_dict):
 
 @celery_app.task(bind=True, base=BaseTask)
 def role(self, address, chain_spec_dict):
-    """Return account role for address and/or role
+    """Return account role for address
 
     :param account: Account to check
     :type account: str, 0x-hex
-    :param chain_spec_dict: Chain spec dict representation
+    :param chain_str: Chain spec string representation
-    :type chain_spec_dict: dict
+    :type chain_str: str
     :returns: Account, or None if not exists
     :rtype: Varies
     """
     session = self.create_session()
     role_tag =  AccountRole.role_for(address, session=session)
     session.close()
-    return [(address, role_tag,)]
+    return role_tag
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def role_account(self, role_tag, chain_spec_dict):
-    """Return address for role.
-
-    If the role parameter is None, will return addresses for all roles.
-
-    :param role_tag: Role to match
-    :type role_tag: str
-    :param chain_spec_dict: Chain spec dict representation
-    :type chain_spec_dict: dict
-    :returns: List with a single account/tag pair for a single tag, or a list of account and tag pairs for all tags
-    :rtype: list
-    """
-    session = self.create_session()
-
-    pairs = None
-    if role_tag != None:
-        addr = AccountRole.get_address(role_tag, session=session)
-        pairs = [(addr, role_tag,)]
-    else:
-        pairs = AccountRole.all(session=session)
-
-    session.close()
-
-    return pairs
 
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)

apps/cic-eth/cic_eth/eth/erc20.py

@@ -10,9 +10,6 @@ from chainlib.eth.tx import (
         TxFormat,
         unpack,
         )
-from chainlib.eth.contract import (
-        ABIContractEncoder,
-        )
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from hexathon import (
@@ -22,7 +19,6 @@ from hexathon import (
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
 from chainqueue.sql.tx import cache_tx_dict
-from okota.token_index.index import to_identifier
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -34,26 +30,25 @@ from cic_eth.error import (
         YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
-from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.eth.gas import (
-from cic_eth.eth.util import CacheGasOracle
+        create_check_gas_task,
+        MaxGasOracle,
+        )
 from cic_eth.ext.address import translate_address
 from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
         CriticalSQLAlchemyAndSignerTask,
-        BaseTask,
     )
 from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.encode import tx_normalize
-from cic_eth.eth.trust import verify_proofs
-from cic_eth.error import SignerError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(bind=True, base=CriticalWeb3Task)
+@celery_app.task(base=CriticalWeb3Task)
-def balance(self, tokens, holder_address, chain_spec_dict):
+def balance(tokens, holder_address, chain_spec_dict):
     """Return token balances for a list of tokens for given address
 
     :param tokens: Token addresses
@@ -72,9 +67,8 @@ def balance(self, tokens, holder_address, chain_spec_dict):
     for t in tokens:
         address = t['address']
         logg.debug('address {} {}'.format(address, holder_address))
-        gas_oracle = self.create_gas_oracle(rpc, min_price=self.min_fee_price)
         token = ERC20Token(chain_spec, rpc, add_0x(address))
-        c = ERC20(chain_spec, gas_oracle=gas_oracle)
+        c = ERC20(chain_spec)
         o = c.balance_of(address, holder_address, sender_address=caller_address)
         r = rpc.do(o)
         t['balance_network'] = c.parse_balance(r)
@@ -157,12 +151,8 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
-
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    enc = ABIContractEncoder()
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
-    enc.method('transferFrom')
-    method = enc.get()
-    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -232,12 +222,8 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
-
-    enc = ABIContractEncoder()
-    enc.method('transfer')
-    method = enc.get()
-    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -305,12 +291,8 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
-
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    enc = ABIContractEncoder()
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
-    enc.method('approve')
-    method = enc.get()
-    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -397,8 +379,6 @@ def cache_transfer_data(
     sender_address = tx_normalize.wallet_address(tx['from'])
     recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
-    source_token_address = tx_normalize.executable_address(tx['to'])
-    destination_token_address = source_token_address
 
 
     session = SessionBase.create_session()
@@ -406,8 +386,8 @@ def cache_transfer_data(
             'hash': tx_hash_hex,
             'from': sender_address,
             'to': recipient_address,
-            'source_token': source_token_address,
+            'source_token': tx['to'],
-            'destination_token': destination_token_address,
+            'destination_token': tx['to'],
             'from_value': token_value,
             'to_value': token_value,
             }
@@ -439,16 +419,14 @@ def cache_transfer_from_data(
     spender_address = tx_data[0]
     recipient_address = tx_data[1]
     token_value = tx_data[2]
-    source_token_address = tx_normalize.executable_address(tx['to'])
-    destination_token_address = source_token_address
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
             'from': tx['from'],
             'to': recipient_address,
-            'source_token': source_token_address,
+            'source_token': tx['to'],
-            'destination_token': destination_token_address,
+            'destination_token': tx['to'],
             'from_value': token_value,
             'to_value': token_value,
             }
@@ -480,16 +458,14 @@ def cache_approve_data(
     sender_address = tx_normalize.wallet_address(tx['from'])
     recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
-    source_token_address = tx_normalize.executable_address(tx['to'])
-    destination_token_address = source_token_address
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
             'from': sender_address,
             'to': recipient_address,
-            'source_token': source_token_address,
+            'source_token': tx['to'],
-            'destination_token': destination_token_address,
+            'destination_token': tx['to'],
             'from_value': token_value,
             'to_value': token_value,
             }
@@ -497,69 +473,3 @@ def cache_approve_data(
     session.close()
     return (tx_hash_hex, cache_id)
 
-
-@celery_app.task(bind=True, base=BaseTask)
-def token_info(self, tokens, chain_spec_dict, proofs=[]):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    i = 0
-
-    for token in tokens:
-        result_data = []
-        token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
-        token_chain_object.load(rpc)
-
-        token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
-        token_proofs = [token_symbol_proof_hex]
-        if len(proofs) > 0:
-            token_proofs += proofs[i]
-
-        tokens[i] = {
-            'decimals': token_chain_object.decimals,
-            'name': token_chain_object.name,
-            'symbol': token_chain_object.symbol,
-            'address': tx_normalize.executable_address(token_chain_object.address),
-            'proofs': token_proofs,
-            'converters': tokens[i]['converters'],
-                }   
-        i += 1
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
-    queue = self.request.delivery_info.get('routing_key')
-
-    for token in tokens:
-        s = celery.signature(
-                'cic_eth.eth.trust.verify_proofs',
-                [
-                    token,
-                    token['address'],
-                    token['proofs'],
-                    chain_spec_dict,
-                    success_callback,
-                    error_callback,
-                    ],
-                queue=queue,
-                )
-
-        if success_callback != None:
-            s.link(success_callback)
-        if error_callback != None:
-            s.on_error(error_callback)
-        s.apply_async()
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-        'symbol': self.default_token_symbol,
-        'address': self.default_token_address,
-        'name': self.default_token_name,
-        'decimals': self.default_token_decimals,
-        }

apps/cic-eth/cic_eth/eth/gas.py

@@ -41,7 +41,6 @@ from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
-from cic_eth.db.models.gas_cache import GasCache
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
 from cic_eth.error import (
@@ -66,56 +65,17 @@ from cic_eth.encode import (
         ZERO_ADDRESS_NORMAL,
         unpack_normal,
         )
-from cic_eth.error import SeppukuError
-from cic_eth.eth.util import MAXIMUM_FEE_UNITS
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
+MAXIMUM_FEE_UNITS = 8000000
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
+class MaxGasOracle:
-def apply_gas_value_cache(address, method, value, tx_hash):
-    return apply_gas_value_cache_local(address, method, value, tx_hash)
 
-
+    def gas(code=None):
-def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
+        return MAXIMUM_FEE_UNITS
-    address = tx_normalize.executable_address(address)
-    tx_hash = tx_normalize.tx_hash(tx_hash)
-    value = int(value)
-
-    session = SessionBase.bind_session(session)
-    q = session.query(GasCache)
-    q = q.filter(GasCache.address==address)
-    q = q.filter(GasCache.method==method)
-    o = q.first()
-
-    if o == None:
-        o = GasCache(address, method, value, tx_hash)
-    elif value > o.value:
-        o.value = value
-        o.tx_hash = strip_0x(tx_hash)
-
-    session.add(o)
-    session.commit()
-
-    SessionBase.release_session(session)
-
-
-def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):
-    if rpc == None:
-        rpc = RPCConnection.connect(chain_spec, 'default')
-    o = balance(add_0x(address))
-    r = rpc.do(o)
-    try: 
-        r = int(r)
-    except ValueError:
-        r = strip_0x(r)
-        r = int(r, 16)
-    logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas))
-    if r < min_gas:
-        return False
-    return True
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -397,13 +357,6 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # set up evm RPC connection
     rpc = RPCConnection.connect(chain_spec, 'default')
 
-    # check the gas balance of the gifter
-    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount):
-        raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider))
-
-    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance):
-        logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider))
-    
     # set up transaction builder
     nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc)

apps/cic-eth/cic_eth/eth/trust.py

@@ -1,77 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from eth_address_declarator import Declarator
-from chainlib.connection import RPCConnection
-from chainlib.chain import ChainSpec
-from cic_eth.db.models.role import AccountRole
-from cic_eth_registry import CICRegistry
-from hexathon import strip_0x
-
-# local imports
-from cic_eth.task import BaseTask
-from cic_eth.error import TrustError
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
-    proof = strip_0x(proof)
-
-    proofs = []
- 
-    logg.debug('proof count {}'.format(len(proofs)))
-    if len(proofs) == 0:
-        logg.debug('error {}'.format(len(proofs)))
-        raise TrustError('foo')
-
-    return (chained_input, (proof, proofs))
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
-    queue = self.request.delivery_info.get('routing_key')
-
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    session = self.create_session()
-    sender_address = AccountRole.get_address('DEFAULT', session)
-
-    registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-
-    declarator = Declarator(chain_spec)
-
-    have_proofs = {}
-
-    for proof in proofs:
-
-        proof = strip_0x(proof)
-
-        have_proofs[proof] = []
-
-        for trusted_address in self.trusted_addresses:
-            o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
-            r = rpc.do(o)
-            declarations = declarator.parse_declaration(r)
-            logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
-
-            for declaration in declarations:
-                declaration = strip_0x(declaration)
-                if declaration == proof:
-                    logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
-                    have_proofs[proof].append(trusted_address)
-
-    out_proofs = {}
-    for proof in have_proofs.keys():
-        if len(have_proofs[proof]) == 0:
-            logg.error('missing signer for proof {} subject {}'.format(proof, subject))
-            raise TrustError((subject, proof,))
-        out_proofs[proof] = have_proofs[proof]
-       
-    return (chained_input, out_proofs)

apps/cic-eth/cic_eth/eth/util.py

@@ -1,54 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.eth.gas import RPCGasOracle
-from hexathon import strip_0x
-
-# local imports
-from cic_eth.db.models.gas_cache import GasCache
-from cic_eth.encode import tx_normalize
-from cic_eth.db.models.base import SessionBase
-
-MAXIMUM_FEE_UNITS = 8000000
-
-logg = logging.getLogger(__name__)
-
-
-class MaxGasOracle(RPCGasOracle):
-
-    def get_fee_units(self, code=None):
-        return MAXIMUM_FEE_UNITS
-
-
-class CacheGasOracle(MaxGasOracle):
-    """Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units.
-
-    :todo: instead of max units, connect a pluggable gas heuristics engine.
-    """
-
-    def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None):
-        super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator)
-        self.value = None
-        self.address = address
-        self.method = method
-
-        address = tx_normalize.executable_address(address)
-        session = SessionBase.bind_session(session)
-        q = session.query(GasCache)
-        q = q.filter(GasCache.address==address)
-        if method != None:
-            method = strip_0x(method)
-            q = q.filter(GasCache.method==method)
-        o = q.first()
-        if o != None:
-            self.value = int(o.value)
-
-        SessionBase.release_session(session)
-
-
-    def get_fee_units(self, code=None):
-        if self.value != None:
-            logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method))
-            return self.value
-        return super(CacheGasOracle, self).get_fee_units(code=code)

apps/cic-eth/cic_eth/pytest/fixtures_celery.py

@@ -4,21 +4,18 @@ import tempfile
 import logging
 import shutil
 
-# local imports
+# local impors
 from cic_eth.task import BaseTask
 
 #logg = logging.getLogger(__name__)
 logg = logging.getLogger()
 
+
 @pytest.fixture(scope='function')
 def init_celery_tasks(
     contract_roles, 
         ):
     BaseTask.call_address = contract_roles['DEFAULT']
-    BaseTask.trusted_addresses = [
-            contract_roles['TRUSTED_DECLARATOR'],
-            contract_roles['CONTRACT_DEPLOYER'],
-            ]
 
 
 # celery fixtures
@@ -41,7 +38,6 @@ def celery_includes():
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
         'cic_eth.pytest.mock.filter',
-        'cic_eth.pytest.mock.callback',
     ]
 
 

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -8,14 +8,15 @@ import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(os.path.dirname(script_dir))
-config_dir = os.path.join(root_dir, 'cic_eth', 'data', 'config')
 logg = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')
 def load_config():
-    override_config_dir = os.path.join(root_dir, 'config', 'test')
+    config_dir = os.environ.get('CONFINI_DIR')
-    conf = confini.Config(config_dir, 'CICTEST', override_dirs=[override_config_dir])
+    if config_dir == None:
+        config_dir = os.path.join(root_dir, 'config/test')
+    conf = confini.Config(config_dir, 'CICTEST')
     conf.process()
     logg.debug('config {}'.format(conf))
     return conf

apps/cic-eth/cic_eth/pytest/mock/__init__.py

@@ -1,2 +1 @@
 from .filter import *
-from .callback import *

apps/cic-eth/cic_eth/pytest/mock/callback.py

@@ -1,38 +0,0 @@
-# standard imports
-import os
-import logging
-import mmap
-
-# standard imports
-import tempfile
-
-# external imports
-import celery
-
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
-
-celery_app = celery.current_app
-
-
-class CallbackTask(celery.Task):
-
-    mmap_path = tempfile.mkdtemp()
-
-
-@celery_app.task(bind=True, base=CallbackTask)
-def test_callback(self, a, b, c):
-    s = 'ok'
-    if c > 0:
-        s = 'err'
-
-    fp = os.path.join(self.mmap_path, b)
-    f = open(fp, 'wb+')
-    f.write(b'\x00')
-    f.seek(0)
-    m = mmap.mmap(f.fileno(), length=1)
-    m.write(c.to_bytes(1, 'big'))
-    m.close()
-    f.close()
-
-    logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))

apps/cic-eth/cic_eth/queue/balance.py

@@ -72,7 +72,7 @@ def __balance_incoming_compatible(token_address, receiver_address):
     status_compare = dead()
     q = q.filter(Otx.status.op('&')(status_compare)==0)
     # TODO: this can change the result for the recipient if tx is later obsoleted and resubmission is delayed. 
-    #q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK)
+    q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK)
     q = q.filter(TxCache.destination_token_address==token_address)
     delta = 0
     for r in q.all():

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -10,6 +10,7 @@ import datetime
 
 # external imports
 import celery
+from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
 from chainlib.connection import RPCConnection

apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py

@@ -3,4 +3,3 @@ from .tx import TxFilter
 from .gas import GasFilter
 from .register import RegistrationFilter
 from .transferauth import TransferAuthFilter
-from .token import TokenFilter

apps/cic-eth/cic_eth/runnable/daemons/filters/token.py

@@ -1,63 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from eth_erc20 import ERC20
-from chainlib.eth.contract import (
-        ABIContractEncoder,
-        ABIContractType,
-        )
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.address import is_same_address
-from chainlib.eth.error import RequestMismatchException
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.erc20 import ERC20Token
-from eth_token_index import TokenUniqueSymbolIndex
-import celery
-
-# local imports
-from .base import SyncFilter
-
-logg = logging.getLogger(__name__)
-
-
-class TokenFilter(SyncFilter):
-
-    def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
-        self.queue = queue
-        self.chain_spec = chain_spec
-        self.caller_address = call_address
-
-
-    def filter(self, conn, block, tx, db_session=None):
-        if not tx.payload:
-            return (None, None)
-
-        try:
-            r = ERC20.parse_transfer_request(tx.payload)
-        except RequestMismatchException:
-            return (None, None)
-
-        token_address = tx.inputs[0]
-        token = ERC20Token(self.chain_spec, conn, token_address)
-
-        registry = CICRegistry(self.chain_spec, conn)
-        r = registry.by_name(token.symbol, sender_address=self.caller_address)
-        if is_same_address(r, ZERO_ADDRESS):
-            return None
-
-        enc = ABIContractEncoder()
-        enc.method('transfer')
-        method = enc.get()
-
-        s = celery.signature(
-                'cic_eth.eth.gas.apply_gas_value_cache',
-                [
-                    token_address,
-                    method,
-                    tx.gas_used,
-                    tx.hash,
-                    ],
-                queue=self.queue,
-                )
-        return s.apply_async()

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -67,10 +67,7 @@ from cic_eth.registry import (
         connect_declarator,
         connect_token_registry,
         )
-from cic_eth.task import (
+from cic_eth.task import BaseTask
-        BaseTask,
-        CriticalWeb3Task,
-        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -79,18 +76,18 @@ arg_flags = cic_eth.cli.argflag_std_read
 local_arg_flags = cic_eth.cli.argflag_local_task
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)
+argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
-argparser.add_argument('--min-fee-price', dest='min_fee_price', type=int, help='set minimum fee price for transactions, in wei')
 argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
 args = argparser.parse_args()
 
 # process config
 extra_args = {
+    'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
     'aux_all': None,
     'aux': None,
     'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
-    'min_fee_price': 'ETH_MIN_FEE_PRICE',
         }
 config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
@@ -190,17 +187,6 @@ elif len(args.aux) > 0:
         logg.info('aux module {} found in path {}'.format(v, aux_dir))
         aux.append(v)
 
-default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-defaullt_token_address = None
-if default_token_symbol:
-    default_token_address = registry.by_name(default_token_symbol)
-else:
-    default_token_address = registry.by_name('DefaultToken')
-    c = ERC20Token(chain_spec, conn, default_token_address)
-    default_token_symbol = c.symbol
-    logg.info('found default token {} address {}'.format(default_token_symbol, default_token_address))
-    config.add(default_token_symbol, 'CIC_DEFAULT_TOKEN_SYMBOL', exists_ok=True)
-
 for v in aux:
     mname = 'cic_eth_aux.' + v
     mod = importlib.import_module(mname)
@@ -218,22 +204,12 @@ def main():
     argv.append('-n')
     argv.append(config.get('CELERY_QUEUE'))
 
-    # TODO: More elegant way of setting queue-wide settings
+    BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-    BaseTask.default_token_symbol = default_token_symbol
+    BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    BaseTask.default_token_address = default_token_address
     default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
     default_token.load(conn)
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name
-    BaseTask.trusted_addresses = trusted_addresses
-    CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS'))
-    CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD'))
-    CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
-    if config.get('ETH_MIN_FEE_PRICE'):
-        BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE'))
-        CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price
-        CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price
-        CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price
 
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -36,7 +36,6 @@ from cic_eth.runnable.daemons.filters import (
         TxFilter,
         RegistrationFilter,
         TransferAuthFilter,
-        TokenFilter,
         )
 from cic_eth.stat import init_chain_stat
 from cic_eth.registry import (
@@ -100,10 +99,10 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = int(config.get('SYNCER_OFFSET'))
+        initial_block_start = config.get('SYNCER_OFFSET')
-        initial_block_offset = int(block_offset)
+        initial_block_offset = block_offset
         if config.true('SYNCER_NO_HISTORY'):
-            initial_block_start = initial_block_offset
+            initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
         logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
@@ -155,8 +154,6 @@ def main():
 
     gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
 
-    token_gas_cache_filter = TokenFilter(chain_spec, config.get('CELERY_QUEUE'))
-
     #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
 
     i = 0
@@ -166,7 +163,6 @@ def main():
         syncer.add_filter(registration_filter)
         # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
         syncer.add_filter(tx_filter)
-        syncer.add_filter(token_gas_cache_filter)
         #syncer.add_filter(transfer_auth_filter)
         for cf in callback_filters:
             syncer.add_filter(cf)

apps/cic-eth/cic_eth/runnable/tag.py

@@ -8,7 +8,6 @@ import re
 # external imports
 import cic_eth.cli
 from chainlib.chain import ChainSpec
-from chainlib.eth.address import is_address
 from xdg.BaseDirectory import xdg_config_home
 
 # local imports
@@ -22,18 +21,12 @@ logg = logging.getLogger()
 arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
 local_arg_flags = cic_eth.cli.argflag_local_taskcallback
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.add_argument('--set', action='store_true', help='sets the given tag')
+argparser.add_positional('tag', type=str, help='address tag')
-argparser.add_argument('--tag', type=str, help='operate on the given tag')
+argparser.add_positional('address', type=str, help='address')
-argparser.add_positional('address', required=False, type=str, help='address associated with tag')
 argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-extra_args = {
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
-        'set': None,
-        'tag': None,
-        'address': None,
-        }
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
 celery_app = cic_eth.cli.CeleryApp.from_config(config)
 
@@ -46,17 +39,7 @@ api = AdminApi(None)
 
 
 def main():
-    if config.get('_ADDRESS') != None and not is_address(config.get('_ADDRESS')):
+    admin_api.tag_account(args.tag, args.address, chain_spec)
-        sys.stderr.write('Invalid address {}'.format(config.get('_ADDRESS')))
-        sys.exit(1)
-
-    if config.get('_SET'):
-        admin_api.tag_account(chain_spec, config.get('_TAG'), config.get('_ADDRESS'))
-    else:
-        t = admin_api.get_tag_account(chain_spec, tag=config.get('_TAG'), address=config.get('_ADDRESS'))
-        r = t.get()
-        for v in r:
-            sys.stdout.write('{}\t{}\n'.format(v[1], v[0]))
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/transfer.py

@@ -18,7 +18,7 @@ from cic_eth.api import Api
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger('create_account_script')
 
-arg_flags = cic_eth.cli.argflag_local_base
+arg_flags = cic_eth.cli.argflag_std_base
 local_arg_flags = cic_eth.cli.argflag_local_taskcallback
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol')

apps/cic-eth/cic_eth/runnable/view.py

@@ -16,14 +16,9 @@ import confini
 import celery
 from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
-from hexathon import (
+from hexathon import add_0x
-        add_0x,
-        strip_0x,
-        uniform as hex_uniform,
-        )
 
 # local imports
-import cic_eth.cli
 from cic_eth.api.admin import AdminApi
 from cic_eth.db.enum import (
     StatusEnum,
@@ -36,35 +31,59 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 default_format = 'terminal'
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
-arg_flags = cic_eth.cli.argflag_std_base
+argparser = argparse.ArgumentParser()
-local_arg_flags = cic_eth.cli.argflag_local_taskcallback
+argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
+argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
 argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
+argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
 argparser.add_argument('query', type=str, help='Transaction, transaction hash, account or "lock"')
-argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
 
-extra_args = {
+
-    'f': '_FORMAT',
+config_dir = os.path.join(args.c)
-    'query': '_QUERY',
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+args_override = {
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         }
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
+# override args
+config.dict_override(args_override, 'cli args')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
-celery_app = cic_eth.cli.CeleryApp.from_config(config)
+try:
-queue = config.get('CELERY_QUEUE')
+    config.add(add_0x(args.query), '_QUERY', True)
+except:
+    config.add(args.query, '_QUERY', True)
 
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
-# connect to celery
+queue = args.q
-celery_app = cic_eth.cli.CeleryApp.from_config(config)
 
-# set up rpc
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-rpc = cic_eth.cli.RPC.from_config(config) #, use_signer=True)
-conn = rpc.get_default()
 
-admin_api = AdminApi(conn)
+rpc = EthHTTPConnection(args.p)
+
+#registry_address = config.get('CIC_REGISTRY_ADDRESS')
+
+admin_api = AdminApi(rpc)
 
 t = admin_api.registry()
 registry_address = t.get()
@@ -94,7 +113,7 @@ def render_tx(o, **kwargs):
     
     for v in o.get('status_log', []):
         d = datetime.datetime.fromisoformat(v[0])
-        e = status_str(v[1], config.get('_RAW'))
+        e = status_str(v[1], args.status_raw)
         content += '{}: {}\n'.format(d, e)
 
     return content
@@ -135,24 +154,20 @@ def render_lock(o, **kwargs):
 def main():
     txs  = []
     renderer = render_tx
+    if len(config.get('_QUERY')) > 66:
+        #registry = connect_registry(rpc, chain_spec, registry_address)
+        #admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
+        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
+    elif len(config.get('_QUERY')) > 42:
+        #registry = connect_registry(rpc, chain_spec, registry_address)
+        #admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
+        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
 
-    query = config.get('_QUERY')
+    elif len(config.get('_QUERY')) == 42:
-    try:
+        #registry = connect_registry(rpc, chain_spec, registry_address)
-        query = hex_uniform(strip_0x(query))
+        txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
-    except TypeError:
-        pass
-    except ValueError:
-        pass
-
-    if len(query) > 64:
-        admin_api.tx(chain_spec, tx_raw=query, renderer=renderer)
-    elif len(query) > 40:
-        admin_api.tx(chain_spec, tx_hash=query, renderer=renderer)
-
-    elif len(query) == 40:
-        txs = admin_api.account(chain_spec, query, include_recipient=False, renderer=render_account)
         renderer = render_account
-    elif len(query) >= 4 and query[:4] == 'lock':
+    elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
         t = admin_api.get_lock()
         txs = t.get()
         renderer = render_lock
@@ -160,7 +175,7 @@ def main():
             r = renderer(txs)
             sys.stdout.write(r + '\n')
     else:
-        raise ValueError('cannot parse argument {}'.format(query))
+        raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))
                    
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/task.py

@@ -17,7 +17,6 @@ from cic_eth_registry.error import UnknownContractError
 # local imports
 from cic_eth.error import SeppukuError
 from cic_eth.db.models.base import SessionBase
-from cic_eth.eth.util import CacheGasOracle, MaxGasOracle
 
 #logg = logging.getLogger().getChild(__name__)
 logg = logging.getLogger()
@@ -25,46 +24,18 @@ logg = logging.getLogger()
 celery_app = celery.current_app
 
 
-
 class BaseTask(celery.Task):
 
     session_func = SessionBase.create_session
     call_address = ZERO_ADDRESS
-    trusted_addresses = []
+    create_nonce_oracle = RPCNonceOracle
-    min_fee_price = 1
+    create_gas_oracle = RPCGasOracle
-    min_fee_limit = 30000
     default_token_address = None
     default_token_symbol = None
     default_token_name = None
     default_token_decimals = None
     run_dir = '/run'
 
-
-    def create_gas_oracle(self, conn, address=None, *args, **kwargs):
-        x = None
-        if address is None:
-            x = RPCGasOracle(
-                conn,
-                code_callback=kwargs.get('code_callback', self.get_min_fee_limit),
-                min_price=self.min_fee_price,
-                id_generator=kwargs.get('id_generator'),
-                )
-        else:
-
-            x = MaxGasOracle(conn)
-            x.code_callback = x.get_fee_units
-
-        return x
-
-
-    def get_min_fee_limit(self, code):
-        return self.min_fee_limit
-
-
-    def get_min_fee_limit(self, code):
-        return self.min_fee_limit
-
-
     def create_session(self):
         return BaseTask.session_func()
 
@@ -106,18 +77,19 @@ class CriticalWeb3Task(CriticalTask):
     autoretry_for = (
         ConnectionError,
         )
-    safe_gas_threshold_amount = 60000 * 3
+    safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
-    safe_gas_gifter_balance = safe_gas_threshold_amount * 5 * 100
 
 
-class CriticalSQLAlchemyAndWeb3Task(CriticalWeb3Task):
+class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
     autoretry_for = (
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
         )
+    safe_gas_threshold_amount = 2000000000 * 60000 * 3
+    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
 
 
 class CriticalSQLAlchemyAndSignerTask(CriticalTask):
@@ -127,10 +99,13 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
         sqlalchemy.exc.ResourceClosedError,
         ) 
 
-class CriticalWeb3AndSignerTask(CriticalWeb3Task):
+class CriticalWeb3AndSignerTask(CriticalTask):
     autoretry_for = (
         ConnectionError,
         )
+    safe_gas_threshold_amount = 2000000000 * 60000 * 3
+    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
+
 
 @celery_app.task()
 def check_health(self):

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         4,
-        'alpha.14',
+        'alpha.8',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/test/accounts.ini

@@ -0,0 +1,2 @@
+[accounts]
+writer_address = 

apps/cic-eth/config/test/bancor.ini

@@ -0,0 +1,2 @@
+[bancor]
+dir = tests/testdata/bancor

apps/cic-eth/config/test/celery.ini

@@ -1,3 +1,5 @@
 [celery]
 broker_url = filesystem://
 result_url = filesystem://
+#broker_url = redis://
+#result_url = redis://

apps/cic-eth/config/test/cic.ini

@@ -0,0 +1,4 @@
+[cic]
+registry_address = 
+chain_spec = 
+trust_address =

apps/cic-eth/config/test/dispatcher.ini

@@ -0,0 +1,2 @@
+[dispatcher]
+loop_interval = 0.1