Merge branch 'philip/translations-v1' into 'master'

Adds updated translations.

See merge request grassrootseconomics/cic-internal-integration!328

.env

@@ -1,39 +0,0 @@
-DOMAIN=localhost
-
-STACK_NAME=cic-net
-
-TRAEFIK_PUBLIC_NETWORK=traefik-public
-TRAEFIK_TAG=cic.net
-TRAEFIK_PUBLIC_TAG=traefik-public
-
-FRONTEND_ENV=dev
-
-# Flower
-FLOWER_BASIC_AUTH=admin:changethis
-
-# Postgres
-DATABASE_HOST=postgres
-DATABASE_PORT=5432
-DATABASE_ENGINE=postgresql
-DATABASE_DRIVER=psycopg2
-
-DATABASE_USER=postgres
-DATABASE_NAME=cic_eth
-
-# Redis
-REDIS_HOST=redis
-REDIS_PORT=6379
-REDIS_DB=0
-
-# Celery Broker
-CELERY_BROKER_URL=redis://redis:6379
-CELERY_RESULT_URL=redis://redis:6379
-
-# Blockchain node
-ETH_PROVIDER=http://eth:8545
-CIC_CHAIN_SPEC=evm:bloxberg:8996
-
-# PgAdmin
-PGADMIN_LISTEN_PORT=5050
-PGADMIN_DEFAULT_EMAIL=admin@cic.net
-PGADMIN_DEFAULT_PASSWORD=changethis

.gitignore

@@ -14,4 +14,4 @@ build/
 **/.venv
 .idea
 **/.vim
-docker-stack.yml
+**/*secret.yaml

.gitlab-ci.yml

@@ -1,99 +1,81 @@
 include:
-#  - local: 'ci_templates/.cic-template.yml'
-#  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
+  # these includes are app specific unit tests
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-#  - local: 'apps/data-seeding/.gitlab-ci.yml'
-
-image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest  
-
+    #- local: 'apps/contract-migration/.gitlab-ci.yml'
+    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
+  - version
   - build
   - test
   - deploy
 
+image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
+
 variables:
   DOCKER_BUILDKIT: "1"
-  COMPOSE_DOCKER_CLI_BUILD: "1" 
-  MR_IMAGE_TAG: mr-$CI_COMMIT_SHORT_SHA
+  COMPOSE_DOCKER_CLI_BUILD: "1"
+  CI_DEBUG_TRACE: "true"
+  SEMVERBOT_VERSION: "0.2.0"
 
-# todo you can probably just build the single image w/o docker-compose
-build-merge-request:
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-  stage: build
+  #before_script:
+  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+version:
+  #image: python:3.7-stretch
+  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
+  stage: version
   tags:
     - integration
-  variables:
-    CI_DEBUG_TRACE: "true"
   script:
-    - TAG=$MR_IMAGE_TAG FRONTEND_ENV=dev sh ./scripts/build-push.sh
+    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
+    - export TAG=$(sbot predict version -m auto)
+    - |
+      if [[ -z $TAG ]]
+      then
+        echo "tag could not be set $@"
+        exit 1
+      fi
+    - echo $TAG > version
+    - git tag -a v$TAG -m "ci tagged"
+    - git push origin v$TAG
+  artifacts:
+    paths:
+      - version
   rules:
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
     when: always
 
-build-staging:
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-  tags:
-    - integration
-       #- blocal
-  variables: 
-    CI_DEBUG_TRACE: "true"
+# runs on protected branches and pushes to repo
+build-push:
   stage: build
-  script:
-    - TAG=stag FRONTEND_ENV=staging sh ./scripts/build-push.sh
-  only:
-    - staging 
-
-deploy-staging:
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - pip install docker-auto-labels
   tags:
     - integration
-  stage: deploy
-  script: 
-        - >
-          DOMAIN=stag.grassrootseconomics.net
-          TRAEFIK_TAG=grassrootseconomics.net
-          STACK_NAME=stag-cic-net
-          TAG=stag
-          sh ./scripts/deploy.sh
-  environment:
-    name: staging
-    url: https://stag.grassrootseconomics.net
-  only:
-    - staging 
+  #script:
+  #  - TAG=$CI_Cbefore_script:
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  script:
+    - TAG=latest ./scripts/build-push.sh
+    - TAG=$(cat ./version) ./scripts/build-push.sh
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
-      #build-prod:
-      #  stage: build
-      #  script:
-      #    - TAG=prod FRONTEND_ENV=production sh ./scripts/build-push.sh
-      #  only:
-      #    - production
-      #  tags:
-      #    - build
-      #    - test
-      #
-      #deploy-prod:
-      #  stage: deploy
-      #  script:
-      #    - >
-      #      DOMAIN=demo1.com
-      #      TRAEFIK_TAG=demo1.com
-      #      STACK_NAME=demo1-com
-      #      TAG=prod
-      #      sh ./scripts/deploy.sh
-      #  environment:
-      #    name: production
-      #    url: https://demo1.com
-      #  only:
-      #    - production
-      #  tags:
-      #    - swarm
-      #    - prod
+deploy-dev:
+  stage: deploy
+  trigger: grassrootseconomics/devops
+  when: manual

.gitlab/issue_templates/bug.md

@@ -0,0 +1,44 @@
+<!---
+Please read this!
+
+Before opening a new issue, make sure to search for keywords in the issues
+filtered by the "bug" label:
+
+- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
+
+and verify the issue you're about to submit isn't a duplicate.
+--->
+
+### Summary
+
+<!-- Summarize the bug encountered concisely. -->
+
+### Steps to reproduce
+
+<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
+
+### Example Project
+
+<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic 
+behavior, and link to it here in the bug report. If you are using an older version of GitLab, this 
+will also determine whether the bug is fixed in a more recent version. -->
+
+### What is the current *bug* behavior?
+
+<!-- Describe what actually happens. -->
+
+### What is the expected *correct* behavior?
+
+<!-- Describe what you should see instead. -->
+
+### Relevant logs and/or screenshots
+
+<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
+ as it's tough to read otherwise. -->
+
+
+### Possible fixes
+
+<!-- If you can, link to the line of code that might be responsible for the problem. -->
+
+/label ~"bug"

.semverbot.toml

@@ -0,0 +1,16 @@
+[git]
+
+[git.config]
+email = "semverbot@grassroots.org"
+name = "semvervot"
+
+[git.tags]
+prefix = "v"
+
+[semver]
+mode = "git-commit"
+
+[semver.detection]
+patch = ["fix", "[fix]", "patch", "[patch]"]
+minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
+major = ["BREAKING CHANGE"]

CODE_OF_CONDUCT.md

@@ -0,0 +1,83 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
+
+
+
+
+...Try to keep in mind the immortal
+words of Bill and Ted, "Be excellent to each other."
+

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider. 
+
+__To request a change to the code please fork this repository and sumbit a merge request.__
+
+__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
+
+__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
+
+__Ask a question in our dev chat:__
+
+[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
+
+[Discord](https://discord.gg/XWunwAsX)
+
+[Matrix, IRC soon?]
+

CONTRIBUTING_CORE.md

@@ -0,0 +1,117 @@
+# CORE TEAM CONTRIBUTION GUIDE
+
+# 1. Transparency
+
+1.1 Use work logs for reflection of work done, aswell as telling your peers about changes that may affect their own tasks
+
+1.2 A work log SHOULD be submitted after a "unit of work" is complete.
+
+1.2.1 A "unit of work" should not span more than one full day's worth of work.
+
+1.2.2 A "unit of work" should be small enough that the log entries give useful insight.
+
+1.3 Individual logs are reviewed in weekly meetings
+
+<!--1.4 Bullet point list of topics and one or more sub-points describing each item in short sentences, eg;
+
+```
+- Core
+	* fixed foo
+	* fixed bar
+- Frontend
+	* connected bar to baz
+
+```-->
+
+1.4 Work log format is defined in []()
+
+1.5 Link to issue/MR in bullet point where appropriate
+
+1.6 
+
+
+# 2. Code hygiene
+
+2.1 Keep function names and variable names short
+
+2.2 Keep code files, functions and test fixtures short
+
+2.3 The less magic the better. Recombinable and replaceable is king
+
+2.4 Group imports by `standard`, `external`, `local`, `test` - in that order
+
+2.5 Only auto-import when necessary, and always with a minimum of side-effects
+
+2.6 Use custom errors. Let them bubble up
+
+2.7 No logs in tight loops
+
+2.8 Keep executable main routine minimal. Pass variables (do not use globals) in main business logic function
+
+2.9 Test coverage MUST be kept higher than 90% after changes
+
+2.10 Docstrings. Always. Always!
+
+
+# 3. Versioning
+
+3.1 Use [Semantic Versioning](https://semver.org/)
+
+3.2 When merging code, explicit dependencies SHOULD NOT use pre-release version
+
+
+# 4. Issues
+
+4.1 Issue title should use [Convention Commit structure](https://www.conventionalcommits.org/en/v1.0.0-beta.2/)
+
+4.2 Issues need proper problem statement
+
+4.2.1. What is the current state
+
+4.2.2. If current state is not behaving as expected, what was the expected state
+
+4.2.3. What is the desired new state.
+
+4.3  Issues need proper resolution statement
+
+4.3.1. Bullet point list of short sentences describing practical steps to reach desired state
+		
+4.3.2. Builet point list of external resources informing the issue and resolution
+
+4.4 Tasks needs to be appropriately labelled using GROUP labels.
+
+
+# 5. Code submission
+
+5.1 A branch and new MR is always created BEFORE THE WORK STARTS
+
+5.2 An MR should solve ONE SINGLE PART of a problem
+
+5.3 Every MR should have at least ONE ISSUE associated with it. Ideally issue can be closed when MR is merged
+
+5.4 MRs should not be open for more than one week (during normal operation periods)
+
+5.5 MR should ideally not be longer than 400 lines of changes of logic
+
+5.6 MRs that MOVE or DELETE code should not CHANGE that same code in a single MR. Scope MOVEs and DELETEs in separate commits (or even better, separate MRs) for transparency
+
+
+# 6. Code reviews
+
+6.1 At least one peer review before merge
+
+6.2 If MR is too long, evaluate whether this affects the quality of the review negatively. If it does, expect to be asked to split it up
+
+6.3 Evaluate changes against associated issues' problem statement and proposed resolution steps. If there is a mismatch, either MR needs to change or issue needs to be amended accordingly
+
+6.4 Make sure all technical debt introduced by MR is documented in issues. Add them according to criteria in section ISSUES if not
+
+6.5 If CI is not working, reviewer MUST make sure code builds and runs
+
+6.6 Behave!
+
+6.6.1 Don't be a jerk
+
+6.6.2 Don't block needlessly
+
+6.6.3 Say please

LICENSE

@@ -1,5 +1,5 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
 
  Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
@@ -7,17 +7,15 @@
 
                             Preamble
 
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
 
   The licenses for most software and other practical works are designed
 to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
+our General Public Licenses are intended to guarantee your freedom to
 share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
+software for all its users.
 
   When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
@@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you
 want it, that you can change the software or use pieces of it in new
 free programs, and that you know you can do these things.
 
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
 
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
 
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
 
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
@@ -72,7 +60,7 @@ modification follow.
 
   0. Definitions.
 
-  "This License" refers to version 3 of the GNU General Public License.
+  "This License" refers to version 3 of the GNU Affero General Public License.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
@@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey
 the Program, the only way you could satisfy both those terms and this
 License would be to refrain entirely from conveying the Program.
 
-  13. Use with the GNU Affero General Public License.
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
 
   Notwithstanding any other provision of this License, you have
 permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
+under version 3 of the GNU General Public License into a single
 combined work, and to convey the resulting work.  The terms of this
 License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
 
   14. Revised Versions of this License.
 
   The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
 address new problems or concerns.
 
   Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
+Program specifies that a certain numbered version of the GNU Affero General
 Public License "or any later version" applies to it, you have the
 option of following the terms and conditions either of that numbered
 version or of any later version published by the Free Software
 Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
+GNU Affero General Public License, you may choose any version ever published
 by the Free Software Foundation.
 
   If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
+versions of the GNU Affero General Public License can be used, that proxy's
 public statement of acceptance of a version permanently authorizes you
 to choose that version for the Program.
 
@@ -631,44 +629,33 @@ to attach them to the start of each source file to most effectively
 state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
-    CIC Staff Client
+    cic-internal-integration
     Copyright (C) 2021  Grassroots Economics
 
     This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
     (at your option) any later version.
 
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+    GNU Affero General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
+    You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) 2021  Grassroots Economics
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
 
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
+For more information on this, and how to apply and follow the GNU AGPL, see
 <https://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.

README.md

@@ -1,104 +1,19 @@
-# cic-internal-integration
+# Community Inclusion Currency Stack (CIC Stack) 
 
-## Backend Requirements
+A custodial evm wallet for executing transactions via USSD
 
-* [Docker](https://www.docker.com/).
-* [Docker Compose](https://docs.docker.com/compose/install/).
+## Getting started 
 
-## Backend local development
-
-* Start the stack with Docker Compose:
-
-```bash
-docker-compose up -d
-```
-
-* Now you can open your browser and interact with these URLs:
-
-Frontend (CICADA), built with Docker, with routes handled based on the path: http://localhost
-
-PGAdmin, PostgreSQL web administration: http://localhost:5050
-
-Flower, administration of Celery tasks: http://localhost:5555
-
-Traefik UI, to see how the routes are being handled by the proxy: http://localhost:8090
-
-**Note**: The first time you start your stack, it might take a minute for it to be ready. While the backend waits for the database to be ready and configures everything. You can check the logs to monitor it.
-
-To check the logs, run:
-
-```bash
-docker-compose logs
-```
-
-To check the logs of a specific service, add the name of the service, e.g.:
-
-```bash
-docker-compose logs backend
-```
-
-If your Docker is not running in `localhost` (the URLs above wouldn't work) check the sections below on **Development with Docker Toolbox** and **Development with a custom IP**.
-
-#### Deploy the stack locally
-
-If you want to run the docker stack locally on swarm 
+This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
 ```
- docker-compose -f docker-compose.yml -f docker-compose.override.yml  config > docker-stack.yml
+export COMPOSE_DOCKER_CLI_BUILD=1
+export DOCKER_BUILDKIT=1
 ```
 
-```
-docker node update z1ehkrw1mvqlxc2udwt4xpype --label-add cic-net.app-db-data=true
- docker stack deploy -c docker-stack.yml  cic-net  
-```
+To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
 
+## Documentation 
 
-## Backend local development, additional details
-
-**fill  me in**
-
-### Docker Compose Override
-
-During development, you can change Docker Compose settings that will only affect the local development environment, in the file `docker-compose.override.yml`.
-
-The changes to that file only affect the local development environment, not the production environment. So, you can add "temporary" changes that help the development workflow.
-
-For example, the directory with the backend code is mounted as a Docker "host volume", mapping the code you change live to the directory inside the container. That allows you to test your changes right away, without having to build the Docker image again. It should only be done during development, for production, you should build the Docker image with a recent version of the backend code. But during development, it allows you to iterate very fast.
-
-There is also a command override that runs `/start-reload.sh` (included in the base image) instead of the default `/start.sh` (also included in the base image). It starts a single server process (instead of multiple, as would be for production) and reloads the process whenever the code changes. Have in mind that if you have a syntax error and save the Python file, it will break and exit, and the container will stop. After that, you can restart the container by fixing the error and running again:
-
-```console
-$ docker-compose up -d
-```
-
-There is also a commented out `command` override, you can uncomment it and comment the default one. It makes the backend container run a process that does "nothing", but keeps the container alive. That allows you to get inside your running container and execute commands inside, for example a Python interpreter to test installed dependencies, or start the development server that reloads when it detects changes, or start a Jupyter Notebook session.
-
-To get inside the container with a `bash` session you can start the stack with:
-
-```console
-$ docker-compose up -d
-```
-
-and then `exec` inside the running container:
-
-```console
-$ docker-compose exec backend bash
-```
-
-You should see an output like:
-
-```console
-root@7f2607af31c3:/app#
-```
-
-
-
-#### Test running stack
-
-If your stack is already up and you just want to run the tests, you can use:
-
-```bash
-docker-compose exec data-seeding /script/run_ussd_user_imports.sh
-```
-
+[https://docs.grassecon.org/software/](https://docs.grassecon.org/software/)
 

apps/.dockerignore

@@ -1,5 +1,4 @@
 .git
 .cache
 .dot
-**/doc
-**/node_modules/
+**/doc

apps/cic-base-os/aux/wait-for-it/.gitignore

@@ -0,0 +1,3 @@
+**/*.pyc
+.pydevproject
+/vendor/

apps/cic-base-os/aux/wait-for-it/.travis.yml

@@ -0,0 +1,7 @@
+language: python
+python:
+  - "2.7"
+
+script:
+  - python test/wait-for-it.py
+

apps/cic-base-os/aux/wait-for-it/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apps/cic-base-os/aux/wait-for-it/README.md

@@ -0,0 +1,75 @@
+# wait-for-it
+
+`wait-for-it.sh` is a pure bash script that will wait on the availability of a
+host and TCP port.  It is useful for synchronizing the spin-up of
+interdependent services, such as linked docker containers.  Since it is a pure
+bash script, it does not have any external dependencies.
+
+## Usage
+
+```text
+wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
+-h HOST | --host=HOST       Host or IP under test
+-p PORT | --port=PORT       TCP port under test
+                            Alternatively, you specify the host and port as host:port
+-s | --strict               Only execute subcommand if the test succeeds
+-q | --quiet                Don't output any status messages
+-t TIMEOUT | --timeout=TIMEOUT
+                            Timeout in seconds, zero for no timeout
+-- COMMAND ARGS             Execute command with args after the test finishes
+```
+
+## Examples
+
+For example, let's test to see if we can access port 80 on `www.google.com`,
+and if it is available, echo the message `google is up`.
+
+```text
+$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+You can set your own timeout with the `-t` or `--timeout=` option.  Setting
+the timeout value to 0 will disable the timeout:
+
+```text
+$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting for www.google.com:80 without a timeout
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+The subcommand will be executed regardless if the service is up or not.  If you
+wish to execute the subcommand only if the service is up, add the `--strict`
+argument. In this example, we will test port 81 on `www.google.com` which will
+fail:
+
+```text
+$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
+wait-for-it.sh: waiting 1 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
+wait-for-it.sh: strict mode, refusing to execute subprocess
+```
+
+If you don't want to execute a subcommand, leave off the `--` argument.  This
+way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
+and determine how to proceed:
+
+```text
+$ ./wait-for-it.sh www.google.com:80
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+$ echo $?
+0
+$ ./wait-for-it.sh www.google.com:81
+wait-for-it.sh: waiting 15 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
+$ echo $?
+124
+```
+
+## Community
+
+*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-base-os/aux/wait-for-it/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-cache/.gitlab-ci.yml

@@ -1,41 +1,17 @@
-.cic_cache_variables:
-    variables:
-        APP_NAME: cic-cache
-
-#build-mr-cic-cache:
-#    extends:
-#        - .py_build_merge_request
-#        - .cic_cache_variables
-#    rules:
-#    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-#      changes:
-#          - apps/cic-cache/**/*
-#      when: always
-
-test-mr-cic-cache:
-    stage: test
-    tags:
-      - integration
-    extends:
-    - .cic_cache_variables
-    cache:
-      key:
-        files:
-        - test_requirements.txt
-      paths:
-      - /root/.cache/pip
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/$APP_NAME:$MR_IMAGE_TAG
-    script:
-      - cd apps/$APP_NAME/
-      - >
-        pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-        --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-        -r test_requirements.txt 
-      - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    allow_failure: true
-    needs: ["build-merge-request"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
+build-test-cic-cache:
+  stage: test 
+  tags:
+    - integration
+  variables:
+    APP_NAME: cic-cache
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+  script:
+    - cd apps/cic-cache
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+  allow_failure: true
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    changes:
+        - apps/$APP_NAME/**/*
+    when: always

apps/cic-cache/MANIFEST.in

@@ -1 +1 @@
-include *requirements.txt cic_cache/data/config/*
+include *requirements.txt cic_cache/data/config/* cic_cache/db/migrations/default/* cic_cache/db/migrations/default/versions/*

apps/cic-cache/README.md

@@ -0,0 +1 @@
+# CIC-CACHE

apps/cic-cache/aux/wait-for-it/.gitignore

@@ -0,0 +1,3 @@
+**/*.pyc
+.pydevproject
+/vendor/

apps/cic-cache/aux/wait-for-it/.travis.yml

@@ -0,0 +1,7 @@
+language: python
+python:
+  - "2.7"
+
+script:
+  - python test/wait-for-it.py
+

apps/cic-cache/aux/wait-for-it/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apps/cic-cache/aux/wait-for-it/README.md

@@ -0,0 +1,75 @@
+# wait-for-it
+
+`wait-for-it.sh` is a pure bash script that will wait on the availability of a
+host and TCP port.  It is useful for synchronizing the spin-up of
+interdependent services, such as linked docker containers.  Since it is a pure
+bash script, it does not have any external dependencies.
+
+## Usage
+
+```text
+wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
+-h HOST | --host=HOST       Host or IP under test
+-p PORT | --port=PORT       TCP port under test
+                            Alternatively, you specify the host and port as host:port
+-s | --strict               Only execute subcommand if the test succeeds
+-q | --quiet                Don't output any status messages
+-t TIMEOUT | --timeout=TIMEOUT
+                            Timeout in seconds, zero for no timeout
+-- COMMAND ARGS             Execute command with args after the test finishes
+```
+
+## Examples
+
+For example, let's test to see if we can access port 80 on `www.google.com`,
+and if it is available, echo the message `google is up`.
+
+```text
+$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+You can set your own timeout with the `-t` or `--timeout=` option.  Setting
+the timeout value to 0 will disable the timeout:
+
+```text
+$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting for www.google.com:80 without a timeout
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+The subcommand will be executed regardless if the service is up or not.  If you
+wish to execute the subcommand only if the service is up, add the `--strict`
+argument. In this example, we will test port 81 on `www.google.com` which will
+fail:
+
+```text
+$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
+wait-for-it.sh: waiting 1 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
+wait-for-it.sh: strict mode, refusing to execute subprocess
+```
+
+If you don't want to execute a subcommand, leave off the `--` argument.  This
+way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
+and determine how to proceed:
+
+```text
+$ ./wait-for-it.sh www.google.com:80
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+$ echo $?
+0
+$ ./wait-for-it.sh www.google.com:81
+wait-for-it.sh: waiting 15 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
+$ echo $?
+124
+```
+
+## Community
+
+*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-cache/aux/wait-for-it/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-cache/cic_cache/cli/arg.py

@@ -14,7 +14,7 @@ class ArgumentParser(BaseArgumentParser):
         if local_arg_flags & CICFlag.CELERY:
             self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--offset', type=int, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
         if local_arg_flags & CICFlag.CHAIN:
             self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/cli/rpc.py

@@ -27,11 +27,11 @@ class RPC:
     @staticmethod
     def from_config(config):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
+        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
         if config.get('SIGNER_PROVIDER'):
             RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
             RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
-        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc
 

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -1,4 +1,4 @@
 [cic]
 registry_address = 
 trust_address =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
+health_modules =

apps/cic-cache/cic_cache/data/config/database.ini

@@ -3,7 +3,8 @@ engine =
 driver =
 host =
 port =
-name = cic-cache
+#name = cic-cache
+prefix =
 user =
 password =
 debug = 0

apps/cic-cache/cic_cache/db/__init__.py

@@ -9,21 +9,26 @@ from .list import (
         tag_transaction,
         add_tag,
     )
+from cic_cache.db.models.base import SessionBase
 
 
 logg = logging.getLogger()
 
 
-def dsn_from_config(config):
+def dsn_from_config(config, name):
     scheme = config.get('DATABASE_ENGINE')
     if config.get('DATABASE_DRIVER') != None:
         scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
 
+    database_name = name
+    if config.get('DATABASE_PREFIX'):
+        database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
     dsn = ''
     if config.get('DATABASE_ENGINE') == 'sqlite':
+        SessionBase.poolable = False
         dsn = '{}:///{}'.format(
                 scheme,
-                config.get('DATABASE_NAME'),    
+                database_name,
             )
 
     else:
@@ -33,7 +38,7 @@ def dsn_from_config(config):
                 config.get('DATABASE_PASSWORD'),
                 config.get('DATABASE_HOST'),
                 config.get('DATABASE_PORT'),
-                config.get('DATABASE_NAME'),    
+                database_name,
             )
     logg.debug('parsed dsn from config: {}'.format(dsn))
     return dsn

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -5,7 +5,11 @@ import re
 import base64
 
 # external imports
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainlib.encode import TxHexNormalizer
 
 # local imports
 from cic_cache.cache import (
@@ -16,27 +20,72 @@ from cic_cache.cache import (
 logg = logging.getLogger(__name__)
 #logg = logging.getLogger()
 
-re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
+re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
 re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
-re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
+re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
+re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
+re_default_limit = r'/defaultlimit/?'
 
 DEFAULT_LIMIT = 100
 
+tx_normalize = TxHexNormalizer()
+
+def parse_query_account(r):
+    address = strip_0x(r[1])
+    #address = tx_normalize.wallet_address(address)
+    limit = DEFAULT_LIMIT
+    g = r.groups()
+    if len(g) > 3:
+        limit = int(r[4])
+        if limit == 0:
+            limit = DEFAULT_LIMIT
+    offset = 0
+    if len(g) > 4:
+        offset = int(r[6])
+
+    logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
+
+    return (address, offset, limit,)
+
+
+# r is an re.Match
+def parse_query_any(r):
+    limit = DEFAULT_LIMIT
+    offset = 0
+    block_offset = None
+    block_end = None
+    if r.lastindex != None:
+        if r.lastindex > 0:
+            limit = int(r[1])
+        if r.lastindex > 1:
+            offset = int(r[2])
+        if r.lastindex > 2:
+            block_offset = int(r[3])
+        if r.lastindex > 3:
+            block_end = int(r[4])
+            if block_end < block_offset:
+                raise ValueError('cart before the horse, dude')
+
+    logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
+
+    return (offset, limit, block_offset, block_end,)
+
+
+def process_default_limit(session, env):
+    r = re.match(re_default_limit, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
+
 
 def process_transactions_account_bloom(session, env):
     r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
     if not r:
         return None
+    logg.debug('match account bloom')
 
-    address = r[1]
-    if r[2] == None:
-        address = add_0x(address)
-    offset = 0
-    if r.lastindex > 2:
-        offset = r[4]
-    limit = DEFAULT_LIMIT
-    if r.lastindex > 4:
-        limit = r[6]
+    (address, offset, limit,) = parse_query_account(r)
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
@@ -59,13 +108,9 @@ def process_transactions_all_bloom(session, env):
     r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
     if not r:
         return None
+    logg.debug('match all bloom')
 
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 0:
-        offset = r[1]
-    limit = 0
-    if r.lastindex > 1:
-        limit = r[2]
+    (limit, offset, block_offset, block_end,) = parse_query_any(r)
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
@@ -88,17 +133,16 @@ def process_transactions_all_data(session, env):
     r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
     if not r:
         return None
-    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-        return None
+    #if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+    #    return None
+    logg.debug('match all data')
 
     logg.debug('got data request {}'.format(env))
-    block_offset = r[1]
-    block_end = r[2]
-    if int(r[2]) < int(r[1]):
-        raise ValueError('cart before the horse, dude')
+
+    (offset, limit, block_offset, block_end) = parse_query_any(r)
 
     c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
 
     for r in tx_cache:
         r['date_block'] = r['date_block'].timestamp()
@@ -113,3 +157,30 @@ def process_transactions_all_data(session, env):
     j = json.dumps(o)
 
     return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_account_data(session, env):
+    r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
+    if not r:
+        return None
+    logg.debug('match account data')
+    #if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+    #    return None
+
+    (address, offset, limit,) = parse_query_account(r)
+
+    c = DataCache(session)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
+
+    for r in tx_cache:
+        r['date_block'] = r['date_block'].timestamp()
+
+    o = {
+        'low': lowest_block,
+        'high': highest_block,
+        'data': tx_cache,
+    }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -8,42 +8,33 @@ import base64
 import confini
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.runnable.daemons.query import (
+        process_default_limit,
         process_transactions_account_bloom,
+        process_transactions_account_data,
         process_transactions_all_bloom,
         process_transactions_all_data,
         )
+import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
+arg_flags = cic_cache.cli.argflag_std_read
+local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task 
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
+# connect to database
+dsn = dsn_from_config(config, 'cic_cache')
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 
 
@@ -55,9 +46,11 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
+            process_transactions_account_data,
+            process_transactions_account_bloom,
             process_transactions_all_data,
             process_transactions_all_bloom,
-            process_transactions_account_bloom,
+            process_default_limit,
             ]:
         r = None
         try:

apps/cic-cache/cic_cache/runnable/daemons/tasker.py

@@ -3,12 +3,14 @@ import logging
 import os
 import sys
 import argparse
+import tempfile
 
 # third-party imports
 import celery
 import confini
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.tasks.tx import *
@@ -16,35 +18,20 @@ from cic_cache.tasks.tx import *
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-
+# process args
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_task
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
 # connect to database
-dsn = dsn_from_config(config)
+dsn = dsn_from_config(config, 'cic_cache')
 SessionBase.connect(dsn)
 
-# verify database connection with minimal sanity query
-#session = SessionBase.create_session()
-#session.execute('select version_num from alembic_version')
-#session.close()
-
 # set up celery
 current_app = celery.Celery(__name__)
 
@@ -87,9 +74,9 @@ def main():
     elif args.v:
         argv.append('--loglevel=INFO')
     argv.append('-Q')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
     argv.append('-n')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
 
     current_app.worker_main(argv)
 

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -40,7 +40,7 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 # process args
-arg_flags = cic_cache.cli.argflag_std_read
+arg_flags = cic_cache.cli.argflag_std_base
 local_arg_flags = cic_cache.cli.argflag_local_sync
 argparser = cic_cache.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)
@@ -50,7 +50,7 @@ args = argparser.parse_args()
 config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
 # connect to database
-dsn = dsn_from_config(config)
+dsn = dsn_from_config(config, 'cic_cache')
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
 # set up rpc
@@ -95,10 +95,10 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_OFFSET')
-        initial_block_offset = block_offset
+        initial_block_start = int(config.get('SYNCER_OFFSET'))
+        initial_block_offset = int(block_offset)
         if config.get('SYNCER_NO_HISTORY'):
-            initial_block_start = block_offset
+            initial_block_start = initial_block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
         logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))

apps/cic-cache/cic_cache/version.py

@@ -5,7 +5,7 @@ version = (
         0,
         2,
         1,
-        'alpha.1',
+        'alpha.3',
         )
 
 version_object = semver.VersionInfo(

apps/cic-cache/config/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis:///
-result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -1,3 +0,0 @@
-[cic]
-registry_address = 
-trust_address = 

apps/cic-cache/config/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/cic.ini

@@ -1,3 +0,0 @@
-[cic]
-registry_address =  
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=grassroots
-PASSWORD=
-HOST=localhost
-PORT=63432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/syncer.ini

@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0

apps/cic-cache/config/test/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/test/cic.ini

@@ -1,4 +1,3 @@
 [cic]
 registry_address = 
-chain_spec = 
 trust_address =

apps/cic-cache/config/test/database.ini

@@ -1,5 +1,5 @@
 [database]
-NAME=cic-cache-test
+PREFIX=cic-cache-test
 USER=postgres
 PASSWORD=
 HOST=localhost

apps/cic-cache/config/test/eth.ini

@@ -1,5 +0,0 @@
-[eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
-provider = http://localhost:8545
-#chain_id = 

apps/cic-cache/doc/openapi/server.yml

@@ -1,4 +1,4 @@
-openapi: "3.0.3"
+openapi: "3.0.2"
 info:
   title: Grassroots Economics CIC Cache
   description: Cache of processed transaction data from Ethereum blockchain and worker queues
@@ -9,17 +9,34 @@ info:
     email: will@grassecon.org
   license:
     name: GPLv3
-  version: 0.1.0
+  version: 0.2.0
 
 paths:
-  /tx/{offset}/{limit}:
-    description: Bloom filter for batch of latest transactions
+  /defaultlimit:
+    summary: The default limit value of result sets.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve default limit
+      operationId: limit.default
+      responses:
+        200:
+          description: Limit query successful
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Limit"
+
+  /tx:
+    summary: Bloom filter for batch of latest transactions
+    description: Generate a bloom filter of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
     get:
       tags:
         - transactions
       description:
         Retrieve transactions
-      operationId: tx.get
+      operationId: tx.get.latest
       responses:
         200:
           description: Transaction query successful.
@@ -29,27 +46,153 @@ paths:
                 $ref: "#/components/schemas/BlocksBloom"
 
 
-    parameters:
-    - name: offset
-      in: path
-      schema:
-        type: integer
-        format: int32
-    - name: limit
-      in: path
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{address}/{offset}/{limit}:
-    description: Bloom filter for batch of latest transactions by account
+  /tx/{limit}:
+    summary: Bloom filter for batch of latest transactions
+    description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
     get:
       tags:
         - transactions
       description:
         Retrieve transactions
-      operationId: tx.get
+      operationId: tx.get.latest.limit
+      responses:
+        200:
+          description: Transaction query successful. Results are ordered from newest to oldest.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BlocksBloom"
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /tx/{limit}/{offset}:
+    summary: Bloom filter for batch of latest transactions
+    description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: tx.get.latest.range
+      responses:
+        200:
+          description: Transaction query successful. Results are ordered from newest to oldest.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BlocksBloom"
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /tx/{limit}/{offset}/{block_offset}:
+    summary: Bloom filter for batch of transactions since a particular block.
+    description: Generate a bloom filter of the latest transactions since a particular block in the cache. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: tx.get.latest.range.block.offset
+      responses:
+        200:
+          description: Transaction query successful. Results are ordered from oldest to newest.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BlocksBloom"
+
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: block_offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /tx/{limit}/{offset}/{block_offset}/{block_end}:
+    summary: Bloom filter for batch of transactions within a particular block range.
+    description: Generate a bloom filter of the latest transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: tx.get.latest.range.block.range
+      responses:
+        200:
+          description: Transaction query successful.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BlocksBloom"
+
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: block_offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: block_end
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /tx/{address}:
+    summary: Bloom filter for batch of latest transactions by account.
+    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: tx.get.user
       responses:
         200:
           description: Transaction query successful. 
@@ -58,6 +201,30 @@ paths:
               schema:
                 $ref: "#/components/schemas/BlocksBloom"
 
+    parameters:
+    - name: address
+      in: path
+      required: true
+      schema:
+        type: string
+
+
+  /tx/{address}/{limit}:
+    summary: Bloom filter for batch of latest transactions by account.
+    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: tx.get.user.limit
+      responses:
+        200:
+          description: Transaction query successful. 
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BlocksBloom"
 
     parameters:
     - name: address
@@ -65,26 +232,317 @@ paths:
       required: true
       schema:
         type: string
-    - name: offset
-      in: path
-      schema:
-        type: integer
-        format: int32
     - name: limit
       in: path
+      required: true
       schema:
         type: integer
         format: int32
 
+
+  /tx/{address}/{limit}/{offset}:
+    summary: Bloom filter for batch of latest transactions by account
+    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: tx.get.user.range
+      responses:
+        200:
+          description: Transaction query successful. 
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BlocksBloom"
+
+    parameters:
+    - name: address
+      in: path
+      required: true
+      schema:
+        type: string
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /txa:
+    summary: Cached data for latest transactions.
+    description: Return data entries of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.latest
+      responses:
+        200:
+          description: Transaction query successful.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    
+  /txa/{limit}:
+    summary: Cached data for latest transactions.
+    description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.latest.limit
+      responses:
+        200:
+          description: Transaction query successful.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /txa/{limit}/{offset}:
+    summary: Cached data for latest transactions.
+    description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.latest.range
+      responses:
+        200:
+          description: Transaction query successful.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /txa/{limit}/{offset}/{block_offset}:
+    summary: Cached data for transactions since a particular block.
+    description: Return cached data entries of transactions since a particular block. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.latest.range.block.offset
+      responses:
+        200:
+          description: Transaction query successful.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: block_offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+  /txa/{limit}/{offset}/{block_offset}/{block_end}:
+    summary: Cached data for transactions within a particular block range.
+    description: Return cached data entries of transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.latest.range.block.range
+      responses:
+        200:
+          description: Transaction query successful.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    parameters:
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: block_offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: block_end
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /txa/{address}:
+    summary: Cached data for batch of latest transactions by account.
+    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.user
+      responses:
+        200:
+          description: Transaction query successful. 
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    parameters:
+    - name: address
+      in: path
+      required: true
+      schema:
+        type: string
+
+
+  /txa/{address}/{limit}:
+    summary: Cached data for batch of latest transactions by account.
+    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.user.limit
+      responses:
+        200:
+          description: Transaction query successful. 
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    parameters:
+    - name: address
+      in: path
+      required: true
+      schema:
+        type: string
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
+  /txa/{address}/{limit}/{offset}:
+    summary: Cached data for batch of latest transactions by account.
+    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+    get:
+      tags:
+        - transactions
+      description:
+        Retrieve transactions
+      operationId: txa.get.user.range
+      responses:
+        200:
+          description: Transaction query successful. 
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/TransactionList"
+
+    parameters:
+    - name: address
+      in: path
+      required: true
+      schema:
+        type: string
+    - name: limit
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+    - name: offset
+      in: path
+      required: true
+      schema:
+        type: integer
+        format: int32
+
+
 components:
   schemas:
+    Limit:
+      type: integer
+      format: int32
     BlocksBloom:
       type: object
       properties:
         low:
-          type: int
+          type: integer
           format: int32
           description: The lowest block number included in the filter
+        high:
+          type: integer
+          format: int32
+          description: The highest block number included in the filter
         block_filter:
           type: string
           format: byte
@@ -97,6 +555,89 @@ components:
           type: string
           description: Hashing algorithm (currently only using sha256)
         filter_rounds:
-          type: int
+          type: integer
           format: int32
           description: Number of hash rounds used to create the filter
+    TransactionList:
+      type: object
+      properties:
+        low:
+          type: integer
+          format: int32
+          description: The lowest block number included in the result set
+        high:
+          type: integer
+          format: int32
+          description: The highest block number included in the filter
+        data:
+          type: array
+          description: Cached transaction data
+          items:
+            $ref: "#/components/schemas/Transaction"
+    Transaction:
+      type: object
+      properties:
+        block_number:
+          type: integer
+          format: int64
+          description: Block number transaction was included in.
+        tx_hash:
+          type: string
+          description: Transaction hash, in hex.
+        date_block:
+          type: integer
+          format: int32
+          description: Block timestamp.
+        sender:
+          type: string
+          description: Spender address, in hex.
+        recipient:
+          type: string
+          description: Beneficiary address, in hex.
+        from_value:
+          type: integer
+          format: int64
+          description: Value deducted from spender's balance.
+        to_value:
+          type: integer
+          format: int64
+          description: Value added to beneficiary's balance.
+        source_token:
+          type: string
+          description: Network address of token in which `from_value` is denominated.
+        destination_token:
+          type: string
+          description: Network address of token in which `to_value` is denominated.
+        success:
+          type: boolean
+          description: Network consensus state on whether the transaction was successful or not.
+        tx_type:
+          type: string
+          enum:
+            - erc20.faucet
+            - faucet.give_to
+
+  examples:
+    data_last:
+      summary: Get the latest cached transactions, using the server's default limit. 
+      value: "/txa"
+
+    data_limit:
+      summary: Get the last 42 cached transactions. 
+      value: "/txa/42"
+
+    data_range:
+      summary: Get the next 42 cached transactions, starting from the 13th (zero-indexed).
+      value: "/txa/42/13"
+
+    data_range_block_offset:
+      summary: Get the next 42 cached transactions, starting from block 1337 (inclusive).
+      value: "/txa/42/0/1337"
+
+    data_range_block_offset:
+      summary: Get the next 42 cached transactions within blocks 1337 and 1453 (inclusive).
+      value: "/txa/42/0/1337/1453"
+
+    data_range_block_range:
+      summary: Get the next 42 cached transactions after the 13th, within blocks 1337 and 1453 (inclusive).
+      value: "/txa/42/13/1337/1453"

apps/cic-cache/docker/Dockerfile

@@ -1,31 +1,25 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 
 COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
+ARG EXTRA_PIP_ARGS=""
+ARG PIP_INDEX_URL=https://pypi.org/simple
+
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt
 
-COPY . . 
 
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
+COPY . .
+RUN pip install .  --extra-index-url $EXTRA_PIP_INDEX_URL
 
 # for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY ./aux/wait-for-it/wait-for-it.sh ./
 COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
 COPY /docker/start_tracker.sh ./start_tracker.sh

apps/cic-cache/docker/db.sh

@@ -2,5 +2,5 @@
 
 set -e
 >&2 echo executing database migration
-python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
+python scripts/migrate_cic_cache.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
 set +e

apps/cic-cache/docker/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+

apps/cic-cache/requirements.txt

@@ -1,17 +1,15 @@
 alembic==1.4.2
-confini>=0.3.6rc4,<0.5.0
+confini~=0.5.3
 uwsgi==2.0.19.1
-moolb~=0.1.1b2
-cic-eth-registry~=0.5.8a1
+moolb~=0.2.0
+cic-eth-registry~=0.6.6
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]>=0.0.6a1,<0.1.0
-erc20-faucet>=0.2.4a2, <0.3.0
-#chainlib-eth==0.0.7a5,<0.1.0
-chainlib-eth==0.0.7a5
-#chainlib==0.0.7a4,<0.1.0
-chainlib==0.0.7a4
-eth-address-index>=0.1.4a1,<0.2.0
+chainsyncer[sql]~=0.0.7
+erc20-faucet~=0.3.2
+chainlib-eth~=0.0.15
+eth-address-index~=0.2.4
+okota~=0.2.5

apps/cic-cache/scripts/migrate_cic_cache.py

@@ -1,53 +1,55 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+
+# standard imports
 import os
 import argparse
 import logging
 import re
 
+# external imports
 import alembic
 from alembic.config import Config as AlembicConfig
 import confini
 
+# local imports
 from cic_cache.db import dsn_from_config
+import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 # BUG: the dbdir doesn't work after script install
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(cic_cache.__file__)))
 dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
+default_migrations_dir = os.path.join(dbdir, 'migrations')
+configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
+#config_dir = os.path.join('/usr/local/etc/cic-cache')
 
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_sync
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
+argparser.add_argument('-f', '--force', action='store_true', help='force action')
+argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='migrations directory')
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
+extra_args = {
+    'reset': None,
+    'force': None,
+    'migrations_dir': None,
+        }
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
+migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE', 'default'))
 if not os.path.isdir(migrations_dir):
     logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
     migrations_dir = os.path.join(args.migrations_dir, 'default')
 
 # connect to database
-dsn = dsn_from_config(config)
+dsn = dsn_from_config(config, 'cic_cache')
 
 
 logg.info('using migrations dir {}'.format(migrations_dir))

apps/cic-cache/setup.cfg

@@ -1,6 +1,7 @@
 [metadata]
 name = cic-cache
 description = CIC Cache API and server
+version = 0.3.0a2
 author = Louis Holbrook
 author_email = dev@holbrook.no
 url = https://gitlab.com/grassrootseconomics/cic-eth
@@ -34,7 +35,7 @@ packages =
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters
 scripts =
-	./scripts/migrate.py
+	./scripts/migrate_cic_cache.py
 
 [options.entry_points]
 console_scripts =

apps/cic-cache/setup.py

@@ -1,38 +1,39 @@
 from setuptools import setup
 
-import configparser
+# import configparser
 import os
-import time
 
-from cic_cache.version import (
-        version_object,
-        version_string
-        )
+# import time
 
-class PleaseCommitFirstError(Exception):
-    pass
-
-def git_hash():
-    import subprocess
-    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
-    if len(git_diff.stdout) > 0:
-        raise PleaseCommitFirstError()
-    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-    return git_hash_brief
-
-version_string = str(version_object)
-
-try:
-    version_git = git_hash()
-    version_string += '+build.{}'.format(version_git)
-except FileNotFoundError:
-    time_string_pair = str(time.time()).split('.')
-    version_string += '+build.{}{:<09d}'.format(
-            time_string_pair[0],
-            int(time_string_pair[1]),
-            )
-print('final version string will be {}'.format(version_string))
+# from cic_cache.version import (
+#        version_object,
+#        version_string
+#        )
+#
+# class PleaseCommitFirstError(Exception):
+#    pass
+#
+# def git_hash():
+#    import subprocess
+#    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
+#    if len(git_diff.stdout) > 0:
+#        raise PleaseCommitFirstError()
+#    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+#    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+#    return git_hash_brief
+#
+# version_string = str(version_object)
+#
+# try:
+#    version_git = git_hash()
+#    version_string += '+build.{}'.format(version_git)
+# except FileNotFoundError:
+#    time_string_pair = str(time.time()).split('.')
+#    version_string += '+build.{}{:<09d}'.format(
+#            time_string_pair[0],
+#            int(time_string_pair[1]),
+#            )
+# print('final version string will be {}'.format(version_string))
 
 requirements = []
 f = open('requirements.txt', 'r')
@@ -52,9 +53,8 @@ while True:
     test_requirements.append(l.rstrip())
 f.close()
 
-
 setup(
-    version=version_string,
+    #    version=version_string,
     install_requires=requirements,
     tests_require=test_requirements,
         )

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-sarafu-faucet~=0.0.5a2
-erc20-transfer-authorization>=0.3.4a1,<0.4.0
+sarafu-faucet~=0.0.7a1
+erc20-transfer-authorization~=0.3.6

apps/cic-cache/tests/conftest.py

@@ -6,6 +6,7 @@ import datetime
 # external imports
 import pytest
 import moolb
+from chainlib.encode import TxHexNormalizer
 
 # local imports
 from cic_cache import db
@@ -42,6 +43,8 @@ def txs(
         list_tokens,
         ):
 
+    tx_normalize = TxHexNormalizer()
+
     session = init_database
 
     tx_number = 13 
@@ -54,10 +57,10 @@ def txs(
         tx_hash_first,
         list_defaults['block'],
         tx_number,
-        list_actors['alice'],
-        list_actors['bob'],
-        list_tokens['foo'],
-        list_tokens['foo'],
+        tx_normalize.wallet_address(list_actors['alice']),
+        tx_normalize.wallet_address(list_actors['bob']),
+        tx_normalize.executable_address(list_tokens['foo']),
+        tx_normalize.executable_address(list_tokens['foo']),
         1024,
         2048,
         True,
@@ -74,10 +77,10 @@ def txs(
         tx_hash_second,
         list_defaults['block']-1,
         tx_number,
-        list_actors['diane'],
-        list_actors['alice'],
-        list_tokens['foo'],
-        list_tokens['foo'],
+        tx_normalize.wallet_address(list_actors['diane']),
+        tx_normalize.wallet_address(list_actors['alice']),
+        tx_normalize.executable_address(list_tokens['foo']),
+        tx_normalize.wallet_address(list_tokens['foo']),
         1024,
         2048,
         False,
@@ -103,6 +106,8 @@ def more_txs(
 
     session = init_database
 
+    tx_normalize = TxHexNormalizer()
+
     tx_number = 666
     tx_hash = '0x' + os.urandom(32).hex()
     tx_signed = '0x' + os.urandom(128).hex()
@@ -115,10 +120,10 @@ def more_txs(
         tx_hash,
         list_defaults['block']+2,
         tx_number,
-        list_actors['alice'],
-        list_actors['diane'],
-        list_tokens['bar'],
-        list_tokens['bar'],
+        tx_normalize.wallet_address(list_actors['alice']),
+        tx_normalize.wallet_address(list_actors['diane']),
+        tx_normalize.executable_address(list_tokens['bar']),
+        tx_normalize.executable_address(list_tokens['bar']),
         2048,
         4096,
         False,

apps/cic-cache/tests/fixtures_config.py

@@ -14,7 +14,8 @@ logg = logging.getLogger(__file__)
 @pytest.fixture(scope='session')
 def load_config():
     config_dir = os.path.join(root_dir, 'config/test')
-    conf = confini.Config(config_dir, 'CICTEST')
+    schema_config_dir = os.path.join(root_dir, 'cic_cache', 'data', 'config')
+    conf = confini.Config(schema_config_dir, 'CICTEST', override_dirs=config_dir)
     conf.process()
     logg.debug('config {}'.format(conf))
     return conf

apps/cic-cache/tests/fixtures_database.py

@@ -24,11 +24,15 @@ def database_engine(
     if load_config.get('DATABASE_ENGINE') == 'sqlite':
         SessionBase.transactional = False
         SessionBase.poolable = False
+        name = 'cic_cache'
+        database_name = name
+        if load_config.get('DATABASE_PREFIX'):
+            database_name = '{}_{}'.format(load_config.get('DATABASE_PREFIX'), database_name)
         try:
-            os.unlink(load_config.get('DATABASE_NAME'))
+            os.unlink(database_name)
         except FileNotFoundError:
             pass
-    dsn = dsn_from_config(load_config)
+    dsn = dsn_from_config(load_config, name)
     SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
     return dsn
 

apps/cic-cache/tests/test_api.py

@@ -14,7 +14,7 @@ def test_api_all_data(
         ):
 
     env = {
-        'PATH_INFO': '/txa/410000/420000',
+        'PATH_INFO': '/txa/100/0/410000/420000',
         'HTTP_X_CIC_CACHE_MODE': 'all',
             }
     j = process_transactions_all_data(init_database, env)
@@ -23,7 +23,7 @@ def test_api_all_data(
     assert len(o['data']) == 2
 
     env = {
-        'PATH_INFO': '/txa/420000/410000',
+        'PATH_INFO': '/txa/100/0/420000/410000',
         'HTTP_X_CIC_CACHE_MODE': 'all',
             }
    

apps/cic-cache/tests/test_cache.py

@@ -6,6 +6,7 @@ import json
 
 # external imports
 import pytest
+from chainlib.encode import TxHexNormalizer
 
 # local imports
 from cic_cache import db
@@ -62,6 +63,8 @@ def test_cache_ranges(
 
     session = init_database
        
+    tx_normalize = TxHexNormalizer()
+
     oldest = list_defaults['block'] - 1
     mid = list_defaults['block']
     newest = list_defaults['block'] + 2
@@ -100,32 +103,39 @@ def test_cache_ranges(
     assert b[1] == mid
 
     # now check when supplying account
-    b = c.load_transactions_account(list_actors['alice'], 0, 100)
+    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
 
-    b = c.load_transactions_account(list_actors['bob'], 0, 100)
+    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account(account, 0, 100)
     assert b[0] == mid
     assert b[1] == mid
 
-    b = c.load_transactions_account(list_actors['diane'], 0, 100)
+    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
 
     # add block filter to the mix
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
     
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
 
-    b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == mid
 
-    b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == oldest
     assert b[1] == oldest
 
@@ -140,6 +150,8 @@ def test_cache_ranges_data(
 
     session = init_database
        
+    tx_normalize = TxHexNormalizer()
+
     oldest = list_defaults['block'] - 1
     mid = list_defaults['block']
     newest = list_defaults['block'] + 2
@@ -203,7 +215,8 @@ def test_cache_ranges_data(
     assert b[2][1]['tx_hash'] == more_txs[1]
 
     # now check when supplying account
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
+    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account_with_data(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
     assert len(b[2]) == 3
@@ -211,13 +224,15 @@ def test_cache_ranges_data(
     assert b[2][1]['tx_hash'] == more_txs[1]
     assert b[2][2]['tx_hash'] == more_txs[2]
 
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
+    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account_with_data(account, 0, 100)
     assert b[0] == mid
     assert b[1] == mid
     assert len(b[2]) == 1
     assert b[2][0]['tx_hash'] == more_txs[1]
 
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
+    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account_with_data(account, 0, 100)
     assert b[0] == oldest
     assert b[1] == newest
     assert len(b[2]) == 2
@@ -225,27 +240,31 @@ def test_cache_ranges_data(
     assert b[2][1]['tx_hash'] == more_txs[2]
 
     # add block filter to the mix
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
     assert len(b[2]) == 2
     assert b[2][0]['tx_hash'] == more_txs[0]
     assert b[2][1]['tx_hash'] == more_txs[1]
 
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['alice'])
+    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == newest
     assert len(b[2]) == 2
     assert b[2][0]['tx_hash'] == more_txs[0]
     assert b[2][1]['tx_hash'] == more_txs[1]
 
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['bob'])
+    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == mid
     assert b[1] == mid
     assert len(b[2]) == 1
     assert b[2][0]['tx_hash'] == more_txs[1]
 
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    account = tx_normalize.wallet_address(list_actors['diane'])
+    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
     assert b[0] == oldest
     assert b[1] == oldest
     assert len(b[2]) == 1

apps/cic-cache/tests/test_query.py

@@ -82,7 +82,7 @@ def test_query_regex(
         [ 
             ('alice', None, None, [(420000, 13), (419999, 42)]),
             ('alice', None, 1, [(420000, 13)]),
-            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
+            ('alice', 1, 1, [(419999, 42)]), # 420000 == list_defaults['block']
             ('alice', 2, None, []), # 420000 == list_defaults['block']
             ],
         )
@@ -107,10 +107,11 @@ def test_query_process_txs_account(
     path_info = '/tx/user/0x' + strip_0x(actor)
     if query_offset != None:
         path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
+    if query_limit == None:
+        query_limit = 100
+    path_info += '/' + str(query_limit)
+    if query_offset == None:
+        path_info += '/0'
     env = {
             'PATH_INFO': path_info,
             }
@@ -192,7 +193,7 @@ def test_query_process_txs_bloom(
 @pytest.mark.parametrize(
         'query_block_start, query_block_end, query_match_count',
         [ 
-            (None, 42, 0),
+            (1, 42, 0),
             (420000, 420001, 1),
             (419999, 419999, 1), # matches are inclusive
             (419999, 420000, 2),
@@ -211,7 +212,7 @@ def test_query_process_txs_data(
         query_match_count,
         ):
 
-    path_info = '/txa'
+    path_info = '/txa/100/0'
     if query_block_start != None:
         path_info += '/' + str(query_block_start)
     if query_block_end != None:
@@ -227,4 +228,5 @@ def test_query_process_txs_data(
     assert r != None
 
     o = json.loads(r[1])
+    logg.debug('oo {}'.format(o))
     assert len(o['data']) == query_match_count

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,5 +1,5 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.3a1
-cic-eth-registry~=0.5.8a1
-chainlib~=0.0.7a1
-cic_eth~=0.12.2a4
+erc20-demurrage-token~=0.0.6
+cic-eth-registry~=0.6.3
+chainlib~=0.0.14
+cic_eth~=0.12.6

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.2a6
+version = 0.0.3
 description = cic-eth tasks supporting erc20 demurrage token
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth/.gitlab-ci.yml

@@ -1,53 +1,16 @@
-.cic_eth_variables:
-    variables:
-        APP_NAME: cic-eth
-
-        #build-mr-cic-eth:
-        #    extends:
-        #        - .cic_eth_variables
-        #        - .py_build_target_dev
-        #    rules:
-        #    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-        #      changes:
-        #          - apps/cic-eth/**/*
-        #      when: always
-
-test-mr-cic-eth:
-    stage: test
-    tags:
-      - integration
-    extends:
-        - .cic_eth_variables
-    cache:
-      key:
-        files:
-        - test_requirements.txt
-      paths:
-      - /root/.cache/pip
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/$APP_NAME:$MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r admin_requirements.txt
-          -r services_requirements.txt
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    allow_failure: true
-    needs: ["build-merge-request"]
-    rules:
+build-test-cic-eth:
+  stage: test
+  tags:
+    - integration
+  variables:
+    APP_NAME: cic-eth
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+  script:
+    - cd apps/cic-eth
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+  rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
           - apps/$APP_NAME/**/*
       when: always
-
-      #build-push-cic-eth:
-      #    extends:
-      #        - .py_build_push
-      #        - .cic_eth_variables
-      #    rules:
-      #    - if:  $CI_COMMIT_BRANCH == "master"
-      #      changes:
-      #          - apps/cic-eth/**/*
-      #      when: always

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,4 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.5.6a2,<0.6.0
-hexathon~=0.0.1a7
-chainqueue>=0.0.3a1,<0.1.0
-eth-erc20>=0.0.10a3,<0.1.0
+hexathon~=0.1.0
+chainqueue~=0.0.6a4
+eth-erc20~=0.1.5

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -4,7 +4,6 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from hexathon import (
         add_0x,
@@ -20,18 +19,17 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         )
 from cic_eth.error import LockedError
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-def normalize_address(a):
-    if a == None:
-        return None
-    return add_0x(hex_uniform(strip_0x(a)))
-
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -43,7 +41,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -53,7 +51,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -65,7 +63,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -75,7 +73,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -85,7 +83,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -93,7 +91,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -103,7 +101,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -111,7 +109,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -121,7 +119,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -129,7 +127,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -139,7 +137,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -148,12 +146,13 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    address = normalize_address(address)
+    if address != None:
+        address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
     if address != None:
         r |= Lock.check(chain_str, lock_flags, address=address, session=session)
     if r > 0:

apps/cic-eth/cic_eth/admin/nonce.py

@@ -33,6 +33,7 @@ from cic_eth.admin.ctrl import (
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
 from cic_eth.task import BaseTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -73,7 +74,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
+    query_address = tx_normalize.wallet_address(address)
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==query_address)

apps/cic-eth/cic_eth/admin/token.py

@@ -1,21 +1,2 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
 # local imports
-from cic_eth.task import BaseTask
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-            'symbol': self.default_token_symbol,
-            'address': self.default_token_address,
-            'name': self.default_token_name,
-            'decimals': self.default_token_decimals,
-            }
+from cic_eth.eth.erc20 import default_token

apps/cic-eth/cic_eth/api/admin.py

@@ -32,7 +32,6 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
 from eth_erc20 import ERC20
 
 # local imports
@@ -40,6 +39,7 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
+from cic_eth.queue.query import get_tx_local
 
 app = celery.current_app
 
@@ -123,7 +123,7 @@ class AdminApi:
         return s_lock.apply_async()
 
 
-    def tag_account(self, tag, address_hex, chain_spec):
+    def tag_account(self, chain_spec, tag, address):
         """Persistently associate an address with a plaintext tag.
 
         Some tags are known by the system and is used to resolve addresses to use for certain transactions. 
@@ -138,7 +138,7 @@ class AdminApi:
             'cic_eth.eth.account.set_role',
             [
                 tag,
-                address_hex,
+                address,
                 chain_spec.asdict(),
                 ],
             queue=self.queue,
@@ -146,6 +146,30 @@ class AdminApi:
         return s_tag.apply_async()
 
 
+    def get_tag_account(self, chain_spec, tag=None, address=None):
+        if address != None:
+            s_tag = celery.signature(
+                'cic_eth.eth.account.role',
+                [
+                    address,
+                    chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+
+        else:
+            s_tag = celery.signature(
+                'cic_eth.eth.account.role_account',
+                [
+                    tag,
+                    chain_spec.asdict(),
+                ],
+                queue=self.queue,
+                )
+
+        return s_tag.apply_async()
+
+
     def have_account(self, address_hex, chain_spec):
         s_have = celery.signature(
             'cic_eth.eth.account.have',
@@ -284,7 +308,7 @@ class AdminApi:
         tx_hash_hex = None
         session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
+            tx_dict = get_tx_local(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
         session.close()
@@ -503,7 +527,7 @@ class AdminApi:
                     queue=self.queue,
                     )
                 t = s.apply_async()
-                role = t.get()
+                role = t.get()[0][1]
                 if role != None:
                     tx['sender_description'] = role
 
@@ -556,7 +580,7 @@ class AdminApi:
                     queue=self.queue,
                     )
                 t = s.apply_async()
-                role = t.get()
+                role = t.get()[0][1]
                 if role != None:
                     tx['recipient_description'] = role
 

apps/cic-eth/cic_eth/api/api_task.py

@@ -9,6 +9,7 @@ import logging
 # external imports 
 import celery
 from chainlib.chain import ChainSpec
+from hexathon import strip_0x
 
 # local imports
 from cic_eth.api.base import ApiBase
@@ -16,15 +17,50 @@ from cic_eth.enum import LockEnum
 
 app = celery.current_app
 
-logg = logging.getLogger(__name__)
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 class Api(ApiBase):
-    
+
+    @staticmethod
+    def to_v_list(v, n):
+        """Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
+
+        :param v: Arguments
+        :type v: str or list
+        :param n: Number of elements to generate arguments for
+        :type n: int
+        :rtype: list
+        :returns: list of assembled arguments
+        """
+        if isinstance(v, str):
+            vv = v
+            v = []
+            for i in range(n):
+                v.append([vv])
+        elif not isinstance(v, list):
+            raise ValueError('argument must be single string, or list or strings or lists')
+        else:
+            if len(v) != n:
+                raise ValueError('v argument count must match integer n')
+            for i in range(n):
+                if isinstance(v[i], str):
+                    v[i] = [v[i]]
+                elif not isinstance(v, list):
+                    raise ValueError('proof argument must be single string, or list or strings or lists')
+
+        return v
+   
 
     def default_token(self):
+        """Retrieves the default fallback token of the custodial network.
+
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
         s_token = celery.signature(
-                'cic_eth.admin.token.default_token',
+                'cic_eth.eth.erc20.default_token',
                 [],
                 queue=self.queue,
                 )
@@ -34,6 +70,97 @@ class Api(ApiBase):
         return s_token.apply_async()
 
 
+    def token(self, token_symbol, proof=None):
+        """Single-token alias for tokens method.
+
+        See tokens method for details.
+
+        :param token_symbol: Token symbol to look up
+        :type token_symbol: str
+        :param proof: Proofs to add to signature verification for the token
+        :type proof: str or list
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        if not isinstance(token_symbol, str):
+            raise ValueError('token symbol must be string')
+
+        return self.tokens([token_symbol], proof=proof)
+
+
+    def tokens(self, token_symbols, proof=None):
+        """Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
+
+        Token symbols are always strings, and should be specified using uppercase letters.
+
+        If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
+
+        The proof argument may be specified in a number of ways:
+
+        - as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
+        - as a single string, where the same proof is used for each token lookup
+        - as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
+        - as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
+
+        The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
+
+        This method is not intended to be used synchronously. Do so at your peril.
+
+        :param token_symbols: Token symbol strings to look up
+        :type token_symbol: list
+        :param proof: Proof(s) to verify tokens against
+        :type proof: None, str or list
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        if not isinstance(token_symbols, list):
+            raise ValueError('token symbols argument must be list')
+
+        if proof == None:
+            logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
+            proof = ''
+
+        logg.debug('proof is {}'.format(proof))
+        l = len(token_symbols)
+        if len(proof) == 0:
+            l = 0 
+        proof = Api.to_v_list(proof, l)
+
+        chain_spec_dict = self.chain_spec.asdict()
+
+        s_token_resolve = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    token_symbols,
+                    chain_spec_dict,
+                    ],
+                queue=self.queue,
+                )
+
+        s_token_info = celery.signature(
+                'cic_eth.eth.erc20.token_info',
+                [
+                    chain_spec_dict,
+                    proof,
+                    ],
+                queue=self.queue,
+                )
+
+        s_token_verify = celery.signature(
+                    'cic_eth.eth.erc20.verify_token_info',
+                    [
+                        chain_spec_dict,
+                        self.callback_success,
+                        self.callback_error,
+                        ],
+                    queue=self.queue,
+                    )
+
+        s_token_info.link(s_token_verify)
+        s_token_resolve.link(s_token_info)
+        return s_token_resolve.apply_async()
+
+
 #    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
 #        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 #
@@ -254,6 +381,8 @@ class Api(ApiBase):
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        #from_address = strip_0x(from_address)
+        #to_address = strip_0x(to_address)
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -386,7 +515,7 @@ class Api(ApiBase):
         :param password: Password to encode the password with in the backend (careful, you will have to remember it)
         :type password: str
         :param register: Register the new account in accounts index backend
-        :type password: bool
+        :type register: bool
         :returns: uuid of root task
         :rtype: celery.Task
         """
@@ -554,3 +683,4 @@ class Api(ApiBase):
 
         t = self.callback_success.apply_async([r])
         return t
+

apps/cic-eth/cic_eth/callbacks/noop.py

@@ -1,7 +1,10 @@
+import logging
+
 import celery
 
 celery_app = celery.current_app
-logg = celery_app.log.get_default_logger()
+#logg = celery_app.log.get_default_logger()
+logg = logging.getLogger()
 
 
 @celery_app.task(bind=True)

apps/cic-eth/cic_eth/check/gas.py

@@ -12,8 +12,9 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import LockedError
 from cic_eth.admin.ctrl import check_lock
+from cic_eth.eth.gas import have_gas_minimum
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 
 def health(*args, **kwargs):
@@ -31,18 +32,15 @@ def health(*args, **kwargs):
         return True
 
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
+    min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
+    if config.get('ETH_MIN_FEE_PRICE'):
+        min_gas *= int(config.get('ETH_MIN_FEE_PRICE'))
+
+    r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session)
+
     session.close()
+    
+    if not r:
+        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas))
 
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    o = balance(gas_provider)
-    r = rpc.do(o)
-    try: 
-        r = int(r, 16)
-    except TypeError:
-        r = int(r)
-    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
-    if r < gas_min:
-        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
-        return False
-
-    return True
+    return r

apps/cic-eth/cic_eth/check/start.py

@@ -0,0 +1,18 @@
+# external imports
+from chainlib.chain import ChainSpec
+
+# local imports
+from cic_eth.admin.ctrl import check_lock
+from cic_eth.enum import LockEnum
+from cic_eth.error import LockedError
+
+
+def health(*args, **kwargs):
+    config = kwargs['config']
+    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+    try:
+        check_lock(None, chain_spec.asdict(), LockEnum.START)
+    except LockedError as e:
+        return False
+    return True

apps/cic-eth/cic_eth/cli/arg.py

@@ -16,16 +16,22 @@ class ArgumentParser(BaseArgumentParser):
             self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
             self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
         if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+            self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
+            self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
             self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
         if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
+            self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
+            self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
+            self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
+            self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
+            self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
+            self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
+            self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
+            self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
             self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--offset', type=int, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
         if local_arg_flags & CICFlag.CHAIN:
             self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
-
-
-

apps/cic-eth/cic_eth/cli/base.py

@@ -24,8 +24,8 @@ class CICFlag(enum.IntEnum):
     # sync - nibble 4
     SYNCER = 4096
 
-
-argflag_local_task = CICFlag.CELERY 
+argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
+argflag_local_task = CICFlag.CELERY
 argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
 argflag_local_chain = CICFlag.CHAIN
 argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-eth/cic_eth/cli/config.py

@@ -1,12 +1,18 @@
 # standard imports
 import os
 import logging
+import urllib.parse
+import copy
 
 # external imports
 from chainlib.eth.cli import (
         Config as BaseConfig,
         Flag,
         )
+from urlybird.merge import (
+        urlhostmerge,
+        urlmerge,
+        )
 
 # local imports
 from .base import CICFlag
@@ -40,6 +46,7 @@ class Config(BaseConfig):
         if local_arg_flags & CICFlag.CHAIN:
             local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
 
+        
         if local_arg_flags & CICFlag.CELERY:
             local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
 
@@ -49,15 +56,71 @@ class Config(BaseConfig):
 
         config.dict_override(local_args_override, 'local cli args')
 
-        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-
+        local_celery_args_override = {}
         if local_arg_flags & CICFlag.CELERY:
+            hostport = urlhostmerge(
+                    None,
+                    config.get('REDIS_HOST'),
+                    config.get('REDIS_PORT'),
+                    )
+            db = getattr(args, 'redis_db', None)
+            if db != None:
+                db = str(db)
+
+            redis_url = (
+                    'redis',
+                    hostport,
+                    db,
+                    )
+
+
+            celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
+            hostport = urlhostmerge(
+                    celery_config_url[1],
+                    getattr(args, 'celery_host', None),
+                    getattr(args, 'celery_port', None),
+                    )
+            db = getattr(args, 'redis_db', None)
+            if db != None:
+                db = str(db)
+            celery_arg_url = (
+                    getattr(args, 'celery_scheme', None),
+                    hostport,
+                    db,
+                    )
+
+            celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
+            celery_url_string = urllib.parse.urlunsplit(celery_url)
+            local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
+            if not getattr(args, 'celery_no_result'):
+                local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
+                if local_celery_args_override['CELERY_RESULT_URL'] == None:
+                    local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
+                celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
+                hostport = urlhostmerge(
+                        celery_config_url[1],
+                        getattr(args, 'celery_result_host', None),
+                        getattr(args, 'celery_result_port', None),
+                        )
+                celery_arg_url = (
+                        getattr(args, 'celery_result_scheme', None),
+                        hostport,
+                        getattr(args, 'celery_result_db', None),
+                        )
+                celery_url = urlmerge(celery_config_url, celery_arg_url)
+                logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
+                celery_url_string = urllib.parse.urlunsplit(celery_url)
+                local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
             config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
 
+        config.dict_override(local_celery_args_override, 'local celery cli args')
+
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
+            redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
+            config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
+            config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
+
         logg.debug('config loaded:\n{}'.format(config))
 
         return config
-
-

apps/cic-eth/cic_eth/cli/rpc.py

@@ -35,14 +35,14 @@ class RPC:
     def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label)
+        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
         if use_signer:
 
             RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
             RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
             RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
             RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
-        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc
 

apps/cic-eth/cic_eth/data/config/celery.ini

@@ -1,5 +1,5 @@
 [celery]
-broker_url = redis://localhost:6379
+broker_url = 
 result_url = 
 queue = cic-eth
 debug = 0

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -2,5 +2,5 @@
 registry_address = 
 trust_address =
 default_token_symbol =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
+health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
 run_dir = /run

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,2 +1,6 @@
 [eth]
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_holder_minimum_units = 180000
+gas_holder_refill_units = 15
+gas_holder_refill_threshold = 3
+gas_gifter_refill_buffer = 3
+min_fee_price = 1

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -8,8 +8,8 @@ Create Date: 2021-04-02 18:41:20.864265
 import datetime
 from alembic import op
 import sqlalchemy as sa
-from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth.db.enum import LockEnum
+from cic_eth.encode import ZERO_ADDRESS_NORMAL
 
 
 # revision identifiers, used by Alembic.
@@ -23,14 +23,14 @@ def upgrade():
     op.create_table(
             'lock',
             sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column("address", sa.String(42), nullable=True),
+            sa.Column("address", sa.String, nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
             sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
             sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
             )
     op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
-    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
+    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS_NORMAL, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
 
 
 def downgrade():

apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py

@@ -0,0 +1,31 @@
+"""Add gas cache
+
+Revision ID: c91cafc3e0c1
+Revises: aee12aeb47ec
+Create Date: 2021-10-28 20:45:34.239865
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c91cafc3e0c1'
+down_revision = 'aee12aeb47ec'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'gas_cache',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column("address", sa.String, nullable=False),
+            sa.Column("tx_hash", sa.String, nullable=True),
+            sa.Column("method", sa.String, nullable=True),
+            sa.Column("value", sa.BIGINT(), nullable=False),
+            )
+
+
+def downgrade():
+    op.drop_table('gas_cache')

apps/cic-eth/cic_eth/db/models/gas_cache.py

@@ -0,0 +1,27 @@
+# standard imports
+import logging
+
+# external imports
+from sqlalchemy import Column, String, NUMERIC
+
+# local imports
+from .base import SessionBase
+
+logg = logging.getLogger(__name__)
+
+
+class GasCache(SessionBase):
+    """Provides gas budget cache for token operations
+    """
+    __tablename__ = 'gas_cache'
+
+    address = Column(String())
+    tx_hash = Column(String())
+    method = Column(String())
+    value = Column(NUMERIC())
+
+    def __init__(self, address, method, value, tx_hash):
+        self.address = address
+        self.tx_hash = tx_hash
+        self.method = method
+        self.value = value

apps/cic-eth/cic_eth/db/models/lock.py

@@ -4,12 +4,12 @@ import logging
 
 # third-party imports
 from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode  import ZERO_ADDRESS_NORMAL
 
 logg = logging.getLogger()
 
@@ -37,7 +37,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
+    def set(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None, tx_hash=None):
         """Sets flags associated with the given address and chain.
 
         If a flags entry does not exist it is created.
@@ -90,7 +90,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def reset(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Resets flags associated with the given address and chain.
 
         If the resulting flags entry value is 0, the entry will be deleted.
@@ -134,7 +134,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def check(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Checks whether all given flags are set for given address and chain. 
 
         Does not validate the address against any other tables or components.

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -12,7 +12,7 @@ from cic_eth.error import (
         IntegrityError,
         )
 
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)
 
 
 class Nonce(SessionBase):
@@ -21,7 +21,7 @@ class Nonce(SessionBase):
     __tablename__ = 'nonce'
 
     nonce = Column(Integer)
-    address_hex = Column(String(42))
+    address_hex = Column(String(40))
 
 
     @staticmethod

apps/cic-eth/cic_eth/db/models/role.py

@@ -24,8 +24,22 @@ class AccountRole(SessionBase):
     tag = Column(Text)
     address_hex = Column(String(42))
 
-    
-    # TODO: 
+
+    @staticmethod
+    def all(session=None):
+        session = SessionBase.bind_session(session)
+
+        pairs = []
+
+        q = session.query(AccountRole.tag, AccountRole.address_hex)
+        for r in q.all():
+            pairs.append((r[1], r[0]),)
+
+        SessionBase.release_session(session)
+        
+        return pairs
+
+
     @staticmethod
     def get_address(tag, session):
         """Get Ethereum address matching the given tag

apps/cic-eth/cic_eth/encode.py

@@ -0,0 +1,16 @@
+# external imports
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainqueue.encode import TxHexNormalizer
+from chainlib.eth.tx import unpack
+
+tx_normalize = TxHexNormalizer()
+
+ZERO_ADDRESS_NORMAL = tx_normalize.wallet_address(ZERO_ADDRESS)
+
+
+def unpack_normal(signed_tx_bytes, chain_spec):
+    tx = unpack(signed_tx_bytes, chain_spec)
+    tx['hash'] = tx_normalize.tx_hash(tx['hash'])
+    tx['from'] = tx_normalize.wallet_address(tx['from'])
+    tx['to'] = tx_normalize.wallet_address(tx['to'])
+    return tx

apps/cic-eth/cic_eth/enum.py

@@ -69,9 +69,12 @@ class StatusEnum(enum.IntEnum):
 class LockEnum(enum.IntEnum):
     """
     STICKY: When set, reset is not possible
+    INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance)
+    START: When set, startup is not possible, regardless of state
     CREATE: Disable creation of accounts
     SEND: Disable sending to network
     QUEUE: Disable queueing new or modified transactions
+    QUERY: Disable all queue state and transaction queries
     """
     STICKY=1
     INIT=2
@@ -79,7 +82,8 @@ class LockEnum(enum.IntEnum):
     SEND=8
     QUEUE=16
     QUERY=32
-    ALL=int(0xfffffffffffffffe)
+    START=int(0x80000000)
+    ALL=int(0x7ffffffe)
 
 
 def status_str(v, bits_only=False):

apps/cic-eth/cic_eth/error.py

@@ -48,8 +48,6 @@ class RoleMissingError(Exception):
     pass
 
 
-
-
 class IntegrityError(Exception):
     """Exception raised to signal irregularities with deduplication and ordering of tasks
 
@@ -66,8 +64,10 @@ class LockedError(Exception):
 
 class SeppukuError(Exception):
     """Exception base class for all errors that should cause system shutdown
-
     """
+    def __init__(self, message, lockdown=False):
+        self.message = message
+        self.lockdown = lockdown
 
 
 class SignerError(SeppukuError):
@@ -85,3 +85,8 @@ class RoleAgencyError(SeppukuError):
 class YouAreBrokeError(Exception):
     """Exception raised when a value transfer is attempted without access to sufficient funds
     """
+
+
+class TrustError(Exception):
+    """Exception raised when required trust proofs are missing for a request
+    """

apps/cic-eth/cic_eth/eth/account.py

@@ -13,11 +13,8 @@ from chainlib.eth.sign import (
         new_account,
         sign_message,
         )
-from chainlib.eth.address import to_checksum_address
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        )
+from chainlib.eth.address import to_checksum_address, is_address
+from chainlib.eth.tx import TxFormat
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
@@ -34,6 +31,7 @@ from cic_eth.eth.gas import (
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
+from cic_eth.encode import tx_normalize
 from cic_eth.error import (
         RoleMissingError,
         SignerError,
@@ -49,6 +47,11 @@ from cic_eth.eth.nonce import (
 from cic_eth.queue.tx import (
         register_tx,
         )
+from cic_eth.encode import (
+        unpack_normal,
+        ZERO_ADDRESS_NORMAL,
+        tx_normalize,
+        )
 
 logg = logging.getLogger()
 celery_app = celery.current_app 
@@ -83,7 +86,7 @@ def create(self, password, chain_spec_dict):
     # TODO: It seems infeasible that a can be None in any case, verify
     if a == None:
         raise SignerError('create account')
-        
+    a = tx_normalize.wallet_address(a)
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -133,7 +136,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
+    gas_oracle = self.create_gas_oracle(rpc, code_callback=AccountRegistry.gas)
     account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -174,6 +177,9 @@ def gift(self, account_address, chain_spec_dict):
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
+    if is_address(account_address):
+        account_address = tx_normalize.wallet_address(account_address)
+
     logg.debug('gift account address {} to index'.format(account_address))
     queue = self.request.delivery_info.get('routing_key')
 
@@ -186,7 +192,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, code_callback=MinterFaucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -247,8 +253,9 @@ def have(self, account, chain_spec_dict):
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):
-    if not to_checksum_address(address):
-        raise ValueError('invalid checksum address {}'.format(address))
+    if not is_address(address):
+        raise ValueError('invalid address {}'.format(address))
+    address = tx_normalize.wallet_address(address)
     session = SessionBase.create_session()
     role = AccountRole.set(tag, address, session=session) 
     session.add(role)
@@ -259,19 +266,46 @@ def set_role(self, tag, address, chain_spec_dict):
 
 @celery_app.task(bind=True, base=BaseTask)
 def role(self, address, chain_spec_dict):
-    """Return account role for address
+    """Return account role for address and/or role
 
     :param account: Account to check
     :type account: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
+    :param chain_spec_dict: Chain spec dict representation
+    :type chain_spec_dict: dict
     :returns: Account, or None if not exists
     :rtype: Varies
     """
     session = self.create_session()
     role_tag =  AccountRole.role_for(address, session=session)
     session.close()
-    return role_tag
+    return [(address, role_tag,)]
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def role_account(self, role_tag, chain_spec_dict):
+    """Return address for role.
+
+    If the role parameter is None, will return addresses for all roles.
+
+    :param role_tag: Role to match
+    :type role_tag: str
+    :param chain_spec_dict: Chain spec dict representation
+    :type chain_spec_dict: dict
+    :returns: List with a single account/tag pair for a single tag, or a list of account and tag pairs for all tags
+    :rtype: list
+    """
+    session = self.create_session()
+
+    pairs = None
+    if role_tag != None:
+        addr = AccountRole.get_address(role_tag, session=session)
+        pairs = [(addr, role_tag,)]
+    else:
+        pairs = AccountRole.all(session=session)
+
+    session.close()
+
+    return pairs
 
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
@@ -295,17 +329,19 @@ def cache_gift_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = Faucet.parse_give_to_request(tx['data'])
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx['to'])
 
     session = self.create_session()
 
     tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
+            'hash': tx['hash'],
+            'from': sender_address,
+            'to': recipient_address,
+            'source_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
             'to_value': 0,
             }
@@ -334,17 +370,19 @@ def cache_account_data(
     :rtype: tuple
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = AccountsIndex.parse_add_request(tx['data'])
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx['to'])
 
     session = SessionBase.create_session()
     tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
+            'hash': tx['hash'],
+            'from': sender_address,
+            'to': recipient_address,
+            'source_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
             'to_value': 0,
             }

apps/cic-eth/cic_eth/eth/erc20.py

@@ -10,12 +10,19 @@ from chainlib.eth.tx import (
         TxFormat,
         unpack,
         )
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        )
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
 from chainqueue.sql.tx import cache_tx_dict
+from okota.token_index.index import to_identifier
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -27,24 +34,26 @@ from cic_eth.error import (
         YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
-from cic_eth.eth.gas import (
-        create_check_gas_task,
-        MaxGasOracle,
-        )
+from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.eth.util import CacheGasOracle
 from cic_eth.ext.address import translate_address
 from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
         CriticalSQLAlchemyAndSignerTask,
+        BaseTask,
     )
 from cic_eth.eth.nonce import CustodialTaskNonceOracle
+from cic_eth.encode import tx_normalize
+from cic_eth.eth.trust import verify_proofs
+from cic_eth.error import SignerError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(base=CriticalWeb3Task)
-def balance(tokens, holder_address, chain_spec_dict):
+@celery_app.task(bind=True, base=CriticalWeb3Task)
+def balance(self, tokens, holder_address, chain_spec_dict):
     """Return token balances for a list of tokens for given address
 
     :param tokens: Token addresses
@@ -62,8 +71,10 @@ def balance(tokens, holder_address, chain_spec_dict):
 
     for t in tokens:
         address = t['address']
-        token = ERC20Token(chain_spec, rpc, address)
-        c = ERC20(chain_spec)
+        logg.debug('address {} {}'.format(address, holder_address))
+        gas_oracle = self.create_gas_oracle(rpc, min_price=self.min_fee_price)
+        token = ERC20Token(chain_spec, rpc, add_0x(address))
+        c = ERC20(chain_spec, gas_oracle=gas_oracle)
         o = c.balance_of(address, holder_address, sender_address=caller_address)
         r = rpc.do(o)
         t['balance_network'] = c.parse_balance(r)
@@ -146,8 +157,12 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
+
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    enc = ABIContractEncoder()
+    enc.method('transferFrom')
+    method = enc.get()
+    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -217,8 +232,12 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
+
+    enc = ABIContractEncoder()
+    enc.method('transfer')
+    method = enc.get()
+    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -286,8 +305,12 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
+
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    enc = ABIContractEncoder()
+    enc.method('approve')
+    method = enc.get()
+    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -371,16 +394,20 @@ def cache_transfer_data(
     tx = unpack(tx_signed_raw_bytes, chain_spec)
 
     tx_data = ERC20.parse_transfer_request(tx['data'])
-    recipient_address = tx_data[0]
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
+    source_token_address = tx_normalize.executable_address(tx['to'])
+    destination_token_address = source_token_address
+
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
-            'from': tx['from'],
+            'from': sender_address,
             'to': recipient_address,
-            'source_token': tx['to'],
-            'destination_token': tx['to'],
+            'source_token': source_token_address,
+            'destination_token': destination_token_address,
             'from_value': token_value,
             'to_value': token_value,
             }
@@ -412,14 +439,16 @@ def cache_transfer_from_data(
     spender_address = tx_data[0]
     recipient_address = tx_data[1]
     token_value = tx_data[2]
+    source_token_address = tx_normalize.executable_address(tx['to'])
+    destination_token_address = source_token_address
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
             'from': tx['from'],
             'to': recipient_address,
-            'source_token': tx['to'],
-            'destination_token': tx['to'],
+            'source_token': source_token_address,
+            'destination_token': destination_token_address,
             'from_value': token_value,
             'to_value': token_value,
             }
@@ -448,16 +477,19 @@ def cache_approve_data(
     tx = unpack(tx_signed_raw_bytes, chain_spec)
 
     tx_data = ERC20.parse_approve_request(tx['data'])
-    recipient_address = tx_data[0]
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
+    source_token_address = tx_normalize.executable_address(tx['to'])
+    destination_token_address = source_token_address
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
-            'from': tx['from'],
+            'from': sender_address,
             'to': recipient_address,
-            'source_token': tx['to'],
-            'destination_token': tx['to'],
+            'source_token': source_token_address,
+            'destination_token': destination_token_address,
             'from_value': token_value,
             'to_value': token_value,
             }
@@ -465,3 +497,69 @@ def cache_approve_data(
     session.close()
     return (tx_hash_hex, cache_id)
 
+
+@celery_app.task(bind=True, base=BaseTask)
+def token_info(self, tokens, chain_spec_dict, proofs=[]):
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    i = 0
+
+    for token in tokens:
+        result_data = []
+        token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
+        token_chain_object.load(rpc)
+
+        token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
+        token_proofs = [token_symbol_proof_hex]
+        if len(proofs) > 0:
+            token_proofs += proofs[i]
+
+        tokens[i] = {
+            'decimals': token_chain_object.decimals,
+            'name': token_chain_object.name,
+            'symbol': token_chain_object.symbol,
+            'address': tx_normalize.executable_address(token_chain_object.address),
+            'proofs': token_proofs,
+            'converters': tokens[i]['converters'],
+                }   
+        i += 1
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
+    queue = self.request.delivery_info.get('routing_key')
+
+    for token in tokens:
+        s = celery.signature(
+                'cic_eth.eth.trust.verify_proofs',
+                [
+                    token,
+                    token['address'],
+                    token['proofs'],
+                    chain_spec_dict,
+                    success_callback,
+                    error_callback,
+                    ],
+                queue=queue,
+                )
+
+        if success_callback != None:
+            s.link(success_callback)
+        if error_callback != None:
+            s.on_error(error_callback)
+        s.apply_async()
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def default_token(self):
+    return {
+        'symbol': self.default_token_symbol,
+        'address': self.default_token_address,
+        'name': self.default_token_name,
+        'decimals': self.default_token_decimals,
+        }

apps/cic-eth/cic_eth/eth/gas.py

@@ -3,10 +3,17 @@ import logging
 
 # external imports
 import celery
-from hexathon import strip_0x
-from chainlib.eth.constant import ZERO_ADDRESS
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+#from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
-from chainlib.eth.address import is_checksum_address
+from chainlib.eth.address import (
+        is_checksum_address,
+        to_checksum_address,
+        is_address
+        )
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
 from chainqueue.sql.tx import cache_tx_dict
@@ -21,7 +28,6 @@ from chainlib.eth.error import (
 from chainlib.eth.tx import (
         TxFactory,
         TxFormat,
-        unpack,
         )
 from chainlib.eth.contract import (
         abi_decode_single,
@@ -35,6 +41,7 @@ from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
+from cic_eth.db.models.gas_cache import GasCache
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
 from cic_eth.error import (
@@ -45,6 +52,7 @@ from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.queue.tx import (
         queue_create,
         register_tx,
+        unpack,
         )
 from cic_eth.queue.query import get_tx
 from cic_eth.task import (
@@ -53,17 +61,61 @@ from cic_eth.task import (
         CriticalSQLAlchemyAndSignerTask,
         CriticalWeb3AndSignerTask,
         )
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        unpack_normal,
+        )
+from cic_eth.error import SeppukuError
+from cic_eth.eth.util import MAXIMUM_FEE_UNITS
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-MAXIMUM_FEE_UNITS = 8000000
 
-class MaxGasOracle:
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def apply_gas_value_cache(address, method, value, tx_hash):
+    return apply_gas_value_cache_local(address, method, value, tx_hash)
 
-    def gas(code=None):
-        return MAXIMUM_FEE_UNITS
+
+def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
+    address = tx_normalize.executable_address(address)
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    value = int(value)
+
+    session = SessionBase.bind_session(session)
+    q = session.query(GasCache)
+    q = q.filter(GasCache.address==address)
+    q = q.filter(GasCache.method==method)
+    o = q.first()
+
+    if o == None:
+        o = GasCache(address, method, value, tx_hash)
+    elif value > o.value:
+        o.value = value
+        o.tx_hash = strip_0x(tx_hash)
+
+    session.add(o)
+    session.commit()
+
+    SessionBase.release_session(session)
+
+
+def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):
+    if rpc == None:
+        rpc = RPCConnection.connect(chain_spec, 'default')
+    o = balance(add_0x(address))
+    r = rpc.do(o)
+    try: 
+        r = int(r)
+    except ValueError:
+        r = strip_0x(r)
+        r = int(r, 16)
+    logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas))
+    if r < min_gas:
+        return False
+    return True
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -130,16 +182,16 @@ def cache_gas_data(
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
 
     session = SessionBase.create_session()
 
     tx_dict = {
-            'hash': tx_hash_hex,
+            'hash': tx['hash'],
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
+            'source_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': tx['value'],
             'to_value': tx['value'],
             }
@@ -150,7 +202,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -170,8 +222,23 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
+    rpc_format_address = None
+    if address != None:
+        if not is_address(address):
+            raise ValueError('invalid address {}'.format(address))
+        address = tx_normalize.wallet_address(address)
+        address = add_0x(address)
+
+    tx_hashes = []
+    txs = []
+    for tx_hash in tx_hashes_hex:
+        tx_hash = tx_normalize.tx_hash(tx_hash)
+        tx_hashes.append(tx_hash)
+    for tx in txs_hex:
+        tx = tx_normalize.tx_wire(tx)
+        txs.append(tx)
+
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
 
     addresspass = None
     if len(txs) == 0:
@@ -187,8 +254,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
                 raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
             addresspass.append(address)
 
-    if not is_checksum_address(address):
-        raise ValueError('invalid address {}'.format(address))
+    rpc_format_address = add_0x(to_checksum_address(address))
 
     queue = self.request.delivery_info.get('routing_key')
 
@@ -196,7 +262,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
 
     gas_balance = 0
     try:
-        o = balance(address)
+        o = balance(rpc_format_address)
         r = conn.do(o)
         conn.disconnect()
         gas_balance = abi_decode_single(ABIContractType.UINT256, r)
@@ -304,6 +370,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # Determine value of gas tokens to send
     # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
     # however, we will perform a 0-value transaction instead
+    recipient_address = tx_normalize.wallet_address(recipient_address)
     zero_amount = False
     session = SessionBase.create_session()
     status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
@@ -330,6 +397,13 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # set up evm RPC connection
     rpc = RPCConnection.connect(chain_spec, 'default')
 
+    # check the gas balance of the gifter
+    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount):
+        raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider))
+
+    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance):
+        logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider))
+    
     # set up transaction builder
     nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc)
@@ -378,6 +452,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     :returns: Transaction hash
     :rtype: str, 0x-hex
     """
+    txold_hash_hex = tx_normalize.tx_hash(txold_hash_hex)
     session = SessionBase.create_session()
 
     otx = Otx.load(txold_hash_hex, session)

apps/cic-eth/cic_eth/eth/meta.py

@@ -2,8 +2,9 @@
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.status import Status as TxStatus
 from cic_eth_registry.erc20 import ERC20Token
+from hexathon import add_0x
 
-# local imports
+# local impor:ts
 from cic_eth.ext.address import translate_address
 
 
@@ -44,8 +45,8 @@ class ExtendedTx:
             destination = source
         if destination_value == None:
             destination_value = source_value
-        st = ERC20Token(self.chain_spec, self.rpc, source)
-        dt = ERC20Token(self.chain_spec, self.rpc, destination)
+        st = ERC20Token(self.chain_spec, self.rpc, add_0x(source))
+        dt = ERC20Token(self.chain_spec, self.rpc, add_0x(destination))
         self.source_token = source
         self.source_token_symbol = st.symbol
         self.source_token_name = st.name

apps/cic-eth/cic_eth/eth/nonce.py

@@ -3,11 +3,12 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.address import is_checksum_address
+from chainlib.eth.address import is_checksum_address, is_address, strip_0x
 
 # local imports
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.nonce import (
         Nonce,
@@ -42,7 +43,8 @@ class CustodialTaskNonceOracle():
         :returns: Nonce
         :rtype: number
         """
-        r = NonceReservation.release(self.address, self.uuid, session=self.session)
+        address = tx_normalize.wallet_address(self.address)
+        r = NonceReservation.release(address, self.uuid, session=self.session)
         return r[1]
 
 
@@ -58,17 +60,18 @@ def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None):
         address = chained_input
         logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
     else:
-        if is_checksum_address(signer_address):
+        if is_address(signer_address):
             address = signer_address
             logg.debug('explicit address for reserve nonce {}'.format(signer_address))
         else:
             address = AccountRole.get_address(signer_address, session=session)
             logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
 
-    if not is_checksum_address(address):
+    if not is_address(address):
         raise ValueError('invalid result when resolving address for nonce {}'.format(address))
 
     root_id = self.request.root_id
+    address = tx_normalize.wallet_address(address)
     r = NonceReservation.next(address, root_id, session=session)
     logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
 

apps/cic-eth/cic_eth/eth/trust.py

@@ -0,0 +1,77 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from eth_address_declarator import Declarator
+from chainlib.connection import RPCConnection
+from chainlib.chain import ChainSpec
+from cic_eth.db.models.role import AccountRole
+from cic_eth_registry import CICRegistry
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.task import BaseTask
+from cic_eth.error import TrustError
+
+celery_app = celery.current_app
+logg = logging.getLogger()
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
+    proof = strip_0x(proof)
+
+    proofs = []
+ 
+    logg.debug('proof count {}'.format(len(proofs)))
+    if len(proofs) == 0:
+        logg.debug('error {}'.format(len(proofs)))
+        raise TrustError('foo')
+
+    return (chained_input, (proof, proofs))
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
+    queue = self.request.delivery_info.get('routing_key')
+
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    session = self.create_session()
+    sender_address = AccountRole.get_address('DEFAULT', session)
+
+    registry = CICRegistry(chain_spec, rpc)
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+
+    declarator = Declarator(chain_spec)
+
+    have_proofs = {}
+
+    for proof in proofs:
+
+        proof = strip_0x(proof)
+
+        have_proofs[proof] = []
+
+        for trusted_address in self.trusted_addresses:
+            o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
+            r = rpc.do(o)
+            declarations = declarator.parse_declaration(r)
+            logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
+
+            for declaration in declarations:
+                declaration = strip_0x(declaration)
+                if declaration == proof:
+                    logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
+                    have_proofs[proof].append(trusted_address)
+
+    out_proofs = {}
+    for proof in have_proofs.keys():
+        if len(have_proofs[proof]) == 0:
+            logg.error('missing signer for proof {} subject {}'.format(proof, subject))
+            raise TrustError((subject, proof,))
+        out_proofs[proof] = have_proofs[proof]
+       
+    return (chained_input, out_proofs)

apps/cic-eth/cic_eth/eth/util.py

@@ -0,0 +1,54 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.eth.gas import RPCGasOracle
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.db.models.gas_cache import GasCache
+from cic_eth.encode import tx_normalize
+from cic_eth.db.models.base import SessionBase
+
+MAXIMUM_FEE_UNITS = 8000000
+
+logg = logging.getLogger(__name__)
+
+
+class MaxGasOracle(RPCGasOracle):
+
+    def get_fee_units(self, code=None):
+        return MAXIMUM_FEE_UNITS
+
+
+class CacheGasOracle(MaxGasOracle):
+    """Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units.
+
+    :todo: instead of max units, connect a pluggable gas heuristics engine.
+    """
+
+    def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None):
+        super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator)
+        self.value = None
+        self.address = address
+        self.method = method
+
+        address = tx_normalize.executable_address(address)
+        session = SessionBase.bind_session(session)
+        q = session.query(GasCache)
+        q = q.filter(GasCache.address==address)
+        if method != None:
+            method = strip_0x(method)
+            q = q.filter(GasCache.method==method)
+        o = q.first()
+        if o != None:
+            self.value = int(o.value)
+
+        SessionBase.release_session(session)
+
+
+    def get_fee_units(self, code=None):
+        if self.value != None:
+            logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method))
+            return self.value
+        return super(CacheGasOracle, self).get_fee_units(code=code)

apps/cic-eth/cic_eth/ext/tx.py

@@ -32,6 +32,7 @@ from potaahto.symbols import snake_and_camel
 from cic_eth.queue.time import tx_times
 from cic_eth.task import BaseTask
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -134,7 +135,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                     tx_address = transfer_data[0]
                     tx_token_value = transfer_data[1]
                     
-                    if address == tx_address:
+                    if tx_normalize.wallet_address(address) == tx_normalize.wallet_address(tx_address):
                         status = StatusEnum.SENT
                         try:
                             o = receipt(tx['hash'])
@@ -152,8 +153,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
-                            'sender': tx['from'],
-                            'recipient': tx_address,
+                            'sender': tx_normalize.wallet_address(tx['from']),
+                            'recipient': tx_normalize.wallet_address(tx_address),
                             'source_value': tx_token_value,
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
@@ -164,10 +165,10 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             tx_r['date_created'] = times['queue']
                         else:
                             tx_r['date_created'] = times['network']
-                        txs[tx['hash']] = tx_r
+                        txs[strip_0x(tx['hash'])] = tx_r
                         break
-    return txs
 
+    return txs
 
 
 # TODO: Surely it must be possible to optimize this
@@ -230,6 +231,8 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr
             except UnknownContractError:
                 logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
                 continue
+        tx['recipient'] = tx_normalize.wallet_address(tx['recipient'])
+        tx['sender'] = tx_normalize.wallet_address(tx['sender'])
         txs.append(tx)
 
     return txs

apps/cic-eth/cic_eth/pytest/fixtures_celery.py

@@ -4,18 +4,21 @@ import tempfile
 import logging
 import shutil
 
-# local impors
+# local imports
 from cic_eth.task import BaseTask
 
 #logg = logging.getLogger(__name__)
 logg = logging.getLogger()
 
-
 @pytest.fixture(scope='function')
 def init_celery_tasks(
     contract_roles, 
         ):
     BaseTask.call_address = contract_roles['DEFAULT']
+    BaseTask.trusted_addresses = [
+            contract_roles['TRUSTED_DECLARATOR'],
+            contract_roles['CONTRACT_DEPLOYER'],
+            ]
 
 
 # celery fixtures
@@ -38,6 +41,7 @@ def celery_includes():
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
         'cic_eth.pytest.mock.filter',
+        'cic_eth.pytest.mock.callback',
     ]
 
 

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -8,15 +8,14 @@ import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(os.path.dirname(script_dir))
+config_dir = os.path.join(root_dir, 'cic_eth', 'data', 'config')
 logg = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.environ.get('CONFINI_DIR')
-    if config_dir == None:
-        config_dir = os.path.join(root_dir, 'config/test')
-    conf = confini.Config(config_dir, 'CICTEST')
+    override_config_dir = os.path.join(root_dir, 'config', 'test')
+    conf = confini.Config(config_dir, 'CICTEST', override_dirs=[override_config_dir])
     conf.process()
     logg.debug('config {}'.format(conf))
     return conf

apps/cic-eth/cic_eth/pytest/mock/__init__.py

@@ -1 +1,2 @@
 from .filter import *
+from .callback import *

apps/cic-eth/cic_eth/pytest/mock/callback.py

@@ -0,0 +1,38 @@
+# standard imports
+import os
+import logging
+import mmap
+
+# standard imports
+import tempfile
+
+# external imports
+import celery
+
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
+
+celery_app = celery.current_app
+
+
+class CallbackTask(celery.Task):
+
+    mmap_path = tempfile.mkdtemp()
+
+
+@celery_app.task(bind=True, base=CallbackTask)
+def test_callback(self, a, b, c):
+    s = 'ok'
+    if c > 0:
+        s = 'err'
+
+    fp = os.path.join(self.mmap_path, b)
+    f = open(fp, 'wb+')
+    f.write(b'\x00')
+    f.seek(0)
+    m = mmap.mmap(f.fileno(), length=1)
+    m.write(c.to_bytes(1, 'big'))
+    m.close()
+    f.close()
+
+    logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))