Added mention of the docs

Update bug.md

See merge request grassrootseconomics/cic-internal-integration!313

.gitignore

@@ -15,4 +15,3 @@ build/
 .idea
 **/.vim
 **/*secret.yaml
-.env

.gitlab-ci.yml

@@ -10,6 +10,7 @@ include:
     #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
+  - version
   - build
   - test
   - deploy
@@ -20,9 +21,41 @@ variables:
   DOCKER_BUILDKIT: "1"
   COMPOSE_DOCKER_CLI_BUILD: "1"
   CI_DEBUG_TRACE: "true"
+  SEMVERBOT_VERSION: "0.2.0"
 
-before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  #before_script:
+  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+version:
+  #image: python:3.7-stretch
+  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
+  stage: version
+  tags:
+    - integration
+  script:
+    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
+    - export TAG=$(sbot predict version -m auto)
+    - |
+      if [[ -z $TAG ]]
+      then
+        echo "tag could not be set $@"
+        exit 1
+      fi
+    - echo $TAG > version
+    - git tag -a v$TAG -m "ci tagged"
+    - git push origin v$TAG
+  artifacts:
+    paths:
+      - version
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 # runs on protected branches and pushes to repo
 build-push:
@@ -30,12 +63,17 @@ build-push:
   tags:
     - integration
   #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
+  #  - TAG=$CI_Cbefore_script:
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - TAG=latest sh ./scripts/build-push.sh
+    - TAG=latest ./scripts/build-push.sh
+    - TAG=$(cat ./version) ./scripts/build-push.sh
   rules:
   - if: $CI_COMMIT_REF_PROTECTED == "true"
     when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 deploy-dev:
   stage: deploy

.gitlab/issue_templates/bug.md

@@ -0,0 +1,44 @@
+<!---
+Please read this!
+
+Before opening a new issue, make sure to search for keywords in the issues
+filtered by the "bug" label:
+
+- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
+
+and verify the issue you're about to submit isn't a duplicate.
+--->
+
+### Summary
+
+<!-- Summarize the bug encountered concisely. -->
+
+### Steps to reproduce
+
+<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
+
+### Example Project
+
+<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic 
+behavior, and link to it here in the bug report. If you are using an older version of GitLab, this 
+will also determine whether the bug is fixed in a more recent version. -->
+
+### What is the current *bug* behavior?
+
+<!-- Describe what actually happens. -->
+
+### What is the expected *correct* behavior?
+
+<!-- Describe what you should see instead. -->
+
+### Relevant logs and/or screenshots
+
+<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
+ as it's tough to read otherwise. -->
+
+
+### Possible fixes
+
+<!-- If you can, link to the line of code that might be responsible for the problem. -->
+
+/label ~"bug"

.semverbot.toml

@@ -0,0 +1,16 @@
+[git]
+
+[git.config]
+email = "semverbot@grassroots.org"
+name = "semvervot"
+
+[git.tags]
+prefix = "v"
+
+[semver]
+mode = "git-commit"
+
+[semver.detection]
+patch = ["fix", "[fix]", "patch", "[patch]"]
+minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
+major = ["BREAKING CHANGE"]

CODE_OF_CONDUCT.md

@@ -0,0 +1,83 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
+
+
+
+
+...Try to keep in mind the immortal
+words of Bill and Ted, "Be excellent to each other."
+

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider. 
+
+__To request a change to the code please fork this repository and sumbit a merge request.__
+
+__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
+
+__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
+
+__Ask a question in our dev chat:__
+
+[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
+
+[Discord](https://discord.gg/XWunwAsX)
+
+[Matrix, IRC soon?]
+

LICENSE

@@ -1,5 +1,5 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
 
  Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
@@ -7,17 +7,15 @@
 
                             Preamble
 
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
 
   The licenses for most software and other practical works are designed
 to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
+our General Public Licenses are intended to guarantee your freedom to
 share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
+software for all its users.
 
   When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
@@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you
 want it, that you can change the software or use pieces of it in new
 free programs, and that you know you can do these things.
 
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
 
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
 
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
 
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
@@ -72,7 +60,7 @@ modification follow.
 
   0. Definitions.
 
-  "This License" refers to version 3 of the GNU General Public License.
+  "This License" refers to version 3 of the GNU Affero General Public License.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
@@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey
 the Program, the only way you could satisfy both those terms and this
 License would be to refrain entirely from conveying the Program.
 
-  13. Use with the GNU Affero General Public License.
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
 
   Notwithstanding any other provision of this License, you have
 permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
+under version 3 of the GNU General Public License into a single
 combined work, and to convey the resulting work.  The terms of this
 License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
 
   14. Revised Versions of this License.
 
   The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
 address new problems or concerns.
 
   Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
+Program specifies that a certain numbered version of the GNU Affero General
 Public License "or any later version" applies to it, you have the
 option of following the terms and conditions either of that numbered
 version or of any later version published by the Free Software
 Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
+GNU Affero General Public License, you may choose any version ever published
 by the Free Software Foundation.
 
   If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
+versions of the GNU Affero General Public License can be used, that proxy's
 public statement of acceptance of a version permanently authorizes you
 to choose that version for the Program.
 
@@ -631,44 +629,33 @@ to attach them to the start of each source file to most effectively
 state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
+    cic-internal-integration
+    Copyright (C) 2021  Grassroots Economics
 
     This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
     (at your option) any later version.
 
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+    GNU Affero General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
+    You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
 
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
+For more information on this, and how to apply and follow the GNU AGPL, see
 <https://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.

README.md

@@ -1,13 +1,12 @@
-# cic-internal-integration
+# CIC Stack (cic-internal-integration)
+
+Documentation as well as community contribution guides can be found [here](https://docs.grassecon.org) and the source of those docs can be found [here](https://gitlab.com/grassrootseconomics/grassrootseconomics.gitlab.io) 
 
 ## Getting started 
 
-### Preperation
-
-Copy the .env_sample file to .env and populate with appropriate env vars
-
 This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
+
 ```
 export COMPOSE_DOCKER_CLI_BUILD=1
 export DOCKER_BUILDKIT=1

apps/cic-base-os/aux/wait-for-it/.gitignore

@@ -0,0 +1,3 @@
+**/*.pyc
+.pydevproject
+/vendor/

apps/cic-base-os/aux/wait-for-it/.travis.yml

@@ -0,0 +1,7 @@
+language: python
+python:
+  - "2.7"
+
+script:
+  - python test/wait-for-it.py
+

apps/cic-base-os/aux/wait-for-it/LICENSE

@@ -1,4 +1,5 @@
-Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com>
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

apps/cic-base-os/aux/wait-for-it/README.md

@@ -0,0 +1,75 @@
+# wait-for-it
+
+`wait-for-it.sh` is a pure bash script that will wait on the availability of a
+host and TCP port.  It is useful for synchronizing the spin-up of
+interdependent services, such as linked docker containers.  Since it is a pure
+bash script, it does not have any external dependencies.
+
+## Usage
+
+```text
+wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
+-h HOST | --host=HOST       Host or IP under test
+-p PORT | --port=PORT       TCP port under test
+                            Alternatively, you specify the host and port as host:port
+-s | --strict               Only execute subcommand if the test succeeds
+-q | --quiet                Don't output any status messages
+-t TIMEOUT | --timeout=TIMEOUT
+                            Timeout in seconds, zero for no timeout
+-- COMMAND ARGS             Execute command with args after the test finishes
+```
+
+## Examples
+
+For example, let's test to see if we can access port 80 on `www.google.com`,
+and if it is available, echo the message `google is up`.
+
+```text
+$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+You can set your own timeout with the `-t` or `--timeout=` option.  Setting
+the timeout value to 0 will disable the timeout:
+
+```text
+$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting for www.google.com:80 without a timeout
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+The subcommand will be executed regardless if the service is up or not.  If you
+wish to execute the subcommand only if the service is up, add the `--strict`
+argument. In this example, we will test port 81 on `www.google.com` which will
+fail:
+
+```text
+$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
+wait-for-it.sh: waiting 1 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
+wait-for-it.sh: strict mode, refusing to execute subprocess
+```
+
+If you don't want to execute a subcommand, leave off the `--` argument.  This
+way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
+and determine how to proceed:
+
+```text
+$ ./wait-for-it.sh www.google.com:80
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+$ echo $?
+0
+$ ./wait-for-it.sh www.google.com:81
+wait-for-it.sh: waiting 15 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
+$ echo $?
+124
+```
+
+## Community
+
+*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-base-os/aux/wait-for-it/wait-for-it.sh

@@ -179,4 +179,4 @@ if [[ $WAITFORIT_CLI != "" ]]; then
     exec "${WAITFORIT_CLI[@]}"
 else
     exit $WAITFORIT_RESULT
-fi
+fi

apps/cic-cache/aux/wait-for-it/.gitignore

@@ -0,0 +1,3 @@
+**/*.pyc
+.pydevproject
+/vendor/

apps/cic-cache/aux/wait-for-it/.travis.yml

@@ -0,0 +1,7 @@
+language: python
+python:
+  - "2.7"
+
+script:
+  - python test/wait-for-it.py
+

apps/cic-cache/aux/wait-for-it/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apps/cic-cache/aux/wait-for-it/README.md

@@ -0,0 +1,75 @@
+# wait-for-it
+
+`wait-for-it.sh` is a pure bash script that will wait on the availability of a
+host and TCP port.  It is useful for synchronizing the spin-up of
+interdependent services, such as linked docker containers.  Since it is a pure
+bash script, it does not have any external dependencies.
+
+## Usage
+
+```text
+wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
+-h HOST | --host=HOST       Host or IP under test
+-p PORT | --port=PORT       TCP port under test
+                            Alternatively, you specify the host and port as host:port
+-s | --strict               Only execute subcommand if the test succeeds
+-q | --quiet                Don't output any status messages
+-t TIMEOUT | --timeout=TIMEOUT
+                            Timeout in seconds, zero for no timeout
+-- COMMAND ARGS             Execute command with args after the test finishes
+```
+
+## Examples
+
+For example, let's test to see if we can access port 80 on `www.google.com`,
+and if it is available, echo the message `google is up`.
+
+```text
+$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+You can set your own timeout with the `-t` or `--timeout=` option.  Setting
+the timeout value to 0 will disable the timeout:
+
+```text
+$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting for www.google.com:80 without a timeout
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+The subcommand will be executed regardless if the service is up or not.  If you
+wish to execute the subcommand only if the service is up, add the `--strict`
+argument. In this example, we will test port 81 on `www.google.com` which will
+fail:
+
+```text
+$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
+wait-for-it.sh: waiting 1 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
+wait-for-it.sh: strict mode, refusing to execute subprocess
+```
+
+If you don't want to execute a subcommand, leave off the `--` argument.  This
+way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
+and determine how to proceed:
+
+```text
+$ ./wait-for-it.sh www.google.com:80
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+$ echo $?
+0
+$ ./wait-for-it.sh www.google.com:81
+wait-for-it.sh: waiting 15 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
+$ echo $?
+124
+```
+
+## Community
+
+*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-cache/aux/wait-for-it/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -8,6 +8,7 @@ import base64
 import confini
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.runnable.daemons.query import (
@@ -23,26 +24,17 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_cache', 'db')
 migrationsdir = os.path.join(dbdir, 'migrations')
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
+# process args
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_task
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
+# connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 

apps/cic-cache/cic_cache/runnable/daemons/tasker.py

@@ -9,6 +9,7 @@ import celery
 import confini
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.tasks.tx import *
@@ -16,35 +17,20 @@ from cic_cache.tasks.tx import *
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-
+# process args
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_task
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn)
 
-# verify database connection with minimal sanity query
-#session = SessionBase.create_session()
-#session.execute('select version_num from alembic_version')
-#session.close()
-
 # set up celery
 current_app = celery.Celery(__name__)
 
@@ -87,9 +73,9 @@ def main():
     elif args.v:
         argv.append('--loglevel=INFO')
     argv.append('-Q')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
     argv.append('-n')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
 
     current_app.worker_main(argv)
 

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -40,7 +40,7 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 # process args
-arg_flags = cic_cache.cli.argflag_std_read
+arg_flags = cic_cache.cli.argflag_std_base
 local_arg_flags = cic_cache.cli.argflag_local_sync
 argparser = cic_cache.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)

apps/cic-cache/docker/Dockerfile

@@ -1,19 +1,17 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 
 COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG EXTRA_PIP_ARGS=""
+ARG PIP_INDEX_URL="https://pypi.org/simple"
+
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt
 
 COPY . . 
@@ -23,10 +21,10 @@ RUN python setup.py install
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
+#COPY config/ /usr/local/etc/cic-cache/
 
 # for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY ./aux/wait-for-it/wait-for-it.sh ./
 COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
 COPY /docker/start_tracker.sh ./start_tracker.sh

apps/cic-cache/scripts/migrate.py

@@ -17,11 +17,12 @@ logg = logging.getLogger()
 rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_cache', 'db')
 migrationsdir = os.path.join(dbdir, 'migrations')
+configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
 
-config_dir = os.path.join('/usr/local/etc/cic-cache')
+#config_dir = os.path.join('/usr/local/etc/cic-cache')
 
 argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-c', type=str, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
@@ -35,7 +36,7 @@ if args.vv:
 elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
-config = confini.Config(args.c, args.env_prefix)
+config = confini.Config(configdir, args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,4 +1,5 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.3a1
-cic-eth-registry>=0.6.1a2,<0.7.0
-cic-eth[services]~=0.12.4a8
+erc20-demurrage-token~=0.0.5a3
+cic-eth-registry~=0.6.1a6
+chainlib~=0.0.9rc1
+cic_eth~=0.12.4a11

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.2a6
+version = 0.0.2a7
 description = cic-eth tasks supporting erc20 demurrage token
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a3,<0.7.0
+cic-eth-registry>=0.6.1a6,<0.7.0
 hexathon~=0.0.1a8
 chainqueue>=0.0.4a6,<0.1.0
 eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/admin/token.py

@@ -1,21 +1,2 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
 # local imports
-from cic_eth.task import BaseTask
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-            'symbol': self.default_token_symbol,
-            'address': self.default_token_address,
-            'name': self.default_token_name,
-            'decimals': self.default_token_decimals,
-            }
+from cic_eth.eth.erc20 import default_token

apps/cic-eth/cic_eth/api/api_task.py

@@ -17,15 +17,50 @@ from cic_eth.enum import LockEnum
 
 app = celery.current_app
 
-logg = logging.getLogger(__name__)
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 class Api(ApiBase):
-    
+
+    @staticmethod
+    def to_v_list(v, n):
+        """Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
+
+        :param v: Arguments
+        :type v: str or list
+        :param n: Number of elements to generate arguments for
+        :type n: int
+        :rtype: list
+        :returns: list of assembled arguments
+        """
+        if isinstance(v, str):
+            vv = v
+            v = []
+            for i in range(n):
+                v.append([vv])
+        elif not isinstance(v, list):
+            raise ValueError('argument must be single string, or list or strings or lists')
+        else:
+            if len(v) != n:
+                raise ValueError('v argument count must match integer n')
+            for i in range(n):
+                if isinstance(v[i], str):
+                    v[i] = [v[i]]
+                elif not isinstance(v, list):
+                    raise ValueError('proof argument must be single string, or list or strings or lists')
+
+        return v
+   
 
     def default_token(self):
+        """Retrieves the default fallback token of the custodial network.
+
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
         s_token = celery.signature(
-                'cic_eth.admin.token.default_token',
+                'cic_eth.eth.erc20.default_token',
                 [],
                 queue=self.queue,
                 )
@@ -35,6 +70,97 @@ class Api(ApiBase):
         return s_token.apply_async()
 
 
+    def token(self, token_symbol, proof=None):
+        """Single-token alias for tokens method.
+
+        See tokens method for details.
+
+        :param token_symbol: Token symbol to look up
+        :type token_symbol: str
+        :param proof: Proofs to add to signature verification for the token
+        :type proof: str or list
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        if not isinstance(token_symbol, str):
+            raise ValueError('token symbol must be string')
+
+        return self.tokens([token_symbol], proof=proof)
+
+
+    def tokens(self, token_symbols, proof=None):
+        """Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
+
+        Token symbols are always strings, and should be specified using uppercase letters.
+
+        If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
+
+        The proof argument may be specified in a number of ways:
+
+        - as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
+        - as a single string, where the same proof is used for each token lookup
+        - as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
+        - as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
+
+        The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
+
+        This method is not intended to be used synchronously. Do so at your peril.
+
+        :param token_symbols: Token symbol strings to look up
+        :type token_symbol: list
+        :param proof: Proof(s) to verify tokens against
+        :type proof: None, str or list
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        if not isinstance(token_symbols, list):
+            raise ValueError('token symbols argument must be list')
+
+        if proof == None:
+            logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
+            proof = ''
+
+        logg.debug('proof is {}'.format(proof))
+        l = len(token_symbols)
+        if len(proof) == 0:
+            l = 0 
+        proof = Api.to_v_list(proof, l)
+
+        chain_spec_dict = self.chain_spec.asdict()
+
+        s_token_resolve = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    token_symbols,
+                    chain_spec_dict,
+                    ],
+                queue=self.queue,
+                )
+
+        s_token_info = celery.signature(
+                'cic_eth.eth.erc20.token_info',
+                [
+                    chain_spec_dict,
+                    proof,
+                    ],
+                queue=self.queue,
+                )
+
+        s_token_verify = celery.signature(
+                    'cic_eth.eth.erc20.verify_token_info',
+                    [
+                        chain_spec_dict,
+                        self.callback_success,
+                        self.callback_error,
+                        ],
+                    queue=self.queue,
+                    )
+
+        s_token_info.link(s_token_verify)
+        s_token_resolve.link(s_token_info)
+        return s_token_resolve.apply_async()
+
+
 #    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
 #        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 #
@@ -557,3 +683,4 @@ class Api(ApiBase):
 
         t = self.callback_success.apply_async([r])
         return t
+

apps/cic-eth/cic_eth/callbacks/noop.py

@@ -1,7 +1,10 @@
+import logging
+
 import celery
 
 celery_app = celery.current_app
-logg = celery_app.log.get_default_logger()
+#logg = celery_app.log.get_default_logger()
+logg = logging.getLogger()
 
 
 @celery_app.task(bind=True)

apps/cic-eth/cic_eth/error.py

@@ -48,8 +48,6 @@ class RoleMissingError(Exception):
     pass
 
 
-
-
 class IntegrityError(Exception):
     """Exception raised to signal irregularities with deduplication and ordering of tasks
 
@@ -85,3 +83,8 @@ class RoleAgencyError(SeppukuError):
 class YouAreBrokeError(Exception):
     """Exception raised when a value transfer is attempted without access to sufficient funds
     """
+
+
+class TrustError(Exception):
+    """Exception raised when required trust proofs are missing for a request
+    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -19,6 +19,7 @@ from hexathon import (
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
 from chainqueue.sql.tx import cache_tx_dict
+from okota.token_index import to_identifier
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -39,9 +40,11 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
         CriticalSQLAlchemyAndSignerTask,
+        BaseTask,
     )
 from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.encode import tx_normalize
+from cic_eth.eth.trust import verify_proofs
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -473,3 +476,69 @@ def cache_approve_data(
     session.close()
     return (tx_hash_hex, cache_id)
 
+
+@celery_app.task(bind=True, base=BaseTask)
+def token_info(self, tokens, chain_spec_dict, proofs=[]):
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    i = 0
+
+    for token in tokens:
+        result_data = []
+        token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
+        token_chain_object.load(rpc)
+
+        token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
+        token_proofs = [token_symbol_proof_hex]
+        if len(proofs) > 0:
+            token_proofs += proofs[i]
+
+        tokens[i] = {
+            'decimals': token_chain_object.decimals,
+            'name': token_chain_object.name,
+            'symbol': token_chain_object.symbol,
+            'address': tx_normalize.executable_address(token_chain_object.address),
+            'proofs': token_proofs,
+            'converters': tokens[i]['converters'],
+                }   
+        i += 1
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
+    queue = self.request.delivery_info.get('routing_key')
+
+    for token in tokens:
+        s = celery.signature(
+                'cic_eth.eth.trust.verify_proofs',
+                [
+                    token,
+                    token['address'],
+                    token['proofs'],
+                    chain_spec_dict,
+                    success_callback,
+                    error_callback,
+                    ],
+                queue=queue,
+                )
+
+        if success_callback != None:
+            s.link(success_callback)
+        if error_callback != None:
+            s.on_error(error_callback)
+        s.apply_async()
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def default_token(self):
+    return {
+        'symbol': self.default_token_symbol,
+        'address': self.default_token_address,
+        'name': self.default_token_name,
+        'decimals': self.default_token_decimals,
+        }

apps/cic-eth/cic_eth/eth/trust.py

@@ -0,0 +1,77 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from eth_address_declarator import Declarator
+from chainlib.connection import RPCConnection
+from chainlib.chain import ChainSpec
+from cic_eth.db.models.role import AccountRole
+from cic_eth_registry import CICRegistry
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.task import BaseTask
+from cic_eth.error import TrustError
+
+celery_app = celery.current_app
+logg = logging.getLogger()
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
+    proof = strip_0x(proof)
+
+    proofs = []
+ 
+    logg.debug('proof count {}'.format(len(proofs)))
+    if len(proofs) == 0:
+        logg.debug('error {}'.format(len(proofs)))
+        raise TrustError('foo')
+
+    return (chained_input, (proof, proofs))
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
+    queue = self.request.delivery_info.get('routing_key')
+
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    session = self.create_session()
+    sender_address = AccountRole.get_address('DEFAULT', session)
+
+    registry = CICRegistry(chain_spec, rpc)
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+
+    declarator = Declarator(chain_spec)
+
+    have_proofs = {}
+
+    for proof in proofs:
+
+        proof = strip_0x(proof)
+
+        have_proofs[proof] = []
+
+        for trusted_address in self.trusted_addresses:
+            o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
+            r = rpc.do(o)
+            declarations = declarator.parse_declaration(r)
+            logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
+
+            for declaration in declarations:
+                declaration = strip_0x(declaration)
+                if declaration == proof:
+                    logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
+                    have_proofs[proof].append(trusted_address)
+
+    out_proofs = {}
+    for proof in have_proofs.keys():
+        if len(have_proofs[proof]) == 0:
+            logg.error('missing signer for proof {} subject {}'.format(proof, subject))
+            raise TrustError((subject, proof,))
+        out_proofs[proof] = have_proofs[proof]
+       
+    return (chained_input, out_proofs)

apps/cic-eth/cic_eth/pytest/fixtures_celery.py

@@ -4,18 +4,21 @@ import tempfile
 import logging
 import shutil
 
-# local impors
+# local imports
 from cic_eth.task import BaseTask
 
 #logg = logging.getLogger(__name__)
 logg = logging.getLogger()
 
-
 @pytest.fixture(scope='function')
 def init_celery_tasks(
     contract_roles, 
         ):
     BaseTask.call_address = contract_roles['DEFAULT']
+    BaseTask.trusted_addresses = [
+            contract_roles['TRUSTED_DECLARATOR'],
+            contract_roles['CONTRACT_DEPLOYER'],
+            ]
 
 
 # celery fixtures
@@ -38,6 +41,7 @@ def celery_includes():
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
         'cic_eth.pytest.mock.filter',
+        'cic_eth.pytest.mock.callback',
     ]
 
 

apps/cic-eth/cic_eth/pytest/mock/__init__.py

@@ -1 +1,2 @@
 from .filter import *
+from .callback import *

apps/cic-eth/cic_eth/pytest/mock/callback.py

@@ -0,0 +1,38 @@
+# standard imports
+import os
+import logging
+import mmap
+
+# standard imports
+import tempfile
+
+# external imports
+import celery
+
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
+
+celery_app = celery.current_app
+
+
+class CallbackTask(celery.Task):
+
+    mmap_path = tempfile.mkdtemp()
+
+
+@celery_app.task(bind=True, base=CallbackTask)
+def test_callback(self, a, b, c):
+    s = 'ok'
+    if c > 0:
+        s = 'err'
+
+    fp = os.path.join(self.mmap_path, b)
+    f = open(fp, 'wb+')
+    f.write(b'\x00')
+    f.seek(0)
+    m = mmap.mmap(f.fileno(), length=1)
+    m.write(c.to_bytes(1, 'big'))
+    m.close()
+    f.close()
+
+    logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -10,7 +10,6 @@ import datetime
 
 # external imports
 import celery
-from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
 from chainlib.connection import RPCConnection

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -76,7 +76,7 @@ arg_flags = cic_eth.cli.argflag_std_read
 local_arg_flags = cic_eth.cli.argflag_local_task
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)
-argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
+#argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
 argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
@@ -84,7 +84,7 @@ args = argparser.parse_args()
 
 # process config
 extra_args = {
-    'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
+#    'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
     'aux_all': None,
     'aux': None,
     'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
@@ -187,6 +187,17 @@ elif len(args.aux) > 0:
         logg.info('aux module {} found in path {}'.format(v, aux_dir))
         aux.append(v)
 
+default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
+defaullt_token_address = None
+if default_token_symbol:
+    default_token_address = registry.by_name(default_token_symbol)
+else:
+    default_token_address = registry.by_name('DefaultToken')
+    c = ERC20Token(chain_spec, conn, default_token_address)
+    default_token_symbol = c.symbol
+    logg.info('found default token {} address {}'.format(default_token_symbol, default_token_address))
+    config.add(default_token_symbol, 'CIC_DEFAULT_TOKEN_SYMBOL', exists_ok=True)
+
 for v in aux:
     mname = 'cic_eth_aux.' + v
     mod = importlib.import_module(mname)
@@ -204,12 +215,13 @@ def main():
     argv.append('-n')
     argv.append(config.get('CELERY_QUEUE'))
 
-    BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-    BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
+    BaseTask.default_token_symbol = default_token_symbol
+    BaseTask.default_token_address = default_token_address
     default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
     default_token.load(conn)
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name
+    BaseTask.trusted_addresses = trusted_addresses
 
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

apps/cic-eth/cic_eth/task.py

@@ -28,6 +28,7 @@ class BaseTask(celery.Task):
 
     session_func = SessionBase.create_session
     call_address = ZERO_ADDRESS
+    trusted_addresses = []
     create_nonce_oracle = RPCNonceOracle
     create_gas_oracle = RPCGasOracle
     default_token_address = None

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         4,
-        'alpha.8',
+        'alpha.14',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/doc/texinfo/accounts.texi

@@ -1,4 +1,3 @@
-@node cic-eth-accounts
 @section Accounts
 
 Accounts are private keys in the signer component keyed by "addresses," a one-way transformation of a public key. Data can be signed by using the account as identifier for corresponding RPC requests.

apps/cic-eth/doc/texinfo/admin.texi

@@ -1,4 +1,4 @@
-@node cic-eth system maintenance
+@anchor{cic-eth-appendix-system-maintenance}
 @appendix Admin API
 
 The admin API is still in an early stage of refinement. User friendliness can be considerably improved.
@@ -33,7 +33,7 @@ Get the current state of a lock
 
 @appendixsection tag_account
 
-Associate an identifier with an account address (@xref{cic-eth system accounts})
+Associate an identifier with an account address (@xref{cic-eth-system-accounts})
 
 @appendixsection have_account
 

apps/cic-eth/doc/texinfo/all.texi

@@ -14,5 +14,6 @@ Released 2021 under GPL3
 @c
 @contents
 
-@include index.texi
+@include content.texi
+@include appendix.texi
 

apps/cic-eth/doc/texinfo/appendix.texi

@@ -0,0 +1,3 @@
+@include admin.texi
+@include chains.texi
+@include transfertypes.texi

apps/cic-eth/doc/texinfo/chains.texi

@@ -1,4 +1,4 @@
-@node cic-eth Appendix Task chains
+@anchor{cic-eth-appendix-task-chains}
 @appendix Task chains
 
 TBC - explain here how to generate these chain diagrams

apps/cic-eth/doc/texinfo/configuration.texi

@@ -1,4 +1,3 @@
-@node cic-eth configuration
 @section Configuration
 
 Configuration parameters are grouped by configuration filename.

apps/cic-eth/doc/texinfo/content.texi

@@ -1,6 +1,6 @@
+@node cic-eth
 @top cic-eth
 
-@include intro.texi
 @include dependencies.texi
 @include configuration.texi
 @include system.texi
@@ -9,6 +9,3 @@
 @include incoming.texi
 @include services.texi
 @include tools.texi
-@include admin.texi
-@include chains.texi
-@include transfertypes.texi

apps/cic-eth/doc/texinfo/dependencies.texi

@@ -1,4 +1,3 @@
-@node cic-eth-dependencies
 @section Dependencies
 
 This application is written in Python 3.8. It is tightly coupled with @code{python-celery}, which provides the task worker ecosystem. It also uses @code{SQLAlchemy} which provides useful abstractions for persistent storage though SQL, and @code{alembic} for database schema migrations.

apps/cic-eth/doc/texinfo/incoming.texi

@@ -1,4 +1,4 @@
-@node cic-eth-incoming
+@anchor{cic-eth-incoming}
 @section Incoming transactions
 
 All transactions in mined blocks will be passed to a selection of plugin filters to the @code{chainsyncer} component. Each of these filters are individual python module files in @code{cic_eth.runnable.daemons.filters}. This section describes their function.

apps/cic-eth/doc/texinfo/interacting.texi

@@ -1,9 +1,8 @@
-@node cic-eth-interacting
 @section Interacting with the system
 
-The API to the @var{cic-eth} component is a proxy for executing @emph{chains of Celery tasks}. The tasks that compose individual chains are documented in @ref{cic-eth Appendix Task chains,the Task Chain appendix}, which also describes a CLI tool that can generate graph representationso of them.
+The API to the @var{cic-eth} component is a proxy for executing @emph{chains of Celery tasks}. The tasks that compose individual chains are documented in @ref{cic-eth-appendix-task-chains,the Task Chain appendix}, which also describes a CLI tool that can generate graph representationso of them.
 
-There are two API classes, @var{Api} and @var{AdminApi}. The former is described later in this section, the latter described in @ref{cic-eth system maintenance,the Admin API appendix}.
+There are two API classes, @var{Api} and @var{AdminApi}. The former is described later in this section, the latter described in @ref{cic-eth-appendix-system-maintenance,the Admin API appendix}.
 
 
 @subsection Interface

apps/cic-eth/doc/texinfo/outgoing.texi

@@ -1,4 +1,3 @@
-@node cic-eth-outgoing
 @section Outgoing transactions
 
 @strong{Important! A pre-requisite for proper functioning of the component is that no other agent is sending transactions to the network for any of the keys in the keystore.}

apps/cic-eth/doc/texinfo/services.texi

@@ -1,4 +1,3 @@
-@node cic-eth-services
 @section Services
 
 There are four daemons that together orchestrate all of the aforementioned recipes. This section will provide a high level description of them. 

apps/cic-eth/doc/texinfo/system.texi

@@ -1,10 +1,10 @@
-@node cic-eth system accounts
 @section System initialization
 
 When the system starts for the first time, it is locked for any state change request other than account creation@footnote{Specifically, the @code{INIT}, @code{SEND} and @code{QUEUE} lock bits are set.}. These locks should be @emph{reset} once system initialization has been completed. Currently, system initialization only involves creating and tagging required system accounts, as specified below.
 
 See @ref{cic-eth-locking,Locking} and @ref{cic-eth-tools-ctrl,ctrl in Tools} for details on locking.
 
+@anchor{cic-eth-system-accounts}
 @subsection System accounts
 
 Certain accounts in the system have special roles. These are defined by @emph{tagging} certain accounts addresses with well-known identifiers.

apps/cic-eth/doc/texinfo/tools.texi

@@ -1,4 +1,3 @@
-@node cic-eth-tools
 @section Tools
 
 A collection of CLI tools have been provided to help with diagnostics and other administrative tasks. These use the same configuration infrastructure as the daemons.
@@ -37,7 +36,7 @@ Execute a token transfer on behalf of a custodial account.
 
 @subsection tag (cic-eth-tag)
 
-Associate an account address with a string identifier. @xref{cic-eth system accounts}
+Associate an account address with a string identifier. @xref{cic-eth-system-accounts}
 
 
 @anchor{cic-eth-tools-ctrl}

apps/cic-eth/doc/texinfo/transfertypes.texi

@@ -1,4 +1,3 @@
-@node cic-eth Appendix Transaction types
 @appendix Transfer types
 
 @table @var

apps/cic-eth/docker/Dockerfile

@@ -1,46 +1,32 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 
 # Copy just the requirements and install....this _might_ give docker a hint on caching but we 
 # do load these all into setup.py later
 # TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
 #COPY cic-eth/requirements.txt .
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
 ARG EXTRA_PIP_ARGS=""
-#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-#    pip install --index-url https://pypi.org/simple \
-#    --force-reinstall \
-#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-#    -r requirements.txt
+ARG PIP_INDEX_URL=https://pypi.org/simple
 
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
+    pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+        cic-eth-aux-erc20-demurrage-token~=0.0.2a7
 
 
 COPY *requirements.txt ./
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
+    pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt 
-   
-# always install the latest signer 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    crypto-dev-signer
-
+  
 COPY . .
 RUN python setup.py install
 
@@ -53,7 +39,7 @@ RUN chmod 755 *.sh
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
+#COPY config/ /usr/local/etc/cic-eth/
 COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 

apps/cic-eth/docker/entrypoints/db.sh

@@ -2,5 +2,6 @@
 
 set -e
 >&2 echo executing database migration
-python scripts/migrate.py -c /usr/local/etc/cic-eth --migrations-dir /usr/local/share/cic-eth/alembic -vv
+#python scripts/migrate.py -c /usr/local/etc/cic-eth --migrations-dir /usr/local/share/cic-eth/alembic -vv
+python scripts/migrate.py --migrations-dir /usr/local/share/cic-eth/alembic -vv
 set +e

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
 celery==4.4.7
-chainlib-eth>=0.0.9rc2,<0.1.0
+chainlib-eth>=0.0.10a16,<0.1.0
 semver==2.13.0
 crypto-dev-signer>=0.4.15rc2,<0.5.0

apps/cic-eth/services_requirements.txt

@@ -1,15 +1,16 @@
-chainqueue>=0.0.5a1,<0.1.0
-chainsyncer[sql]>=0.0.6a3,<0.1.0
+chainqueue>=0.0.6a1,<0.1.0
+chainsyncer[sql]>=0.0.7a3,<0.1.0
 alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.2.3a4,<0.3.0
+eth-address-index>=0.2.4a1,<0.3.0
 eth-accounts-index>=0.1.2a3,<0.2.0
-cic-eth-registry>=0.6.1a3,<0.7.0
+cic-eth-registry>=0.6.1a6,<0.7.0
 erc20-faucet>=0.3.2a2,<0.4.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0
 moolb~=0.1.1b2
+okota>=0.2.4a6,<0.3.0

apps/cic-eth/setup.cfg

@@ -1,6 +1,7 @@
 [metadata]
 name = cic-eth
-version = attr: cic_eth.version.__version_string__
+#version = attr: cic_eth.version.__version_string__
+version = 0.12.4a13
 description = CIC Network Ethereum interaction 
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth/tests/task/api/test_app_noncritical.py

@@ -1,6 +1,27 @@
+# standard imports
+import logging
+import os
+import uuid
+import time
+import mmap
+
+# external imports
+import celery
+import pytest
+from hexathon import (
+        strip_0x,
+        uniform as hex_uniform,
+        )
+
 # local imports
 from cic_eth.api.api_task import Api
 from cic_eth.task import BaseTask
+from cic_eth.error import TrustError
+from cic_eth.encode import tx_normalize
+from cic_eth.pytest.mock.callback import CallbackTask
+
+logg = logging.getLogger()
+
 
 def test_default_token(
         default_chain_spec,
@@ -17,3 +38,175 @@ def test_default_token(
     t = api.default_token()
     r = t.get_leaf()
     assert r['address'] == foo_token
+
+
+def test_to_v_list():
+    assert Api.to_v_list('', 0) == []
+    assert Api.to_v_list([], 0) == []
+    assert Api.to_v_list('foo', 1) == [['foo']]
+    assert Api.to_v_list(['foo'], 1) == [['foo']]
+    assert Api.to_v_list(['foo', 'bar'], 2) == [['foo'], ['bar']]
+    assert Api.to_v_list('foo', 3) == [['foo'], ['foo'], ['foo']]
+    assert Api.to_v_list([['foo'], ['bar']], 2) == [['foo'], ['bar']]
+    with pytest.raises(ValueError):
+        Api.to_v_list([['foo'], ['bar']], 3)
+    with pytest.raises(ValueError):
+        Api.to_v_list(['foo', 'bar'], 3)
+    with pytest.raises(ValueError):
+        Api.to_v_list([['foo'], ['bar'], ['baz']], 2)
+
+    assert Api.to_v_list([
+            ['foo'],
+            'bar',
+            ['inky', 'pinky', 'blinky', 'clyde'],
+        ], 3) == [
+            ['foo'],
+            ['bar'],
+            ['inky', 'pinky', 'blinky', 'clyde'],
+            ]
+
+
+def test_token_single(
+        default_chain_spec,
+        foo_token,
+        bar_token,
+        token_registry,
+        register_tokens,
+        register_lookups,
+        cic_registry,
+        init_database,
+        init_celery_tasks,
+        custodial_roles,
+        foo_token_declaration,
+        bar_token_declaration,
+        celery_session_worker,
+        ):
+
+    api = Api(str(default_chain_spec), queue=None, callback_param='foo')     
+
+    t = api.token('FOO', proof=None)
+    r = t.get()
+    logg.debug('rr {}'.format(r))
+    assert len(r) == 1
+    assert r[0]['address'] == strip_0x(foo_token)
+
+
+    t = api.token('FOO', proof=foo_token_declaration)
+    r = t.get()
+    assert len(r) == 1
+    assert r[0]['address'] == strip_0x(foo_token)
+
+
+def test_tokens_noproof(
+        default_chain_spec,
+        foo_token,
+        bar_token,
+        token_registry,
+        register_tokens,
+        register_lookups,
+        cic_registry,
+        init_database,
+        init_celery_tasks,
+        custodial_roles,
+        foo_token_declaration,
+        bar_token_declaration,
+        celery_session_worker,
+        ):
+
+    api = Api(str(default_chain_spec), queue=None, callback_param='foo')     
+
+    t = api.tokens(['FOO'], proof=[])
+    r = t.get()
+    assert len(r) == 1
+    assert r[0]['address'] == strip_0x(foo_token)
+
+    t = api.tokens(['BAR'], proof='')
+    r = t.get()
+    assert len(r) == 1
+    assert r[0]['address'] == strip_0x(bar_token)
+
+    t = api.tokens(['FOO'], proof=None)
+    r = t.get()
+    assert len(r) == 1
+    assert r[0]['address'] == strip_0x(foo_token)
+
+
+def test_tokens(
+        default_chain_spec,
+        foo_token,
+        bar_token,
+        token_registry,
+        register_tokens,
+        register_lookups,
+        cic_registry,
+        init_database,
+        init_celery_tasks,
+        custodial_roles,
+        foo_token_declaration,
+        bar_token_declaration,
+        celery_session_worker,
+        ):
+
+    api = Api(str(default_chain_spec), queue=None, callback_param='foo')     
+
+    t = api.tokens(['FOO'], proof=[[foo_token_declaration]])
+    r = t.get()
+    logg.debug('rr {}'.format(r))
+    assert len(r) == 1
+    assert r[0]['address'] == strip_0x(foo_token)
+    
+    t = api.tokens(['BAR', 'FOO'], proof=[[bar_token_declaration], [foo_token_declaration]])
+    r = t.get()
+    logg.debug('results {}'.format(r))
+    assert len(r) == 2
+    assert r[1]['address'] == strip_0x(foo_token)
+    assert r[0]['address'] == strip_0x(bar_token)
+
+    celery_app = celery.current_app
+
+    results = []
+    targets = []
+
+    api_param = str(uuid.uuid4())
+    api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
+    bogus_proof = os.urandom(32).hex()
+    t = api.tokens(['FOO'], proof=[[bogus_proof]])
+    r = t.get()
+    logg.debug('r {}'.format(r))
+
+    while True:
+        fp = os.path.join(CallbackTask.mmap_path, api_param)
+        try:
+            f = open(fp, 'rb')
+        except FileNotFoundError:
+            time.sleep(0.1)
+            logg.debug('look for {}'.format(fp))
+            continue
+        f = open(fp, 'rb')
+        m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
+        v = m.read(1)
+        m.close()
+        f.close()
+        assert v == b'\x01'
+        break
+
+    api_param = str(uuid.uuid4())
+    api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
+    t = api.tokens(['BAR'], proof=[[bar_token_declaration]])
+    r = t.get()
+    logg.debug('rr  {} {}'.format(r, t.children))
+
+    while True:
+        fp = os.path.join(CallbackTask.mmap_path, api_param)
+        try:
+            f = open(fp, 'rb')
+        except FileNotFoundError:
+            time.sleep(0.1)
+            continue
+        m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
+        v = m.read(1)
+        m.close()
+        f.close()
+        assert v == b'\x00'
+        break
+

apps/cic-eth/tests/unit/admin/test_default_token.py

@@ -10,7 +10,7 @@ def test_default_token(
         ):
       
     s = celery.signature(
-            'cic_eth.admin.token.default_token',
+            'cic_eth.eth.erc20.default_token',
             [],
             queue=None,
             )

apps/cic-eth/tools_requirements.txt

@@ -1,6 +1,6 @@
-crypto-dev-signer>=0.4.15a7,<=0.4.15
-chainqueue>=0.0.5a1,<0.1.0
-cic-eth-registry>=0.6.1a3,<0.7.0
+crypto-dev-signer>=0.4.15rc2,<=0.4.15
+chainqueue>=0.0.5a3,<0.1.0
+cic-eth-registry>=0.6.1a6,<0.7.0
 redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1

apps/cic-meta/docker/Dockerfile

@@ -1,19 +1,23 @@
-# syntax = docker/dockerfile:1.2
-#FROM node:15.3.0-alpine3.10
-FROM node:lts-alpine3.14
+FROM node:15.3.0-alpine3.10
 
 WORKDIR /root
 
 RUN apk add --no-cache postgresql bash
 
-# copy the dependencies
-COPY package.json package-lock.json .
-RUN --mount=type=cache,id=meta,mode=0755,target=/root/.npm \
-    npm set cache /root/.npm && \
-    npm ci
+ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org}
+RUN npm config set snyk=false
+#RUN npm config set registry={NPM_REPOSITORY}
+RUN npm config set registry=${NPM_REPOSITORY}
 
-COPY webpack.config.js .
-COPY tsconfig.json . 
+# copy the dependencies
+COPY package.json package-lock.json ./
+RUN --mount=type=cache,mode=0755,target=/root/.npm \
+    npm set cache /root/.npm && \
+    npm cache verify && \
+    npm ci --verbose
+
+COPY webpack.config.js ./
+COPY tsconfig.json ./
 ## required to build the cic-client-meta module
 COPY . .
 COPY tests/*.asc /root/pgp/

apps/cic-meta/scripts/initdb/server.postgres.sql

@@ -1,8 +1,9 @@
 create table if not exists store (
 	id serial primary key not null,
-	owner_fingerprint text not null,
+	owner_fingerprint text default null,
 	hash char(64) not null unique,
-	content text not null
+	content text not null,
+	mime_type text
 );
 
 create index if not exists idx_fp on store ((lower(owner_fingerprint)));

apps/cic-meta/scripts/initdb/server.sqlite.sql

@@ -1,9 +1,10 @@
 create table if not exists store (
 	/*id serial primary key not null,*/
 	id integer primary key autoincrement,
-	owner_fingerprint text not null,
+	owner_fingerprint text default null,
 	hash char(64) not null unique,
-	content text not null
+	content text not null,
+	mime_type text
 );
 
 create index if not exists idx_fp on store ((lower(owner_fingerprint)));

apps/cic-meta/scripts/server/handlers.ts

@@ -1,12 +1,13 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
+import * as crypto from 'crypto';
 
-import { Envelope, Syncable } from '@cicnet/crdt-meta';
+import { Envelope, Syncable, bytesToHex } from '@cicnet/crdt-meta';
 
 
 function handleNoMergeGet(db, digest, keystore) {
-	const sql = "SELECT content FROM store WHERE hash = '" + digest + "'";
-	return new Promise<string|boolean>((whohoo, doh) => {
+	const sql = "SELECT owner_fingerprint, content, mime_type FROM store WHERE hash = '" + digest + "'";
+	return new Promise<any>((whohoo, doh) => {
 		db.query(sql, (e, rs) => {
 			if (e !== null && e !== undefined) {
 				doh(e);
@@ -16,16 +17,36 @@ function handleNoMergeGet(db, digest, keystore) {
 				return;
 			}
 
+			const immutable = rs.rows[0]['owner_fingerprint'] == undefined;
+			let mimeType;
+			if (immutable) {
+				if (rs.rows[0]['mime_type'] === undefined) {
+					mimeType = 'application/octet-stream';
+				} else {
+					mimeType = rs.rows[0]['mime_type'];
+				}
+			} else {
+				mimeType = 'application/json';
+			}
+
 			const cipherText = rs.rows[0]['content'];
 			pgp.message.readArmored(cipherText).then((m) => {
 				const opts = {
 					message: m,
 					privateKeys: [keystore.getPrivateKey()],
+					format: 'binary',
 				};
 				pgp.decrypt(opts).then((plainText) => {
-					const o = Syncable.fromJSON(plainText.data);
-					const r = JSON.stringify(o.m['data']);
-					whohoo(r);
+					let r;
+				     	if (immutable) {
+						r = plainText.data;
+					} else {
+						mimeType = 'application/json';
+						const d = new TextDecoder().decode(plainText.data);
+						const o = Syncable.fromJSON(d);
+						r = JSON.stringify(o.m['data']);
+					}
+					whohoo([r, mimeType]);
 				}).catch((e) => {
 					console.error('decrypt', e);
 					doh(e);
@@ -57,6 +78,7 @@ function handleServerMergePost(data, db, digest, keystore, signer) {
 			} else {
 				e = Envelope.fromJSON(v);
 				s = e.unwrap();
+				console.debug('s', s, o)
 				s.replace(o, 'server merge');
 				e.set(s);
 				s.onwrap = (e) => {
@@ -139,7 +161,13 @@ function handleClientMergeGet(db, digest, keystore) {
 					privateKeys: [keystore.getPrivateKey()],
 				};
 				pgp.decrypt(opts).then((plainText) => {
-					const o = Syncable.fromJSON(plainText.data);
+					let d;
+					if (typeof(plainText.data) == 'string') {
+						d = plainText.data;
+					} else {
+						d = new TextDecoder().decode(plainText.data);
+					}
+					const o = Syncable.fromJSON(d);
 					const e = new Envelope(o);
 					whohoo(e.toJSON());
 				}).catch((e) => {
@@ -201,10 +229,65 @@ function handleClientMergePut(data, db, digest, keystore, signer) {
 	});
 }
 
+
+function handleImmutablePost(data, db, digest, keystore, contentType) {
+	return new Promise<Array<string|boolean>>((whohoo, doh) => {
+		let data_binary = data;
+		const h = crypto.createHash('sha256');
+		h.update(data_binary);
+		const z = h.digest();
+		const r = bytesToHex(z);
+
+		if (digest) {
+			if (r != digest) {
+				doh('hash mismatch: ' + r + ' != ' +  digest);
+				return;
+			}
+		} else {
+			digest = r;
+			console.debug('calculated digest ' + digest);
+		}
+
+		handleNoMergeGet(db, digest, keystore).then((haveDigest) => {
+			if (haveDigest !== false) {
+				whohoo([false, digest]);
+				return;
+			}
+			let message;
+			if (typeof(data) == 'string') {
+				data_binary = new TextEncoder().encode(data);
+				message = pgp.message.fromText(data);
+			} else {
+				message = pgp.message.fromBinary(data);
+			}
+
+					const opts = {
+				message: message,
+				publicKeys: keystore.getEncryptKeys(),
+			};
+			pgp.encrypt(opts).then((cipherText) => {
+				const sql = "INSERT INTO store (hash, content, mime_type) VALUES ('" + digest + "', '" + cipherText.data + "', '" + contentType + "') ON CONFLICT (hash) DO UPDATE SET content = EXCLUDED.content;";
+				db.query(sql, (e, rs) => {
+					if (e !== null && e !== undefined) {
+						doh(e);
+						return;
+					}
+					whohoo([true, digest]);
+				});
+			}).catch((e) => {
+				doh(e);	
+			});
+		}).catch((e) => {
+			doh(e);	
+		});
+	});
+}
+
 export {
 	handleClientMergePut,
 	handleClientMergeGet,
 	handleServerMergePost,
 	handleServerMergePut,
 	handleNoMergeGet,
+	handleImmutablePost,
 };

apps/cic-meta/scripts/server/server.ts

@@ -118,37 +118,71 @@ async function processRequest(req, res) {
 		return;
 	}
 
+	let mod = req.method.toLowerCase() + ":automerge:";
+	let modDetail = undefined;
+	let immutablePost = false;
 	try {
 		digest = parseDigest(req.url);
 	} catch(e) {
-		console.error('digest error: ' + e)
-		res.writeHead(400, {"Content-Type": "text/plain"});
-		res.end();
-		return;
+		if (req.url == '/') {
+			immutablePost = true;
+			modDetail = 'immutable';
+		} else {
+			console.error('url is not empty (' + req.url + ') and not valid digest error: ' + e)
+			res.writeHead(400, {"Content-Type": "text/plain"});
+			res.end();
+			return;
+		}
 	}
 
-	const mergeHeader = req.headers['x-cic-automerge'];
-	let mod = req.method.toLowerCase() + ":automerge:";
-	switch (mergeHeader) {
-		case "client":
-			mod += "client"; // client handles merges
-			break;
-		case "server":
-			mod += "server"; // server handles merges
-			break;
-		default:
-			mod += "none"; // merged object only (get only)
+	if (modDetail === undefined) {
+		const mergeHeader = req.headers['x-cic-automerge'];
+		switch (mergeHeader) {
+			case "client":
+				if (immutablePost) {
+					res.writeHead(400, 'Valid digest missing', {"Content-Type": "text/plain"});
+					res.end();
+					return;
+				}
+				modDetail = "client"; // client handles merges
+				break;
+			case "server":
+				if (immutablePost) {
+					res.writeHead(400, 'Valid digest missing', {"Content-Type": "text/plain"});
+					res.end();
+					return;
+				}
+				modDetail = "server"; // server handles merges
+				break;
+			case "immutable":
+				modDetail = "immutable"; // no merging, literal immutable content with content-addressing
+				break;
+			default:
+				modDetail = "none"; // merged object only (get only)
+		}
 	}
+	mod += modDetail;
 
-	let data = '';
+
+	// handle bigger chunks of data
+	let data;
 	req.on('data', (d) => {
-		data += d;
+		if (data === undefined) {
+			data = d;
+		} else {
+			data += d;
+		}
 	});
-	req.on('end', async () => {
-		console.debug('mode', mod);
-		let content = '';
+	req.on('end', async (d) => {
+		let inputContentType = req.headers['content-type'];
+		let debugString = 'executing mode ' + mod ;
+		if (data !== undefined) {
+			debugString += ' for content type ' + inputContentType + ' length ' + data.length;
+		}
+		console.debug(debugString);
+		let content;
 		let contentType = 'application/json';
-		console.debug('handling data', data);
+		let statusCode = 200;
 		let r:any = undefined;
 		try {
 			switch (mod) {
@@ -159,6 +193,7 @@ async function processRequest(req, res) {
 						res.end();
 						return;
 					}
+					content = '';
 					break;
 
 				case 'get:automerge:client':
@@ -176,6 +211,7 @@ async function processRequest(req, res) {
 						res.end();
 						return;
 					}
+					content = '';
 					break;
 				//case 'get:automerge:server':
 				//	content = await handlers.handleServerMergeGet(db, digest, keystore);	
@@ -183,12 +219,24 @@ async function processRequest(req, res) {
 
 				case 'get:automerge:none':
 					r = await handlers.handleNoMergeGet(db, digest, keystore);	
-					if (r == false) {
+					if (r === false) {
 						res.writeHead(404, {"Content-Type": "text/plain"});
 						res.end();
 						return;
 					}
-					content = r;
+					content = r[0];
+					contentType = r[1];
+					break;
+
+				case 'post:automerge:immutable':
+					if (inputContentType === undefined) {
+						inputContentType = 'application/octet-stream';
+					}
+					r = await handlers.handleImmutablePost(data, db, digest, keystore, inputContentType);
+					if (r[0]) {
+						statusCode = 201;
+					}
+					content = r[1];
 					break;
 
 				default:
@@ -204,14 +252,21 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty content', data);
+			console.error('empty content', mod, digest, data);
 			res.writeHead(404, {"Content-Type": "text/plain"});
 			res.end();
 			return;
 		}
 
-		const responseContentLength = (new TextEncoder().encode(content)).length;
-		res.writeHead(200, {
+		//let responseContentLength;
+		//if (typeof(content) == 'string') {
+		//	(new TextEncoder().encode(content)).length;
+		//}
+		const responseContentLength = content.length;
+		//if (responseContentLength === undefined) {
+		//	responseContentLength = 0;
+		//}
+		res.writeHead(statusCode, {
 			"Access-Control-Allow-Origin": "*",
 			"Content-Type": contentType,
 			"Content-Length": responseContentLength,

apps/cic-meta/tests/server.ts

@@ -7,6 +7,8 @@ import * as handlers from '../scripts/server/handlers';
 import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer } from '@cicnet/crdt-meta';
 import { SqliteAdapter } from '../src/db';
 
+const hashOfFoo = '2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae';
+
 function createKeystore() {
 	const pksa = fs.readFileSync(__dirname + '/privatekeys.asc', 'utf-8');
 	const pubksa = fs.readFileSync(__dirname + '/publickeys.asc', 'utf-8');
@@ -44,11 +46,13 @@ function createDatabase(sqlite_file:string):Promise<any> {
 //				doh(e);
 //				return;
 //			}
+		// get this from real sql files sources
 			const sql = `CREATE TABLE store (
 id integer primary key autoincrement,
-owner_fingerprint text not null,
+owner_fingerprint text default null,
 hash char(64) not null unique,
-content text not null
+content text not null,
+mime_type text default null
 );
 `
 
@@ -111,15 +115,18 @@ describe('server', async () => {
 		let j = env.toJSON();
 		const content = await handlers.handleClientMergePut(j, db, digest, keystore, signer);
 		assert(content); // true-ish
+		console.debug('content', content);
 
 		let v = await handlers.handleNoMergeGet(db, digest, keystore);
-		if (v === undefined) {
+		if (v === false) {
 			db.close();
 			assert.fail('');
 		}
+		db.close();
+		return;
 
 		v = await handlers.handleClientMergeGet(db, digest, keystore);
-		if (v === undefined) {
+		if (v === false) {
 			db.close();
 			assert.fail('');
 		}
@@ -187,7 +194,7 @@ describe('server', async () => {
 		j = await handlers.handleNoMergeGet(db, digest, keystore);
 		assert(v); // true-ish
 
-		let o = JSON.parse(j);
+		let o = JSON.parse(j[0]);
 		o.bar = 'xyzzy';
 		j = JSON.stringify(o);
 
@@ -212,82 +219,39 @@ describe('server', async () => {
 
 		j = await handlers.handleNoMergeGet(db, digest, keystore);
 		assert(j); // true-ish
-		o = JSON.parse(j);
+		o = JSON.parse(j[0]);
 		console.log(o);
 
 		db.close();
 	});
 
-	await it('server_merge', async () => {
-		const keystore = await createKeystore();
-		const signer = new PGPSigner(keystore);
-
-		const db = await createDatabase(__dirname + '/db.three.sqlite');
-
-		const digest = 'deadbeef';
-		let s = new Syncable(digest, {
-			bar: 'baz',
-		});
-		let env = await wrap(s, signer)
-		let j:any = env.toJSON();
-
-		let v = await handlers.handleClientMergePut(j, db, digest, keystore, signer);
-		assert(v); // true-ish
-
-		j = await handlers.handleNoMergeGet(db, digest, keystore);
-		assert(v); // true-ish
-
-		let o = JSON.parse(j);
-		o.bar = 'xyzzy';
-		j = JSON.stringify(o);
-
-		let signMaterial = await handlers.handleServerMergePost(j, db, digest, keystore, signer);
-		assert(signMaterial)
-
-		env = Envelope.fromJSON(signMaterial);
-
-		console.log('envvvv', env);
-
-		const signedData = await signData(env.o['digest'], keystore);
-		console.log('signed', signedData);
-
-		o = {
-			'm': env,
-			's': signedData,
-		}
-		j = JSON.stringify(o);
-		console.log(j);
-
-		v = await handlers.handleServerMergePut(j, db, digest, keystore, signer);
-		assert(v);
-
-		j = await handlers.handleNoMergeGet(db, digest, keystore);
-		assert(j); // true-ish
-		o = JSON.parse(j);
-		console.log(o);
-
-		db.close();
-	});
-
-
-
-//	await it('server_merge_empty', async () => {
+//	await it('server_merge', async () => {
 //		const keystore = await createKeystore();
 //		const signer = new PGPSigner(keystore);
 //
 //		const db = await createDatabase(__dirname + '/db.three.sqlite');
 //
-//		const digest = '0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef';
-//		let o:any = {
-//			foo: 'bar',
-//			xyzzy: 42,
-//		}
-//		let j:any = JSON.stringify(o);
+//		const digest = 'deadbeef';
+//		let s = new Syncable(digest, {
+//			bar: 'baz',
+//		});
+//		let env = await wrap(s, signer)
+//		let j:any = env.toJSON();
+//
+//		let v = await handlers.handleClientMergePut(j, db, digest, keystore, signer);
+//		assert(v); // true-ish
+//
+//		j = await handlers.handleNoMergeGet(db, digest, keystore);
+//		assert(v); // true-ish
+//
+//		let o = JSON.parse(j);
+//		o.bar = 'xyzzy';
+//		j = JSON.stringify(o);
 //
 //		let signMaterial = await handlers.handleServerMergePost(j, db, digest, keystore, signer);
 //		assert(signMaterial)
 //
-//		const env = Envelope.fromJSON(signMaterial);
+//		env = Envelope.fromJSON(signMaterial);
 //
 //		console.log('envvvv', env);
 //
@@ -301,7 +265,7 @@ describe('server', async () => {
 //		j = JSON.stringify(o);
 //		console.log(j);
 //
-//		let v = await handlers.handleServerMergePut(j, db, digest, keystore, signer);
+//		v = await handlers.handleServerMergePut(j, db, digest, keystore, signer);
 //		assert(v);
 //
 //		j = await handlers.handleNoMergeGet(db, digest, keystore);
@@ -311,5 +275,88 @@ describe('server', async () => {
 //
 //		db.close();
 //	});
+//
+
+
+	await it('server_merge_empty', async () => {
+		const keystore = await createKeystore();
+		const signer = new PGPSigner(keystore);
+
+		const db = await createDatabase(__dirname + '/db.three.sqlite');
+
+		const digest = '0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef';
+		let o:any = {
+			foo: 'bar',
+			xyzzy: 42,
+		}
+		let j:any = JSON.stringify(o);
+
+		let signMaterial = await handlers.handleServerMergePost(j, db, digest, keystore, signer);
+		assert(signMaterial)
+
+		const env = Envelope.fromJSON(signMaterial);
+
+		console.log('envvvv', env);
+
+		const signedData = await signData(env.o['digest'], keystore);
+		console.log('signed', signedData);
+
+		o = {
+			'm': env,
+			's': signedData,
+		}
+		j = JSON.stringify(o);
+		console.log(j);
+
+		let v = await handlers.handleServerMergePut(j, db, digest, keystore, signer);
+		assert(v);
+
+		j = await handlers.handleNoMergeGet(db, digest, keystore);
+		assert(j); // true-ish
+		o = JSON.parse(j[0]);
+		console.log(o);
+
+		db.close();
+	});
+
+	await it('immutable_nodigest', async() => {
+		const keystore = await createKeystore();
+		const db = await createDatabase(__dirname + '/db.three.sqlite');
+
+		const s:string = 'foo';
+		let r;
+		r = await handlers.handleImmutablePost(s, db, undefined, keystore, 'text/plain');
+		assert(r[0]);
+		assert(hashOfFoo == r[1]);
+
+		r = await handlers.handleImmutablePost(s, db, undefined, keystore, 'text/plain');
+		assert(!r[0]);
+		assert(hashOfFoo == r[1]);
+
+		const b:Uint8Array = new TextEncoder().encode(s);
+		r = await handlers.handleImmutablePost(b, db, undefined, keystore, 'text/plain');
+		assert(!r[0]);
+		assert(hashOfFoo == r[1]);
+	});
+
+	await it('immutable_digest', async() => {
+		const keystore = await createKeystore();
+		const db = await createDatabase(__dirname + '/db.three.sqlite');
+
+		const s:string = 'foo';
+		const b:Uint8Array = new TextEncoder().encode(s);
+		let r;
+		r = await handlers.handleImmutablePost(b, db, hashOfFoo, keystore, 'application/octet-stream');
+		assert(r[0]);
+		assert(hashOfFoo == r[1]);
+
+		r = await handlers.handleImmutablePost(b, db, hashOfFoo, keystore, 'application/octet-stream');
+		assert(!r[0]);
+		assert(hashOfFoo == r[1]);
+
+		r = await handlers.handleImmutablePost(s, db, hashOfFoo, keystore, 'text/plain');
+		assert(!r[0]);
+		assert(hashOfFoo == r[1]);
+	});
 });
 

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.10')
+version = (0, 4, 0, 'alpha.11')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/docker/Dockerfile

@@ -1,22 +1,28 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 
 #RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+RUN apt-get install libffi-dev -y
+
+
+ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
+ARG EXTRA_PIP_ARGS=""
+ARG PIP_INDEX_URL=https://pypi.org/simple
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 COPY requirements.txt .
 
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt
 
 COPY . .
 RUN python setup.py install
 
-COPY docker/*.sh .
-RUN chmod +x *.sh 
+COPY docker/*.sh ./
+RUN chmod +x /root/*.sh
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables

apps/cic-notify/requirements.txt

@@ -1,7 +1,8 @@
-confini~=0.4.1a1
+confini>=0.3.6rc4,<0.5.0
 africastalking==1.2.3
 SQLAlchemy==1.3.20
 alembic==1.4.2
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
+semver==2.13.0

apps/cic-signer/Dockerfile

@@ -0,0 +1,22 @@
+ARG DOCKER_REGISTRY=registry.gitlab.com/grassrootseconomics
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2 as dev
+
+WORKDIR /root
+
+RUN apt-get install libffi-dev -y
+
+COPY requirements.txt . 
+
+ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_ARGS=""
+ARG PIP_INDEX_URL="https://pypi.org/simple"
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+    -r requirements.txt
+
+COPY . .
+
+#RUN chmod +x *.sh

apps/cic-signer/requirements.txt

@@ -0,0 +1 @@
+funga-eth[sql]>=0.5.1a1,<0.6.0

apps/cic-ussd/cic_ussd/account/balance.py

@@ -2,18 +2,19 @@
 
 import json
 import logging
-from typing import Optional
+from typing import Union, Optional
 
 # third-party imports
 from cic_eth.api import Api
 from cic_eth_aux.erc20_demurrage_token.api import Api as DemurrageApi
+from cic_types.condiments import MetadataPointer
 
 # local imports
 from cic_ussd.account.transaction import from_wei
 from cic_ussd.cache import cache_data_key, get_cached_data
 from cic_ussd.error import CachedDataNotFoundError
 
-logg = logging.getLogger()
+logg = logging.getLogger(__file__)
 
 
 def get_balances(address: str,
@@ -42,7 +43,7 @@ def get_balances(address: str,
     :return: A list containing balance data if called synchronously. | None
     :rtype: list | None
     """
-    logg.debug(f'retrieving balance for address: {address}')
+    logg.debug(f'retrieving {token_symbol} balance for address: {address}')
     if asynchronous:
         cic_eth_api = Api(
             chain_str=chain_str,
@@ -59,11 +60,13 @@ def get_balances(address: str,
         return balance_request_task.get()
 
 
-def calculate_available_balance(balances: dict) -> float:
+def calculate_available_balance(balances: dict, decimals: int) -> float:
     """This function calculates an account's balance at a specific point in time by computing the difference from the
     outgoing balance and the sum of the incoming and network balances.
     :param balances: incoming, network and outgoing balances.
     :type balances: dict
+    :param decimals:
+    :type decimals: int
     :return: Token value of the available balance.
     :rtype: float
     """
@@ -72,7 +75,7 @@ def calculate_available_balance(balances: dict) -> float:
     network_balance = balances.get('balance_network')
 
     available_balance = (network_balance + incoming_balance) - outgoing_balance
-    return from_wei(value=available_balance)
+    return from_wei(decimals=decimals, value=available_balance)
 
 
 def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_symbol: str):
@@ -93,29 +96,49 @@ def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_sym
     return demurrage_api.get_adjusted_balance(token_symbol, balance, timestamp).result
 
 
-def get_cached_available_balance(blockchain_address: str) -> float:
+def get_cached_available_balance(decimals: int, identifier: Union[list, bytes]) -> float:
     """This function attempts to retrieve balance data from the redis cache.
-    :param blockchain_address: Ethereum address of an account.
-    :type blockchain_address: str
+    :param decimals:
+    :type decimals: int
+    :param identifier: An identifier needed to create a unique pointer to a balances resource.
+    :type identifier: bytes | list
     :raises CachedDataNotFoundError: No cached balance data could be found.
     :return: Operational balance of an account.
     :rtype: float
     """
-    identifier = bytes.fromhex(blockchain_address)
-    key = cache_data_key(identifier, salt=':cic.balances')
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.BALANCES)
     cached_balances = get_cached_data(key=key)
     if cached_balances:
-        return calculate_available_balance(json.loads(cached_balances))
+        return calculate_available_balance(balances=json.loads(cached_balances), decimals=decimals)
     else:
-        raise CachedDataNotFoundError(f'No cached available balance for address: {blockchain_address}')
+        raise CachedDataNotFoundError(f'No cached available balance at {key}')
 
 
-def get_cached_adjusted_balance(identifier: bytes):
+def get_cached_adjusted_balance(identifier: Union[list, bytes]):
     """
     :param identifier:
     :type identifier:
     :return:
     :rtype:
     """
-    key = cache_data_key(identifier, ':cic.adjusted_balance')
+    key = cache_data_key(identifier, MetadataPointer.BALANCES_ADJUSTED)
     return get_cached_data(key)
+
+
+def get_account_tokens_balance(blockchain_address: str, chain_str: str, token_symbols_list: list):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param chain_str:
+    :type chain_str:
+    :param token_symbols_list:
+    :type token_symbols_list:
+    :return:
+    :rtype:
+    """
+    for token_symbol in token_symbols_list:
+        get_balances(address=blockchain_address,
+                     chain_str=chain_str,
+                     token_symbol=token_symbol,
+                     asynchronous=True,
+                     callback_param=f'{blockchain_address},{token_symbol}')

apps/cic-ussd/cic_ussd/account/statement.py

@@ -7,6 +7,7 @@ from typing import Optional
 import celery
 from chainlib.hash import strip_0x
 from cic_eth.api import Api
+from cic_types.condiments import MetadataPointer
 
 # local import
 from cic_ussd.account.chain import Chain
@@ -53,7 +54,7 @@ def get_cached_statement(blockchain_address: str) -> bytes:
     :rtype: str
     """
     identifier = bytes.fromhex(strip_0x(blockchain_address))
-    key = cache_data_key(identifier=identifier, salt=':cic.statement')
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.STATEMENT)
     return get_cached_data(key=key)
 
 
@@ -68,7 +69,8 @@ def parse_statement_transactions(statement: list):
     parsed_transactions = []
     for transaction in statement:
         action_tag = transaction.get('action_tag')
-        amount = from_wei(transaction.get('token_value'))
+        decimals = transaction.get('token_decimals')
+        amount = from_wei(decimals, transaction.get('token_value'))
         direction_tag = transaction.get('direction_tag')
         token_symbol = transaction.get('token_symbol')
         metadata_id = transaction.get('metadata_id')

apps/cic-ussd/cic_ussd/account/tokens.py

@@ -1,18 +1,115 @@
 # standard imports
+import hashlib
 import json
 import logging
-from typing import Dict, Optional
+from typing import Optional, Union
 
 # external imports
 from cic_eth.api import Api
+from cic_types.condiments import MetadataPointer
 
 # local imports
+from cic_ussd.account.balance import get_cached_available_balance
 from cic_ussd.account.chain import Chain
-from cic_ussd.cache import cache_data_key, get_cached_data
-from cic_ussd.error import SeppukuError
+from cic_ussd.cache import cache_data, cache_data_key, get_cached_data
+from cic_ussd.error import CachedDataNotFoundError, SeppukuError
+from cic_ussd.metadata.tokens import query_token_info, query_token_metadata
+from cic_ussd.processor.util import wait_for_cache
+from cic_ussd.translation import translation_for
+
+logg = logging.getLogger(__file__)
 
 
-logg = logging.getLogger(__name__)
+def collate_token_metadata(token_info: dict, token_metadata: dict) -> dict:
+    """
+    :param token_info:
+    :type token_info:
+    :param token_metadata:
+    :type token_metadata:
+    :return:
+    :rtype:
+    """
+    logg.debug(f'Collating token info: {token_info} and token metadata: {token_metadata}')
+    description = token_info.get('description')
+    issuer = token_info.get('issuer')
+    location = token_metadata.get('location')
+    contact = token_metadata.get('contact')
+    return {
+        'description': description,
+        'issuer': issuer,
+        'location': location,
+        'contact': contact
+    }
+
+
+def create_account_tokens_list(blockchain_address: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    token_symbols_list = get_cached_token_symbol_list(blockchain_address=blockchain_address)
+    token_list_entries = []
+    if token_symbols_list:
+        logg.debug(f'Token symbols: {token_symbols_list} for account: {blockchain_address}')
+        for token_symbol in token_symbols_list:
+            entry = {}
+            logg.debug(f'Processing token data for: {token_symbol}')
+            key = cache_data_key([bytes.fromhex(blockchain_address), token_symbol.encode('utf-8')], MetadataPointer.TOKEN_DATA)
+            token_data = get_cached_data(key)
+            token_data = json.loads(token_data)
+            logg.debug(f'Retrieved token data: {token_data} for: {token_symbol}')
+            token_name = token_data.get('name')
+            entry['name'] = token_name
+            token_symbol = token_data.get('symbol')
+            entry['symbol'] = token_symbol
+            token_issuer = token_data.get('issuer')
+            entry['issuer'] = token_issuer
+            token_contact = token_data['contact'].get('phone')
+            entry['contact'] = token_contact
+            token_location = token_data.get('location')
+            entry['location'] = token_location
+            decimals = token_data.get('decimals')
+            identifier = [bytes.fromhex(blockchain_address), token_symbol.encode('utf-8')]
+            wait_for_cache(identifier, f'Cached available balance for token: {token_symbol}', MetadataPointer.BALANCES)
+            token_balance = get_cached_available_balance(decimals=decimals, identifier=identifier)
+            entry['balance'] = token_balance
+            token_list_entries.append(entry)
+    account_tokens_list = order_account_tokens_list(token_list_entries, bytes.fromhex(blockchain_address))
+    key = cache_data_key(bytes.fromhex(blockchain_address), MetadataPointer.TOKEN_DATA_LIST)
+    cache_data(key, json.dumps(account_tokens_list))
+
+
+def get_active_token_symbol(blockchain_address: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    identifier = bytes.fromhex(blockchain_address)
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_ACTIVE)
+    active_token_symbol = get_cached_data(key)
+    if not active_token_symbol:
+        raise CachedDataNotFoundError('No active token set.')
+    return active_token_symbol
+
+
+def get_cached_token_data(blockchain_address: str, token_symbol: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    identifier = [bytes.fromhex(blockchain_address), token_symbol.encode('utf-8')]
+    key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA)
+    logg.debug(f'Retrieving token data for: {token_symbol} at: {key}')
+    token_data = get_cached_data(key)
+    return json.loads(token_data)
 
 
 def get_cached_default_token(chain_str: str) -> Optional[str]:
@@ -23,7 +120,7 @@ def get_cached_default_token(chain_str: str) -> Optional[str]:
     :rtype:
     """
     logg.debug(f'Retrieving default token from cache for chain: {chain_str}')
-    key = cache_data_key(identifier=chain_str.encode('utf-8'), salt=':cic.default_token_data')
+    key = cache_data_key(identifier=chain_str.encode('utf-8'), salt=MetadataPointer.TOKEN_DEFAULT)
     return get_cached_data(key=key)
 
 
@@ -48,6 +145,132 @@ def get_default_token_symbol():
             raise SeppukuError(f'Could not retrieve default token for: {chain_str}')
 
 
+def get_cached_token_symbol_list(blockchain_address: str) -> Optional[list]:
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    key = cache_data_key(identifier=bytes.fromhex(blockchain_address), salt=MetadataPointer.TOKEN_SYMBOLS_LIST)
+    token_symbols_list = get_cached_data(key)
+    if token_symbols_list:
+        return json.loads(token_symbols_list)
+    return token_symbols_list
+
+
+def get_cached_token_data_list(blockchain_address: str) -> Optional[list]:
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    key = cache_data_key(bytes.fromhex(blockchain_address), MetadataPointer.TOKEN_DATA_LIST)
+    token_data_list = get_cached_data(key)
+    if token_data_list:
+        return json.loads(token_data_list)
+    return token_data_list
+
+
+def handle_token_symbol_list(blockchain_address: str, token_symbol: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    token_symbol_list = get_cached_token_symbol_list(blockchain_address)
+    if token_symbol_list:
+        if token_symbol not in token_symbol_list:
+            token_symbol_list.append(token_symbol)
+    else:
+        token_symbol_list = [token_symbol]
+
+    identifier = bytes.fromhex(blockchain_address)
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_SYMBOLS_LIST)
+    data = json.dumps(token_symbol_list)
+    cache_data(key, data)
+
+
+def hashed_token_proof(token_proof: Union[dict, str]) -> str:
+    """
+    :param token_proof:
+    :type token_proof:
+    :return:
+    :rtype:
+    """
+    if isinstance(token_proof, dict):
+        token_proof = json.dumps(token_proof)
+    logg.debug(f'Hashing token proof: {token_proof}')
+    hash_object = hashlib.new("sha256")
+    hash_object.update(token_proof.encode('utf-8'))
+    return hash_object.digest().hex()
+
+
+def order_account_tokens_list(account_tokens_list: list, identifier: bytes) -> list:
+    """
+    :param account_tokens_list:
+    :type account_tokens_list:
+    :param identifier:
+    :type identifier:
+    :return:
+    :rtype:
+    """
+    ordered_tokens_list = []
+    # get last sent token
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_LAST_SENT)
+    last_sent_token_symbol = get_cached_data(key)
+
+    # get last received token
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_LAST_RECEIVED)
+    last_received_token_symbol = get_cached_data(key)
+
+    last_sent_token_data, remaining_accounts_token_list = remove_from_account_tokens_list(account_tokens_list, last_sent_token_symbol)
+    if last_sent_token_data:
+        ordered_tokens_list.append(last_sent_token_data[0])
+
+    last_received_token_data, remaining_accounts_token_list = remove_from_account_tokens_list(remaining_accounts_token_list, last_received_token_symbol)
+    if last_received_token_data:
+        ordered_tokens_list.append(last_received_token_data[0])
+
+    # order the by balance
+    ordered_by_balance = sorted(remaining_accounts_token_list, key=lambda d: d['balance'], reverse=True)
+    return ordered_tokens_list + ordered_by_balance
+
+
+def parse_token_list(account_token_list: list):
+    parsed_token_list = []
+    for i in range(len(account_token_list)):
+        token_symbol = account_token_list[i].get('symbol')
+        token_balance = account_token_list[i].get('balance')
+        token_data_repr = f'{i+1}. {token_symbol} {token_balance}'
+        parsed_token_list.append(token_data_repr)
+    return parsed_token_list
+
+
+def process_token_data(blockchain_address: str, token_symbol: str):
+    """
+
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    logg.debug(f'Processing token data for token: {token_symbol}')
+    identifier = token_symbol.encode('utf-8')
+    query_token_metadata(identifier=identifier)
+    token_info = query_token_info(identifier=identifier)
+    hashed_token_info = hashed_token_proof(token_proof=token_info)
+    query_token_data(blockchain_address=blockchain_address,
+                     hashed_proofs=[hashed_token_info],
+                     token_symbols=[token_symbol])
+
+
 def query_default_token(chain_str: str):
     """This function synchronously queries cic-eth for the deployed system's default token.
     :param chain_str: Chain name and network id.
@@ -59,3 +282,60 @@ def query_default_token(chain_str: str):
     cic_eth_api = Api(chain_str=chain_str)
     default_token_request_task = cic_eth_api.default_token()
     return default_token_request_task.get()
+
+
+def query_token_data(blockchain_address: str, hashed_proofs: list, token_symbols: list):
+    """"""
+    logg.debug(f'Retrieving token metadata for tokens: {", ".join(token_symbols)}')
+    api = Api(callback_param=blockchain_address,
+              callback_queue='cic-ussd',
+              chain_str=Chain.spec.__str__(),
+              callback_task='cic_ussd.tasks.callback_handler.token_data_callback')
+    api.tokens(token_symbols=token_symbols, proof=hashed_proofs)
+
+
+def remove_from_account_tokens_list(account_tokens_list: list, token_symbol: str):
+    """
+    :param account_tokens_list:
+    :type account_tokens_list:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    removed_token_data = []
+    for i in range(len(account_tokens_list)):
+        if account_tokens_list[i]['symbol'] == token_symbol:
+            removed_token_data.append(account_tokens_list[i])
+            del account_tokens_list[i]
+            break
+    return removed_token_data, account_tokens_list
+
+
+def set_active_token(blockchain_address: str, token_symbol: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    logg.info(f'Active token set to: {token_symbol}')
+    key = cache_data_key(identifier=bytes.fromhex(blockchain_address), salt=MetadataPointer.TOKEN_ACTIVE)
+    cache_data(key=key, data=token_symbol)
+
+
+def token_list_set(preferred_language: str, token_data_reprs: list):
+    """
+    :param preferred_language:
+    :type preferred_language:
+    :param token_data_reprs:
+    :type token_data_reprs:
+    :return:
+    :rtype:
+    """
+    if not token_data_reprs:
+        return translation_for('helpers.no_tokens_list', preferred_language)
+    return ''.join(f'{token_data_repr}\n' for token_data_repr in token_data_reprs)
+

apps/cic-ussd/cic_ussd/account/transaction.py

@@ -1,7 +1,6 @@
 # standard import
-import decimal
-import json
 import logging
+from math import trunc
 from typing import Dict, Tuple
 
 # external import
@@ -9,8 +8,6 @@ from cic_eth.api import Api
 from sqlalchemy.orm.session import Session
 
 # local import
-from cic_ussd.account.chain import Chain
-from cic_ussd.account.tokens import get_cached_default_token
 from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.error import UnknownUssdRecipient
@@ -55,32 +52,32 @@ def aux_transaction_data(preferred_language: str, transaction: dict) -> dict:
     return transaction
 
 
-def from_wei(value: int) -> float:
+def from_wei(decimals: int, value: int) -> float:
     """This function converts values in Wei to a token in the cic network.
+    :param decimals: The decimals required for wei values.
+    :type decimals: int
     :param value: Value in Wei
     :type value: int
     :return: SRF equivalent of value in Wei
     :rtype: float
     """
-    cached_token_data = json.loads(get_cached_default_token(Chain.spec.__str__()))
-    token_decimals: int = cached_token_data.get('decimals')
-    value = float(value) / (10**token_decimals)
+    value = float(value) / (10**decimals)
     return truncate(value=value, decimals=2)
 
 
-def to_wei(value: int) -> int:
+def to_wei(decimals: int, value: int) -> int:
     """This functions converts values from a token in the cic network to Wei.
+    :param decimals: The decimals required for wei values.
+    :type decimals: int
     :param value: Value in SRF
     :type value: int
     :return: Wei equivalent of value in SRF
     :rtype: int
     """
-    cached_token_data = json.loads(get_cached_default_token(Chain.spec.__str__()))
-    token_decimals: int = cached_token_data.get('decimals')
-    return int(value * (10**token_decimals))
+    return int(value * (10**decimals))
 
 
-def truncate(value: float, decimals: int):
+def truncate(value: float, decimals: int) -> float:
     """This function truncates a value to a specified number of decimals places.
     :param value: The value to be truncated.
     :type value: float
@@ -89,9 +86,8 @@ def truncate(value: float, decimals: int):
     :return: The truncated value.
     :rtype: int
     """
-    decimal.getcontext().rounding = decimal.ROUND_DOWN
-    contextualized_value = decimal.Decimal(value)
-    return round(contextualized_value, decimals)
+    stepper = 10.0**decimals
+    return trunc(stepper*value) / stepper
 
 
 def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
@@ -104,14 +100,17 @@ def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
     """
     destination_token_symbol = transaction.get('destination_token_symbol')
     destination_token_value = transaction.get('destination_token_value') or transaction.get('to_value')
+    destination_token_decimals = transaction.get('destination_token_decimals')
     recipient_blockchain_address = transaction.get('recipient')
     sender_blockchain_address = transaction.get('sender')
     source_token_symbol = transaction.get('source_token_symbol')
     source_token_value = transaction.get('source_token_value') or transaction.get('from_value')
+    source_token_decimals = transaction.get('source_token_decimals')
 
     recipient_transaction_data = {
         "token_symbol": destination_token_symbol,
         "token_value": destination_token_value,
+        "token_decimals": destination_token_decimals,
         "blockchain_address": recipient_blockchain_address,
         "role": "recipient",
     }
@@ -119,6 +118,7 @@ def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
         "blockchain_address": sender_blockchain_address,
         "token_symbol": source_token_symbol,
         "token_value": source_token_value,
+        "token_decimals": source_token_decimals,
         "role": "sender",
     }
     return recipient_transaction_data, sender_transaction_data
@@ -166,14 +166,16 @@ class OutgoingTransaction:
         self.from_address = from_address
         self.to_address = to_address
 
-    def transfer(self, amount: int, token_symbol: str):
+    def transfer(self, amount: int, decimals: int, token_symbol: str):
         """This function initiates standard transfers between one account to another
         :param amount: The amount of tokens to be sent
         :type amount: int
+        :param decimals: The decimals for the token being transferred.
+        :type decimals: int
         :param token_symbol: ERC20 token symbol of token to send
         :type token_symbol: str
         """
         self.cic_eth_api.transfer(from_address=self.from_address,
                                   to_address=self.to_address,
-                                  value=to_wei(value=amount),
+                                  value=to_wei(decimals=decimals, value=amount),
                                   token_symbol=token_symbol)

apps/cic-ussd/cic_ussd/cache.py

@@ -1,11 +1,13 @@
 # standard imports
 import hashlib
 import logging
+from typing import Union
 
-# third-party imports
+# external imports
+from cic_types.condiments import MetadataPointer
 from redis import Redis
 
-logg = logging.getLogger()
+logg = logging.getLogger(__file__)
 
 
 class Cache:
@@ -38,7 +40,7 @@ def get_cached_data(key: str):
     return cache.get(name=key)
 
 
-def cache_data_key(identifier: bytes, salt: str):
+def cache_data_key(identifier: Union[list, bytes], salt: MetadataPointer):
     """
     :param identifier:
     :type identifier:
@@ -48,6 +50,10 @@ def cache_data_key(identifier: bytes, salt: str):
     :rtype:
     """
     hash_object = hashlib.new("sha256")
-    hash_object.update(identifier)
-    hash_object.update(salt.encode(encoding="utf-8"))
+    if isinstance(identifier, list):
+        for identity in identifier:
+            hash_object.update(identity)
+    else:
+        hash_object.update(identifier)
+    hash_object.update(salt.value.encode(encoding="utf-8"))
     return hash_object.digest().hex()

apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py

@@ -24,6 +24,8 @@ def upgrade():
                     sa.Column('preferred_language', sa.String(), nullable=True),
                     sa.Column('password_hash', sa.String(), nullable=True),
                     sa.Column('failed_pin_attempts', sa.Integer(), nullable=False),
+                    sa.Column('guardians', sa.String(), nullable=True),
+                    sa.Column('guardian_quora', sa.Integer(), nullable=False),
                     sa.Column('status', sa.Integer(), nullable=False),
                     sa.Column('created', sa.DateTime(), nullable=False),
                     sa.Column('updated', sa.DateTime(), nullable=False),

apps/cic-ussd/cic_ussd/db/models/account.py

@@ -3,6 +3,9 @@ import json
 
 # external imports
 from cic_eth.api import Api
+from cic_types.condiments import MetadataPointer
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.orm.session import Session
 
 # local imports
 from cic_ussd.account.metadata import get_cached_preferred_language, parse_account_metadata
@@ -11,8 +14,9 @@ from cic_ussd.db.enum import AccountStatus
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.encoder import check_password_hash, create_password_hash
-from sqlalchemy import Column, Integer, String
-from sqlalchemy.orm.session import Session
+from cic_ussd.phone_number import Support
+
+support_phone = Support.phone_number
 
 
 class Account(SessionBase):
@@ -28,12 +32,16 @@ class Account(SessionBase):
     failed_pin_attempts = Column(Integer)
     status = Column(Integer)
     preferred_language = Column(String)
+    guardians = Column(String)
+    guardian_quora = Column(Integer)
 
     def __init__(self, blockchain_address, phone_number):
         self.blockchain_address = blockchain_address
         self.phone_number = phone_number
         self.password_hash = None
         self.failed_pin_attempts = 0
+        # self.guardians = f'{support_phone}' if support_phone else None
+        self.guardian_quora = 1
         self.status = AccountStatus.PENDING.value
 
     def __repr__(self):
@@ -44,6 +52,28 @@ class Account(SessionBase):
         self.failed_pin_attempts = 0
         self.status = AccountStatus.ACTIVE.value
 
+    def add_guardian(self, phone_number: str):
+        set_guardians = phone_number
+        if self.guardians:
+            set_guardians = self.guardians.split(',')
+            set_guardians.append(phone_number)
+            ','.join(set_guardians)
+        self.guardians = set_guardians
+
+    def remove_guardian(self, phone_number: str):
+        set_guardians = self.guardians.split(',')
+        set_guardians.remove(phone_number)
+        if len(set_guardians) > 1:
+            self.guardians = ','.join(set_guardians)
+        else:
+            self.guardians = set_guardians[0]
+
+    def get_guardians(self) -> list:
+        return self.guardians.split(',') if self.guardians else []
+
+    def set_guardian_quora(self, quora: int):
+        self.guardian_quora = quora
+
     def create_password(self, password):
         """This method takes a password value and hashes the value before assigning it to the corresponding
         `hashed_password` attribute in the user record.
@@ -109,7 +139,7 @@ class Account(SessionBase):
         :rtype: str
         """
         identifier = bytes.fromhex(self.blockchain_address)
-        key = cache_data_key(identifier, ':cic.person')
+        key = cache_data_key(identifier, MetadataPointer.PERSON)
         account_metadata = get_cached_data(key)
         if not account_metadata:
             return self.phone_number

apps/cic-ussd/cic_ussd/db/ussd_menu.json

@@ -287,7 +287,132 @@
             "display_key": "ussd.kenya.dob_edit_pin_authorization",
             "name": "dob_edit_pin_authorization",
             "parent": "metadata_management"
+        },
+        "49": {
+            "description": "Menu to display first set of tokens in the account's token list.",
+            "display_key": "ussd.kenya.first_account_tokens_set",
+            "name": "first_account_tokens_set",
+            "parent": null
+        },
+        "50": {
+            "description": "Menu to display middle set of tokens in the account's token list.",
+            "display_key": "ussd.kenya.middle_account_tokens_set",
+            "name": "middle_account_tokens_set",
+            "parent": null
+        },
+        "51": {
+            "description": "Menu to display last set of tokens in the account's token list.",
+            "display_key": "ussd.kenya.last_account_tokens_set",
+            "name": "last_account_tokens_set",
+            "parent": null
+        },
+        "52": {
+            "description": "Pin entry menu for setting an active token.",
+            "display_key": "ussd.kenya.token_selection_pin_authorization",
+            "name": "token_selection_pin_authorization",
+            "parent": null
+        },
+        "53": {
+            "description": "Exit following a successful active token setting.",
+            "display_key": "ussd.kenya.exit_successful_token_selection",
+            "name": "exit_successful_token_selection",
+            "parent": null
+        },
+        "54": {
+            "description": "Pin management menu for operations related to an account's pin.",
+            "display_key": "ussd.kenya.pin_management",
+            "name": "pin_management",
+            "parent": "start"
+        },
+        "55": {
+            "description": "Phone number entry for account whose pin is being reset.",
+            "display_key": "ussd.kenya.reset_guarded_pin",
+            "name": "reset_guarded_pin",
+            "parent": "pin_management"
+        },
+        "56": {
+            "description": "Pin entry for initiating request to reset an account's pin.",
+            "display_key": "ussd.kenya.reset_guarded_pin_authorization",
+            "name": "reset_guarded_pin_authorization",
+            "parent": "pin_management"
+        },
+        "57": {
+            "description": "Exit menu following successful pin reset initiation.",
+            "display_key": "ussd.kenya.exit_pin_reset_initiated_success",
+            "name": "exit_pin_reset_initiated_success",
+            "parent": "pin_management"
+        },
+        "58": {
+            "description": "Exit menu in the event that an account is not a set guardian.",
+            "display_key": "ussd.kenya.exit_not_authorized_for_pin_reset",
+            "name": "exit_not_authorized_for_pin_reset",
+            "parent": "pin_management"
+        },
+        "59": {
+            "description": "Pin guard menu for handling guardianship operations.",
+            "display_key": "ussd.kenya.guard_pin",
+            "name": "guard_pin",
+            "parent": "pin_management"
+        },
+        "60": {
+            "description": "Pin entry to display a list of set guardians.",
+            "display_key": "ussd.kenya.guardian_list_pin_authorization",
+            "name": "guardian_list_pin_authorization",
+            "parent": "guard_pin"
+        },
+        "61": {
+            "description": "Menu to display list of set guardians.",
+            "display_key": "ussd.kenya.guardian_list",
+            "name": "guardian_list",
+            "parent": "guard_pin"
+        },
+        "62": {
+            "description": "Phone number entry to add an account as a guardian to reset pin.",
+            "display_key": "ussd.kenya.add_guardian",
+            "name": "add_guardian",
+            "parent": "guard_pin"
+        },
+        "63": {
+            "description": "Pin entry to confirm addition of an account as a guardian.",
+            "display_key": "ussd.kenya.add_guardian_pin_authorization",
+            "name": "add_guardian_pin_authorization",
+            "parent": "guard_pin"
+        },
+        "64": {
+            "description": "Exit menu when an account is successfully added as pin reset guardian.",
+            "display_key": "ussd.kenya.exit_guardian_addition_success",
+            "name": "exit_guardian_addition_success",
+            "parent": "guard_pin"
+        },
+        "65": {
+            "description": "Phone number entry to remove an account as a guardian to reset pin.",
+            "display_key": "ussd.kenya.remove_guardian",
+            "name": "remove_guardian",
+            "parent": "guard_pin"
+        },
+        "66": {
+            "description": "Pin entry to confirm removal of an account as a guardian.",
+            "display_key": "ussd.kenya.remove_guardian_pin_authorization",
+            "name": "remove_guardian_pin_authorization",
+            "parent": "guard_pin"
+        },
+        "67": {
+            "description": "Exit menu when an account is successfully removed as pin reset guardian.",
+            "display_key": "ussd.kenya.exit_guardian_removal_success",
+            "name": "exit_guardian_removal_success",
+            "parent": "guard_pin"
+        },
+        "68": {
+            "description": "Exit menu when invalid phone number entry for guardian addition. ",
+            "display_key": "ussd.kenya.exit_invalid_guardian_addition",
+            "name": "exit_invalid_guardian_addition",
+            "parent": "guard_pin"
+        },
+        "69": {
+            "description": "Exit menu when invalid phone number entry for guardian removal. ",
+            "display_key": "ussd.kenya.exit_invalid_guardian_removal",
+            "name": "exit_invalid_guardian_removal",
+            "parent": "guard_pin"
         }
-
     }
 }

apps/cic-ussd/cic_ussd/metadata/__init__.py

@@ -3,7 +3,6 @@
 # external imports
 
 # local imports
-from .base import Metadata
 from .custom import CustomMetadata
 from .person import PersonMetadata
 from .phone import PhonePointerMetadata

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -1,99 +1,30 @@
 # standard imports
-import json
 import logging
-import os
-from typing import Dict, Union
 
-# third-part imports
-from cic_types.models.person import generate_metadata_pointer, Person
+# external imports
+from cic_types.condiments import MetadataPointer
+from cic_types.ext.metadata import MetadataRequestsHandler
+from cic_types.processor import generate_metadata_pointer
 
 # local imports
 from cic_ussd.cache import cache_data, get_cached_data
-from cic_ussd.http.requests import error_handler, make_request
-from cic_ussd.metadata.signer import Signer
 
 logg = logging.getLogger(__file__)
 
 
-class Metadata:
-    """
-    :cvar base_url: The base url or the metadata server.
-    :type base_url: str
-    """
+class UssdMetadataHandler(MetadataRequestsHandler):
+    def __init__(self, identifier: bytes, cic_type: MetadataPointer = None):
+        super().__init__(cic_type, identifier)
 
-    base_url = None
-
-
-class MetadataRequestsHandler(Metadata):
-
-    def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'):
-        """"""
-        self.cic_type = cic_type
-        self.engine = engine
-        self.headers = {
-            'X-CIC-AUTOMERGE': 'server',
-            'Content-Type': 'application/json'
-        }
-        self.identifier = identifier
-        self.metadata_pointer = generate_metadata_pointer(
-            identifier=self.identifier,
-            cic_type=self.cic_type
-        )
-        if self.base_url:
-            self.url = os.path.join(self.base_url, self.metadata_pointer)
-
-    def create(self, data: Union[Dict, str]):
-        """"""
-        data = json.dumps(data).encode('utf-8')
-        result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
-
-        error_handler(result=result)
-        metadata = result.json()
-        return self.edit(data=metadata)
-
-    def edit(self, data: Union[Dict, str]):
-        """"""
-        cic_meta_signer = Signer()
-        signature = cic_meta_signer.sign_digest(data=data)
-        algorithm = cic_meta_signer.get_operational_key().get('algo')
-        formatted_data = {
-            'm': json.dumps(data),
-            's': {
-                'engine': self.engine,
-                'algo': algorithm,
-                'data': signature,
-                'digest': data.get('digest'),
-            }
-        }
-        formatted_data = json.dumps(formatted_data)
-        result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
-        logg.info(f'signed metadata submission status: {result.status_code}.')
-        error_handler(result=result)
-        try:
-            decoded_identifier = self.identifier.decode("utf-8")
-        except UnicodeDecodeError:
-            decoded_identifier = self.identifier.hex()
-        logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.')
-        return result
-
-    def query(self):
-        """"""
-        result = make_request(method='GET', url=self.url)
-        error_handler(result=result)
-        result_data = result.json()
-        if not isinstance(result_data, dict):
-            raise ValueError(f'Invalid result data object: {result_data}.')
-        if result.status_code == 200:
-            if self.cic_type == ':cic.person':
-                person = Person()
-                person_data = person.deserialize(person_data=result_data)
-                serialized_person_data = person_data.serialize()
-                data = json.dumps(serialized_person_data)
-            else:
-                data = json.dumps(result_data)
-            cache_data(key=self.metadata_pointer, data=data)
-            logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
-        return result_data
+    def cache_metadata(self, data: str):
+        """
+        :param data:
+        :type data:
+        :return:
+        :rtype:
+        """
+        cache_data(self.metadata_pointer, data)
+        logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
 
     def get_cached_metadata(self):
         """"""

apps/cic-ussd/cic_ussd/metadata/custom.py

@@ -1,12 +1,13 @@
 # standard imports
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class CustomMetadata(MetadataRequestsHandler):
+class CustomMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.custom', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.CUSTOM, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -1,12 +1,13 @@
 # standard imports
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PersonMetadata(MetadataRequestsHandler):
+class PersonMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.person', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PERSON, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -2,12 +2,13 @@
 import logging
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PhonePointerMetadata(MetadataRequestsHandler):
+class PhonePointerMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.phone', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PHONE, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -1,13 +1,13 @@
 # standard imports
 
 # external imports
-import celery
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PreferencesMetadata(MetadataRequestsHandler):
+class PreferencesMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.preferences', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PREFERENCES, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/signer.py

@@ -1,60 +0,0 @@
-# standard imports
-import json
-import logging
-from typing import Optional
-from urllib.request import Request, urlopen
-
-# third-party imports
-import gnupg
-
-# local imports
-
-logg = logging.getLogger()
-
-
-class Signer:
-    """
-    :cvar gpg_path:
-    :type gpg_path:
-    :cvar gpg_passphrase:
-    :type gpg_passphrase:
-    :cvar key_file_path:
-    :type key_file_path:
-
-    """
-    gpg_path: str = None
-    gpg_passphrase: str = None
-    key_file_path: str = None
-
-    def __init__(self):
-        self.gpg = gnupg.GPG(gnupghome=self.gpg_path)
-
-        with open(self.key_file_path, 'r') as key_file:
-            self.key_data = key_file.read()
-
-    def get_operational_key(self):
-        """
-        :return:
-        :rtype:
-        """
-        # import key data into keyring
-        self.gpg.import_keys(key_data=self.key_data)
-        gpg_keys = self.gpg.list_keys()
-        key_algorithm = gpg_keys[0].get('algo')
-        key_id = gpg_keys[0].get("keyid")
-        logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}')
-        return gpg_keys[0]
-
-    def sign_digest(self, data: dict):
-        """
-        :param data:
-        :type data:
-        :return:
-        :rtype:
-        """
-        digest = data['digest']
-        key_id = self.get_operational_key().get('keyid')
-        signature = self.gpg.sign(digest, passphrase=self.gpg_passphrase, keyid=key_id)
-        return str(signature)
-
-

apps/cic-ussd/cic_ussd/metadata/tokens.py

@@ -0,0 +1,54 @@
+# standard imports
+from typing import Dict, Optional
+
+# external imports
+import json
+
+from cic_types.condiments import MetadataPointer
+
+# local imports
+from .base import UssdMetadataHandler
+from cic_ussd.cache import cache_data
+from cic_ussd.error import MetadataNotFoundError
+
+
+class TokenMetadata(UssdMetadataHandler):
+    def __init__(self, identifier: bytes, **kwargs):
+        super(TokenMetadata, self).__init__(identifier=identifier, **kwargs)
+
+
+def token_metadata_handler(metadata_client: TokenMetadata) -> Optional[Dict]:
+    """
+    :param metadata_client:
+    :type metadata_client:
+    :return:
+    :rtype:
+    """
+    result = metadata_client.query()
+    token_metadata = result.json()
+    if not token_metadata:
+        raise MetadataNotFoundError(f'No metadata found at: {metadata_client.metadata_pointer} for: {metadata_client.identifier.decode("utf-8")}')
+    cache_data(metadata_client.metadata_pointer, json.dumps(token_metadata))
+    return token_metadata
+
+
+def query_token_metadata(identifier: bytes):
+    """
+    :param identifier:
+    :type identifier:
+    :return:
+    :rtype:
+    """
+    token_metadata_client = TokenMetadata(identifier=identifier, cic_type=MetadataPointer.TOKEN_META_SYMBOL)
+    return token_metadata_handler(token_metadata_client)
+
+
+def query_token_info(identifier: bytes):
+    """
+    :param identifier:
+    :type identifier:
+    :return:
+    :rtype:
+    """
+    token_info_client = TokenMetadata(identifier=identifier, cic_type=MetadataPointer.TOKEN_PROOF_SYMBOL)
+    return token_metadata_handler(token_info_client)

apps/cic-ussd/cic_ussd/phone_number.py

@@ -5,7 +5,6 @@ from typing import Optional
 import phonenumbers
 
 # local imports
-from cic_ussd.db.models.account import Account
 
 
 class E164Format:

apps/cic-ussd/cic_ussd/processor/menu.py

@@ -5,9 +5,11 @@ from datetime import datetime, timedelta
 
 # external imports
 import i18n.config
+from cic_types.condiments import MetadataPointer
 
 # local imports
 from cic_ussd.account.balance import (calculate_available_balance,
+                                      get_account_tokens_balance,
                                       get_adjusted_balance,
                                       get_balances,
                                       get_cached_adjusted_balance,
@@ -20,13 +22,20 @@ from cic_ussd.account.statement import (
     query_statement,
     statement_transaction_set
 )
-from cic_ussd.account.tokens import get_default_token_symbol
+from cic_ussd.account.tokens import (create_account_tokens_list,
+                                     get_active_token_symbol,
+                                     get_cached_token_data,
+                                     get_cached_token_symbol_list,
+                                     get_cached_token_data_list,
+                                     parse_token_list,
+                                     token_list_set)
 from cic_ussd.account.transaction import from_wei, to_wei
 from cic_ussd.cache import cache_data_key, cache_data
 from cic_ussd.db.models.account import Account
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
 from cic_ussd.processor.util import parse_person_metadata
+from cic_ussd.session.ussd_session import save_session_data
 from cic_ussd.translation import translation_for
 from sqlalchemy.orm.session import Session
 
@@ -47,22 +56,27 @@ class MenuProcessor:
         :return:
         :rtype:
         """
-        available_balance = get_cached_available_balance(self.account.blockchain_address)
+
         adjusted_balance = get_cached_adjusted_balance(self.identifier)
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
         with_available_balance = f'{self.display_key}.available_balance'
         with_fees = f'{self.display_key}.with_fees'
+        decimals = token_data.get('decimals')
+        available_balance = get_cached_available_balance(decimals, [self.identifier, token_symbol.encode('utf-8')])
         if not adjusted_balance:
             return translation_for(key=with_available_balance,
                                    preferred_language=preferred_language,
                                    available_balance=available_balance,
                                    token_symbol=token_symbol)
+
         adjusted_balance = json.loads(adjusted_balance)
-        tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
-        tax = from_wei(int(tax_wei))
+
+        tax_wei = to_wei(decimals, int(available_balance)) - int(adjusted_balance)
+        tax = from_wei(decimals, int(tax_wei))
         return translation_for(key=with_fees,
                                preferred_language=preferred_language,
                                available_balance=available_balance,
@@ -75,21 +89,25 @@ class MenuProcessor:
         :rtype:
         """
         cached_statement = get_cached_statement(self.account.blockchain_address)
-        statement = json.loads(cached_statement)
-        statement_transactions = parse_statement_transactions(statement)
-        transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)]
+        transaction_sets = []
+        if cached_statement:
+            statement = json.loads(cached_statement)
+            statement_transactions = parse_statement_transactions(statement)
+            transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)]
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
-        first_transaction_set = []
-        middle_transaction_set = []
-        last_transaction_set = []
+        no_transaction_history = statement_transaction_set(preferred_language, transaction_sets)
+        first_transaction_set = no_transaction_history
+        middle_transaction_set = no_transaction_history
+        last_transaction_set = no_transaction_history
         if transaction_sets:
             first_transaction_set = statement_transaction_set(preferred_language, transaction_sets[0])
         if len(transaction_sets) >= 2:
             middle_transaction_set = statement_transaction_set(preferred_language, transaction_sets[1])
         if len(transaction_sets) >= 3:
             last_transaction_set = statement_transaction_set(preferred_language, transaction_sets[2])
+
         if self.display_key == 'ussd.kenya.first_transaction_set':
             return translation_for(
                 self.display_key, preferred_language, first_transaction_set=first_transaction_set
@@ -103,7 +121,63 @@ class MenuProcessor:
                 self.display_key, preferred_language, last_transaction_set=last_transaction_set
             )
 
-    def help(self):
+    def add_guardian_pin_authorization(self):
+        guardian_information = self.guardian_metadata()
+        return self.pin_authorization(guardian_information=guardian_information)
+
+    def guardian_list(self):
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        set_guardians = self.account.get_guardians()
+        if set_guardians:
+            guardians_list = ''
+            guardians_list_header = translation_for('helpers.guardians_list_header', preferred_language)
+            for phone_number in set_guardians:
+                guardian = Account.get_by_phone_number(phone_number, self.session)
+                guardian_information = guardian.standard_metadata_id()
+                guardians_list += f'{guardian_information}\n'
+            guardians_list = guardians_list_header + '\n' + guardians_list
+        else:
+            guardians_list = translation_for('helpers.no_guardians_list', preferred_language)
+        return translation_for(self.display_key, preferred_language, guardians_list=guardians_list)
+
+    def account_tokens(self) -> str:
+        cached_token_data_list = get_cached_token_data_list(self.account.blockchain_address)
+        token_data_list = parse_token_list(cached_token_data_list)
+        token_list_sets = [token_data_list[tds:tds + 3] for tds in range(0, len(token_data_list), 3)]
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        no_token_list = token_list_set(preferred_language, [])
+        first_account_tokens_set = no_token_list
+        middle_account_tokens_set = no_token_list
+        last_account_tokens_set = no_token_list
+        if token_list_sets:
+            data = {
+                'account_tokens_list': cached_token_data_list
+            }
+            save_session_data(data=data, queue='cic-ussd', session=self.session, ussd_session=self.ussd_session)
+            first_account_tokens_set = token_list_set(preferred_language, token_list_sets[0])
+
+        if len(token_list_sets) >= 2:
+            middle_account_tokens_set = token_list_set(preferred_language, token_list_sets[1])
+        if len(token_list_sets) >= 3:
+            last_account_tokens_set = token_list_set(preferred_language, token_list_sets[2])
+        if self.display_key == 'ussd.kenya.first_account_tokens_set':
+            return translation_for(
+                self.display_key, preferred_language, first_account_tokens_set=first_account_tokens_set
+            )
+        if self.display_key == 'ussd.kenya.middle_account_tokens_set':
+            return translation_for(
+                self.display_key, preferred_language, middle_account_tokens_set=middle_account_tokens_set
+            )
+        if self.display_key == 'ussd.kenya.last_account_tokens_set':
+            return translation_for(
+                self.display_key, preferred_language, last_account_tokens_set=last_account_tokens_set
+            )
+
+    def help(self) -> str:
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
@@ -154,30 +228,48 @@ class MenuProcessor:
             f'{self.display_key}.retry', preferred_language, retry_pin_entry=retry_pin_entry
         )
 
+    def guarded_account_metadata(self):
+        guarded_account_phone_number = self.ussd_session.get('data').get('guarded_account_phone_number')
+        guarded_account = Account.get_by_phone_number(guarded_account_phone_number, self.session)
+        return guarded_account.standard_metadata_id()
+
+    def guardian_metadata(self):
+        guardian_phone_number = self.ussd_session.get('data').get('guardian_phone_number')
+        guardian = Account.get_by_phone_number(guardian_phone_number, self.session)
+        return guardian.standard_metadata_id()
+
+    def reset_guarded_pin_authorization(self):
+        guarded_account_information = self.guarded_account_metadata()
+        return self.pin_authorization(guarded_account_information=guarded_account_information)
+
     def start_menu(self):
         """
         :return:
         :rtype:
         """
         chain_str = Chain.spec.__str__()
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
+        decimals = token_data.get('decimals')
         blockchain_address = self.account.blockchain_address
         balances = get_balances(blockchain_address, chain_str, token_symbol, False)[0]
-        key = cache_data_key(self.identifier, ':cic.balances')
+        key = cache_data_key([self.identifier, token_symbol.encode('utf-8')], MetadataPointer.BALANCES)
         cache_data(key, json.dumps(balances))
-        available_balance = calculate_available_balance(balances)
+        available_balance = calculate_available_balance(balances, decimals)
         now = datetime.now()
         if (now - self.account.created).days >= 30:
             if available_balance <= 0:
                 logg.info(f'Not retrieving adjusted balance, available balance: {available_balance} is insufficient.')
             else:
                 timestamp = int((now - timedelta(30)).timestamp())
-                adjusted_balance = get_adjusted_balance(to_wei(int(available_balance)), chain_str, timestamp, token_symbol)
-                key = cache_data_key(self.identifier, ':cic.adjusted_balance')
+                adjusted_balance = get_adjusted_balance(to_wei(decimals, int(available_balance)), chain_str, timestamp, token_symbol)
+                key = cache_data_key([self.identifier, token_symbol.encode('utf-8')], MetadataPointer.BALANCES_ADJUSTED)
                 cache_data(key, json.dumps(adjusted_balance))
 
         query_statement(blockchain_address)
-
+        token_symbols_list = get_cached_token_symbol_list(blockchain_address)
+        get_account_tokens_balance(blockchain_address, chain_str, token_symbols_list)
+        create_account_tokens_list(blockchain_address)
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
@@ -185,6 +277,20 @@ class MenuProcessor:
             self.display_key, preferred_language, account_balance=available_balance, account_token_name=token_symbol
         )
 
+    def token_selection_pin_authorization(self) -> str:
+        """
+        :return:
+        :rtype:
+        """
+        selected_token = self.ussd_session.get('data').get('selected_token')
+        token_name = selected_token.get('name')
+        token_symbol = selected_token.get('symbol')
+        token_issuer = selected_token.get('issuer')
+        token_contact = selected_token.get('contact')
+        token_location = selected_token.get('location')
+        token_data = f'{token_name} ({token_symbol})\n{token_issuer}\n{token_contact}\n{token_location}\n'
+        return self.pin_authorization(token_data=token_data)
+
     def transaction_pin_authorization(self) -> str:
         """
         :return:
@@ -194,36 +300,81 @@ class MenuProcessor:
         recipient = Account.get_by_phone_number(recipient_phone_number, self.session)
         tx_recipient_information = recipient.standard_metadata_id()
         tx_sender_information = self.account.standard_metadata_id()
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
         user_input = self.ussd_session.get('data').get('transaction_amount')
-        transaction_amount = to_wei(value=int(user_input))
+        decimals = token_data.get('decimals')
+        transaction_amount = to_wei(decimals=decimals, value=int(user_input))
         return self.pin_authorization(
             recipient_information=tx_recipient_information,
-            transaction_amount=from_wei(transaction_amount),
+            transaction_amount=from_wei(decimals, transaction_amount),
             token_symbol=token_symbol,
             sender_information=tx_sender_information
         )
 
+    def exit_guardian_addition_success(self) -> str:
+        guardian_information = self.guardian_metadata()
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,
+                               preferred_language,
+                               guardian_information=guardian_information)
+
+    def exit_guardian_removal_success(self):
+        guardian_information = self.guardian_metadata()
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,
+                               preferred_language,
+                               guardian_information=guardian_information)
+
+    def exit_invalid_guardian_addition(self):
+        failure_reason = self.ussd_session.get('data').get('failure_reason')
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key, preferred_language, error_exit=failure_reason)
+
+    def exit_invalid_guardian_removal(self):
+        failure_reason = self.ussd_session.get('data').get('failure_reason')
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key, preferred_language, error_exit=failure_reason)
+
+    def exit_pin_reset_initiated_success(self):
+        guarded_account_information = self.guarded_account_metadata()
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,
+                               preferred_language,
+                               guarded_account_information=guarded_account_information)
+
     def exit_insufficient_balance(self):
         """
         :return:
         :rtype:
         """
-        available_balance = get_cached_available_balance(self.account.blockchain_address)
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
         session_data = self.ussd_session.get('data')
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
+        decimals = token_data.get('decimals')
+        available_balance = get_cached_available_balance(decimals, [self.identifier, token_symbol.encode('utf-8')])
         transaction_amount = session_data.get('transaction_amount')
-        transaction_amount = to_wei(value=int(transaction_amount))
-        token_symbol = get_default_token_symbol()
+        transaction_amount = to_wei(decimals=decimals, value=int(transaction_amount))
         recipient_phone_number = self.ussd_session.get('data').get('recipient_phone_number')
         recipient = Account.get_by_phone_number(recipient_phone_number, self.session)
         tx_recipient_information = recipient.standard_metadata_id()
         return translation_for(
             self.display_key,
             preferred_language,
-            amount=from_wei(transaction_amount),
+            amount=from_wei(decimals, transaction_amount),
             token_symbol=token_symbol,
             recipient_information=tx_recipient_information,
             token_balance=available_balance
@@ -241,6 +392,14 @@ class MenuProcessor:
             preferred_language = i18n.config.get('fallback')
         return translation_for('ussd.kenya.exit_pin_blocked', preferred_language, support_phone=Support.phone_number)
 
+    def exit_successful_token_selection(self) -> str:
+        selected_token = self.ussd_session.get('data').get('selected_token')
+        token_symbol = selected_token.get('symbol')
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,preferred_language,token_symbol=token_symbol)
+
     def exit_successful_transaction(self):
         """
         :return:
@@ -250,8 +409,10 @@ class MenuProcessor:
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
-        transaction_amount = to_wei(amount)
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
+        decimals = token_data.get('decimals')
+        transaction_amount = to_wei(decimals, amount)
         recipient_phone_number = self.ussd_session.get('data').get('recipient_phone_number')
         recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=self.session)
         tx_recipient_information = recipient.standard_metadata_id()
@@ -259,7 +420,7 @@ class MenuProcessor:
         return translation_for(
             self.display_key,
             preferred_language,
-            transaction_amount=from_wei(transaction_amount),
+            transaction_amount=from_wei(decimals, transaction_amount),
             token_symbol=token_symbol,
             recipient_information=tx_recipient_information,
             sender_information=tx_sender_information
@@ -293,15 +454,42 @@ def response(account: Account, display_key: str, menu_name: str, session: Sessio
     if menu_name == 'transaction_pin_authorization':
         return menu_processor.transaction_pin_authorization()
 
+    if menu_name == 'token_selection_pin_authorization':
+        return menu_processor.token_selection_pin_authorization()
+
     if menu_name == 'exit_insufficient_balance':
         return menu_processor.exit_insufficient_balance()
 
+    if menu_name == 'exit_invalid_guardian_addition':
+        return menu_processor.exit_invalid_guardian_addition()
+
+    if menu_name == 'exit_invalid_guardian_removal':
+        return menu_processor.exit_invalid_guardian_removal()
+
     if menu_name == 'exit_successful_transaction':
         return menu_processor.exit_successful_transaction()
 
+    if menu_name == 'exit_guardian_addition_success':
+        return menu_processor.exit_guardian_addition_success()
+
+    if menu_name == 'exit_guardian_removal_success':
+        return menu_processor.exit_guardian_removal_success()
+
+    if menu_name == 'exit_pin_reset_initiated_success':
+        return menu_processor.exit_pin_reset_initiated_success()
+
     if menu_name == 'account_balances':
         return menu_processor.account_balances()
 
+    if menu_name == 'guardian_list':
+        return menu_processor.guardian_list()
+
+    if menu_name == 'add_guardian_pin_authorization':
+        return menu_processor.add_guardian_pin_authorization()
+
+    if menu_name == 'reset_guarded_pin_authorization':
+        return menu_processor.reset_guarded_pin_authorization()
+
     if 'pin_authorization' in menu_name:
         return menu_processor.pin_authorization()
 
@@ -311,6 +499,9 @@ def response(account: Account, display_key: str, menu_name: str, session: Sessio
     if 'transaction_set' in menu_name:
         return menu_processor.account_statement()
 
+    if 'account_tokens_set' in menu_name:
+        return menu_processor.account_tokens()
+
     if menu_name == 'display_user_metadata':
         return menu_processor.person_metadata()
 
@@ -320,6 +511,9 @@ def response(account: Account, display_key: str, menu_name: str, session: Sessio
     if menu_name == 'exit_pin_blocked':
         return menu_processor.exit_pin_blocked()
 
+    if menu_name == 'exit_successful_token_selection':
+        return menu_processor.exit_successful_token_selection()
+
     preferred_language = get_cached_preferred_language(account.blockchain_address)
 
     return translation_for(display_key, preferred_language)

apps/cic-ussd/cic_ussd/processor/util.py

@@ -1,15 +1,22 @@
 # standard imports
 import datetime
 import json
+import logging
+import time
+from typing import Union
 
 # external imports
+from cic_types.condiments import MetadataPointer
 from cic_types.models.person import get_contact_data_from_vcard
 from tinydb.table import Document
 
 # local imports
+from cic_ussd.cache import cache_data_key, get_cached_data
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.translation import translation_for
 
+logg = logging.getLogger(__file__)
+
 
 def latest_input(user_input: str) -> str:
     """
@@ -76,3 +83,66 @@ def resume_last_ussd_session(last_state: str) -> Document:
     if last_state in non_reusable_states:
         return UssdMenu.find_by_name('start')
     return UssdMenu.find_by_name(last_state)
+
+
+def wait_for_cache(identifier: Union[list, bytes], resource_name: str, salt: MetadataPointer, interval: int = 1, max_retry: int = 5):
+    """
+    :param identifier:
+    :type identifier:
+    :param interval:
+    :type interval:
+    :param resource_name:
+    :type resource_name:
+    :param salt:
+    :type salt:
+    :param max_retry:
+    :type max_retry:
+    :return:
+    :rtype:
+    """
+    key = cache_data_key(identifier=identifier, salt=salt)
+    resource = get_cached_data(key)
+    counter = 0
+    while resource is None:
+        logg.debug(f'Waiting for: {resource_name} at: {key}. Checking after: {interval} ...')
+        time.sleep(interval)
+        counter += 1
+        resource = get_cached_data(key)
+        if resource is not None:
+            logg.debug(f'{resource_name} now available.')
+            break
+        else:
+            if counter == max_retry:
+                logg.debug(f'Could not find: {resource_name} within: {max_retry}')
+                break
+
+
+def wait_for_session_data(resource_name: str, session_data_key: str, ussd_session: dict, interval: int = 1, max_retry: int = 5):
+    """
+    :param interval:
+    :type interval:
+    :param resource_name:
+    :type resource_name:
+    :param session_data_key:
+    :type session_data_key:
+    :param ussd_session:
+    :type ussd_session:
+    :param max_retry:
+    :type max_retry:
+    :return:
+    :rtype:
+    """
+    session_data = ussd_session.get('data').get(session_data_key)
+    counter = 0
+    while session_data is None:
+        logg.debug(f'Waiting for: {resource_name}. Checking after: {interval} ...')
+        time.sleep(interval)
+        counter += 1
+        session_data = ussd_session.get('data').get(session_data_key)
+        if session_data is not None:
+            logg.debug(f'{resource_name} now available.')
+            break
+        else:
+            if counter == max_retry:
+                logg.debug(f'Could not find: {resource_name} within: {max_retry}')
+                break

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -63,7 +63,6 @@ elif ssl == 0:
 else:
     ssl = True
 
-valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
 def main():
 
     # TODO: improve url building 
@@ -79,7 +78,7 @@ def main():
     session = uuid.uuid4().hex
     data = {
             'sessionId': session,
-            'serviceCode': valid_service_codes[0],
+            'serviceCode': config.get('USSD_SERVICE_CODE'),
             'phoneNumber': args.phone,
             'text': "",
         }

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py

@@ -10,14 +10,14 @@ import i18n
 import redis
 from chainlib.chain import ChainSpec
 from confini import Config
+from cic_types.ext.metadata import Metadata
+from cic_types.ext.metadata.signer import Signer
 
 # local imports
 from cic_ussd.account.chain import Chain
 from cic_ussd.cache import Cache
 from cic_ussd.db import dsn_from_config
 from cic_ussd.db.models.base import SessionBase
-from cic_ussd.metadata.signer import Signer
-from cic_ussd.metadata.base import Metadata
 from cic_ussd.phone_number import Support
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.validator import validate_presence
@@ -87,11 +87,8 @@ Signer.key_file_path = key_file_path
 i18n.load_path.append(config.get('LOCALE_PATH'))
 i18n.set('fallback', config.get('LOCALE_FALLBACK'))
 
-chain_spec = ChainSpec(
-    common_name=config.get('CIC_COMMON_NAME'),
-    engine=config.get('CIC_ENGINE'),
-    network_id=config.get('CIC_NETWORK_ID')
-)
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
 
 Chain.spec = chain_spec
 Support.phone_number = config.get('OFFICE_SUPPORT_PHONE')

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -12,6 +12,9 @@ import i18n
 import redis
 from chainlib.chain import ChainSpec
 from confini import Config
+from cic_types.condiments import MetadataPointer
+from cic_types.ext.metadata import Metadata
+from cic_types.ext.metadata.signer import Signer
 
 # local imports
 from cic_ussd.account.chain import Chain
@@ -25,8 +28,6 @@ from cic_ussd.files.local_files import create_local_file_data_stores, json_file_
 from cic_ussd.http.requests import get_request_endpoint, get_request_method
 from cic_ussd.http.responses import with_content_headers
 from cic_ussd.menu.ussd_menu import UssdMenu
-from cic_ussd.metadata.base import Metadata
-from cic_ussd.metadata.signer import Signer
 from cic_ussd.phone_number import process_phone_number, Support, E164Format
 from cic_ussd.processor.ussd import handle_menu_operations
 from cic_ussd.runnable.server_base import exportable_parser, logg
@@ -96,11 +97,7 @@ celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY
 states = json_file_parser(filepath=config.get('MACHINE_STATES'))
 transitions = json_file_parser(filepath=config.get('MACHINE_TRANSITIONS'))
 
-chain_spec = ChainSpec(
-    common_name=config.get('CIC_COMMON_NAME'),
-    engine=config.get('CIC_ENGINE'),
-    network_id=config.get('CIC_NETWORK_ID')
-)
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
 Chain.spec = chain_spec
 UssdStateMachine.states = states
@@ -113,7 +110,7 @@ default_token_data = query_default_token(chain_str)
 
 # cache default token for re-usability
 if default_token_data:
-    cache_key = cache_data_key(chain_str.encode('utf-8'), ':cic.default_token_data')
+    cache_key = cache_data_key(chain_str.encode('utf-8'), MetadataPointer.TOKEN_DEFAULT)
     cache_data(key=cache_key, data=json.dumps(default_token_data))
 else:
     raise InitializationError(f'Default token data for: {chain_str} not found.')

apps/cic-ussd/cic_ussd/session/ussd_session.py

@@ -13,7 +13,7 @@ from cic_ussd.cache import Cache
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.ussd_session import UssdSession as DbUssdSession
 
-logg = logging.getLogger()
+logg = logging.getLogger(__file__)
 
 
 class UssdSession:
@@ -239,11 +239,16 @@ def save_session_data(queue: Optional[str], session: Session, data: dict, ussd_s
     :param ussd_session: A ussd session passed to the state machine.
     :type ussd_session: UssdSession
     """
+    logg.debug(f'Saving: {data} session data to: {ussd_session}')
     cache = Cache.store
     external_session_id = ussd_session.get('external_session_id')
     existing_session_data = ussd_session.get('data')
     if existing_session_data:
-        data = {**existing_session_data, **data}
+        # replace session data entry
+        keys = data.keys()
+        for key in keys:
+            existing_session_data[key] = data[key]
+        data = existing_session_data
     in_redis_ussd_session = cache.get(external_session_id)
     in_redis_ussd_session = json.loads(in_redis_ussd_session)
     create_or_update_session(

apps/cic-ussd/cic_ussd/state_machine/logic/menu.py

@@ -81,6 +81,18 @@ def menu_six_selected(state_machine_data: Tuple[str, dict, Account, Session]) ->
     return user_input == '6'
 
 
+def menu_nine_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
+    """
+    This function checks that user input matches a string with value '6'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '6'
+    :rtype: bool
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    return user_input == '9'
+
+
 def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """
     This function checks that user input matches a string with value '00'
@@ -93,6 +105,30 @@ def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account, Sessio
     return user_input == '00'
 
 
+def menu_eleven_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
+    """
+    This function checks that user input matches a string with value '11'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '99'
+    :rtype: bool
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    return user_input == '11'
+
+
+def menu_twenty_two_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
+    """
+    This function checks that user input matches a string with value '22'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '99'
+    :rtype: bool
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    return user_input == '22'
+
+
 def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """
     This function checks that user input matches a string with value '99'

apps/cic-ussd/cic_ussd/state_machine/logic/pin.py

@@ -3,7 +3,6 @@ user's pin.
 """
 
 # standard imports
-import json
 import logging
 import re
 from typing import Tuple
@@ -16,6 +15,7 @@ from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.enum import AccountStatus
 from cic_ussd.encoder import create_password_hash, check_password_hash
+from cic_ussd.processor.util import wait_for_session_data
 from cic_ussd.session.ussd_session import create_or_update_session, persist_ussd_session
 
 
@@ -31,11 +31,8 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool
     :rtype: bool
     """
     user_input, ussd_session, account, session = state_machine_data
-    pin_is_valid = False
     matcher = r'^\d{4}$'
-    if re.match(matcher, user_input):
-        pin_is_valid = True
-    return pin_is_valid
+    return bool(re.match(matcher, user_input))
 
 
 def is_authorized_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
@@ -68,7 +65,7 @@ def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Accoun
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user, session = state_machine_data
+    user_input, ussd_session, account, session = state_machine_data
     initial_pin = create_password_hash(user_input)
     if ussd_session.get('data'):
         data = ussd_session.get('data')
@@ -97,7 +94,8 @@ def pins_match(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user, session = state_machine_data
+    user_input, ussd_session, account, session = state_machine_data
+    wait_for_session_data('Initial pin', session_data_key='initial_pin', ussd_session=ussd_session)
     initial_pin = ussd_session.get('data').get('initial_pin')
     return check_password_hash(user_input, initial_pin)
 
@@ -107,11 +105,12 @@ def complete_pin_change(state_machine_data: Tuple[str, dict, Account, Session]):
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user, session = state_machine_data
+    user_input, ussd_session, account, session = state_machine_data
     session = SessionBase.bind_session(session=session)
+    wait_for_session_data('Initial pin', session_data_key='initial_pin', ussd_session=ussd_session)
     password_hash = ussd_session.get('data').get('initial_pin')
-    user.password_hash = password_hash
-    session.add(user)
+    account.password_hash = password_hash
+    session.add(account)
     session.flush()
     SessionBase.release_session(session=session)
 
@@ -134,6 +133,6 @@ def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account, Session]) ->
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user, session = state_machine_data
-    is_old_pin = user.verify_password(password=user_input)
+    user_input, ussd_session, account, session = state_machine_data
+    is_old_pin = account.verify_password(password=user_input)
     return is_valid_pin(state_machine_data=state_machine_data) and not is_old_pin

apps/cic-ussd/cic_ussd/state_machine/logic/pin_guard.py

@@ -0,0 +1,200 @@
+# standard imports
+import logging
+from typing import Tuple
+
+# external imports
+import celery
+import i18n
+from phonenumbers.phonenumberutil import NumberParseException
+from sqlalchemy.orm.session import Session
+
+# local imports
+from cic_ussd.account.metadata import get_cached_preferred_language
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.phone_number import process_phone_number, E164Format
+from cic_ussd.session.ussd_session import save_session_data
+from cic_ussd.translation import translation_for
+
+
+logg = logging.getLogger(__file__)
+
+
+def save_guardian_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    session_data = ussd_session.get('data') or {}
+    guardian_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    session_data['guardian_phone_number'] = guardian_phone_number
+    save_session_data('cic-ussd', session, session_data, ussd_session)
+
+
+def save_guarded_account_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    session_data = ussd_session.get('data') or {}
+    guarded_account_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    session_data['guarded_account_phone_number'] = guarded_account_phone_number
+    save_session_data('cic-ussd', session, session_data, ussd_session)
+
+
+def retrieve_person_metadata(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    guardian_phone_number = process_phone_number(user_input, E164Format.region)
+    guardian = Account.get_by_phone_number(guardian_phone_number, session)
+    blockchain_address = guardian.blockchain_address
+    s_query_person_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.query_person_metadata', [blockchain_address], queue='cic-ussd')
+    s_query_person_metadata.apply_async()
+
+
+def is_valid_guardian_addition(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    try:
+        phone_number = process_phone_number(user_input, E164Format.region)
+    except NumberParseException:
+        phone_number = None
+
+    preferred_language = get_cached_preferred_language(account.blockchain_address)
+    if not preferred_language:
+        preferred_language = i18n.config.get('fallback')
+
+    is_valid_account = Account.get_by_phone_number(phone_number, session) is not None
+    is_initiator = phone_number == account.phone_number
+    is_existent_guardian = phone_number in account.get_guardians()
+
+    failure_reason = ''
+    if not is_valid_account:
+        failure_reason = translation_for('helpers.error.no_matching_account', preferred_language)
+
+    if is_initiator:
+        failure_reason = translation_for('helpers.error.is_initiator', preferred_language)
+
+    if is_existent_guardian:
+        failure_reason = translation_for('helpers.error.is_existent_guardian', preferred_language)
+
+    if failure_reason:
+        session_data = ussd_session.get('data') or {}
+        session_data['failure_reason'] = failure_reason
+        save_session_data('cic-ussd', session, session_data, ussd_session)
+
+    return phone_number is not None and is_valid_account and not is_existent_guardian and not is_initiator
+
+
+def add_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    guardian_phone_number = ussd_session.get('data').get('guardian_phone_number')
+    account.add_guardian(guardian_phone_number)
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
+
+
+def is_set_pin_guardian(account: Account, checked_number: str, preferred_language: str, session: Session, ussd_session: dict):
+    """"""
+    failure_reason = ''
+    set_guardians = []
+    if account:
+        set_guardians = account.get_guardians()
+    else:
+        failure_reason = translation_for('helpers.error.no_matching_account', preferred_language)
+
+    is_set_guardian = checked_number in set_guardians
+    is_initiator = checked_number == account.phone_number
+
+    if not is_set_guardian:
+        failure_reason = translation_for('helpers.error.is_not_existent_guardian', preferred_language)
+
+    if is_initiator:
+        failure_reason = translation_for('helpers.error.is_initiator', preferred_language)
+
+    if failure_reason:
+        session_data = ussd_session.get('data') or {}
+        session_data['failure_reason'] = failure_reason
+        save_session_data('cic-ussd', session, session_data, ussd_session)
+
+    return is_set_guardian and not is_initiator
+
+
+def is_dialers_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    user_input, ussd_session, account, session = state_machine_data
+    phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    preferred_language = get_cached_preferred_language(account.blockchain_address)
+    if not preferred_language:
+        preferred_language = i18n.config.get('fallback')
+    return is_set_pin_guardian(account, phone_number, preferred_language, session, ussd_session)
+
+
+def is_others_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    user_input, ussd_session, account, session = state_machine_data
+    preferred_language = get_cached_preferred_language(account.blockchain_address)
+    phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    guarded_account = Account.get_by_phone_number(phone_number, session)
+    if not preferred_language:
+        preferred_language = i18n.config.get('fallback')
+    return is_set_pin_guardian(guarded_account, account.phone_number, preferred_language, session, ussd_session)
+
+
+def remove_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    guardian_phone_number = ussd_session.get('data').get('guardian_phone_number')
+    account.remove_guardian(guardian_phone_number)
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
+
+
+def initiate_pin_reset(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    session_data = ussd_session.get('data')
+    quorum_count = session_data['quorum_count'] if session_data.get('quorum_count') else 0
+    quorum_count += 1
+    session_data['quorum_count'] = quorum_count
+    save_session_data('cic-ussd', session, session_data, ussd_session)
+    guarded_account_phone_number = session_data.get('guarded_account_phone_number')
+    guarded_account = Account.get_by_phone_number(guarded_account_phone_number, session)
+    if quorum_count >= guarded_account.guardian_quora:
+        guarded_account.reset_pin(session)
+        logg.debug(f'Reset initiated for: {guarded_account.phone_number}')
+        session_data['quorum_count'] = 0
+        save_session_data('cic-ussd', session, session_data, ussd_session)