Add missing package file encode

.gitignore

@@ -14,4 +14,3 @@ build/
 **/.venv
 .idea
 **/.vim
-**/*secret.yaml

.gitlab-ci.yml

@@ -1,43 +1,14 @@
 include:
-  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
+  - local: 'ci_templates/.cic-template.yml'
-  # these includes are app specific unit tests
+  - local: 'apps/contract-migration/.gitlab-ci.yml'
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-    #- local: 'apps/contract-migration/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'
-    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build
   - test
-  - deploy
+  - release
-
-image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
-
-variables:
-  DOCKER_BUILDKIT: "1"
-  COMPOSE_DOCKER_CLI_BUILD: "1"
-  CI_DEBUG_TRACE: "true"
-
-before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-
-# runs on protected branches and pushes to repo
-build-push:
-  stage: build
-  tags:
-    - integration
-  #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
-  script:
-    - TAG=latest sh ./scripts/build-push.sh
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-
-deploy-dev:
-  stage: deploy
-  trigger: grassrootseconomics/devops
-  when: manual

README.md

@@ -2,21 +2,25 @@
 
 ## Getting started 
 
-This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
+## Make some keys
 
 ```
-export COMPOSE_DOCKER_CLI_BUILD=1
+docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
-export DOCKER_BUILDKIT=1
 ```
 
-start services, database, redis and local ethereum node
-```
-docker-compose up -d
-```
 
-Run app/contract-migration to deploy contracts
+### Prepare the repo
+
+This is stuff we need to put in  makefile but for now...
+
+File mounts and permisssions need to be set
 ```
-RUN_MASK=3 docker-compose up contract-migration
+chmod -R 755 scripts/initdb apps/cic-meta/scripts/initdb
+````
+
+start cluster
+```
+docker-compose up
 ```
 
 stop cluster
@@ -24,9 +28,9 @@ stop cluster
 docker-compose down
 ```
 
-stop cluster and delete data
+delete data
 ```
-docker-compose down -v --remove-orphans
+docker-compose down -v
 ```
 
 rebuild an images
@@ -34,7 +38,5 @@ rebuild an images
 docker-compose up --build <service_name>
 ```
 
-to delete the buildkit cache
+Deployment variables are writtend to service-configs/.env after everthing is up.
-```
+
-docker builder prune --filter type=exec.cachemount
-```

apps/cic-base-os/Dockerfile

@@ -0,0 +1,34 @@
+# The solc image messes up the alpine environment, so we have to go all over again
+FROM python:3.8.6-slim-buster
+
+LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
+LABEL spdx-license-identifier="GPL-3.0-or-later"
+LABEL description="Base layer for buiding development images for the cic component suite"
+
+RUN apt-get update && \
+	apt-get install -y git gcc g++ libpq-dev && \
+	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
+
+
+RUN echo installing nodejs tooling
+
+COPY ./dev/nvm.sh /root/
+
+# Install nvm with node and npm
+# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
+ENV NVM_DIR /root/.nvm
+ENV NODE_VERSION 15.3.0
+ENV BANCOR_NODE_VERSION 10.16.0
+
+RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
+	&& . $NVM_DIR/nvm.sh \
+	&& nvm install $NODE_VERSION \
+	&& nvm alias default $NODE_VERSION \
+	&& nvm use $NODE_VERSION \
+# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
+# owner of these files is "1001" by default - wtf
+	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
+
+ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
+ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
+

apps/cic-base-os/README.md

@@ -0,0 +1 @@
+## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-cache/.gitlab-ci.yml

@@ -1,17 +1,52 @@
-build-test-cic-cache:
+.cic_cache_variables:
-  stage: test 
+    variables:
-  tags:
+        APP_NAME: cic-cache
-    - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-cache
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-cache:
-  script:
+    extends:
-    - cd apps/cic-cache
+        - .py_build_merge_request
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .cic_cache_variables
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-  allow_failure: true
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-  rules:
+      changes:
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+          - apps/cic-cache/**/*
-    changes:
+      when: always
-        - apps/$APP_NAME/**/*
+
-    when: always
+test-mr-cic-cache:
+    stage: test
+    extends:
+        - .cic_cache_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-cache"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
+
+build-push-cic-cache:
+    extends:
+        - .py_build_push
+        - .cic_cache_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
+        
+

apps/cic-cache/README.md

@@ -1 +0,0 @@
-# CIC-CACHE

apps/cic-cache/cic_cache/cli/rpc.py

@@ -27,11 +27,11 @@ class RPC:
     @staticmethod
     def from_config(config):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
         if config.get('SIGNER_PROVIDER'):
             RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
             RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
-        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc
 

apps/cic-cache/docker/Dockerfile_ci

@@ -0,0 +1,38 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_ARGS=""
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r requirements.txt
+
+COPY . . 
+
+RUN python setup.py install
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-cache/
+
+# for db migrations
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
+RUN chmod 755 ./*.sh
+# Tracker
+# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
+# Server
+# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []

apps/cic-cache/docker/run_tests.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
---extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-

apps/cic-cache/requirements.txt

@@ -9,6 +9,7 @@ psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
 chainsyncer[sql]>=0.0.6a3,<0.1.0
-erc20-faucet>=0.3.2a2, <0.4.0
+erc20-faucet>=0.3.2a1, <0.4.0
-chainlib-eth>=0.0.9a14,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
-eth-address-index>=0.2.3a4,<0.3.0
+chainlib>=0.0.9a3,<0.1.0
+eth-address-index>=0.2.3a1,<0.3.0

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,4 +1,5 @@
 celery==4.4.7
 erc20-demurrage-token~=0.0.3a1
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry~=0.5.8a1
-cic-eth[services]~=0.12.4a8
+chainlib~=0.0.7a1
+cic_eth~=0.12.2a4

apps/cic-eth/.gitlab-ci.yml

@@ -1,16 +1,52 @@
-build-test-cic-eth:
+.cic_eth_variables:
-  stage: test
+    variables:
-  tags:
+        APP_NAME: cic-eth
-    - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-eth
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-eth:
-  script:
+    extends:
-    - cd apps/cic-eth
+        - .cic_eth_variables
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .py_build_target_dev
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-  rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-          - apps/$APP_NAME/**/*
+          - apps/cic-eth/**/*
+      when: always
+
+test-mr-cic-eth:
+    stage: test
+    extends:
+        - .cic_eth_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r admin_requirements.txt
+          -r services_requirements.txt
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-eth"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-eth/**/*
+      when: always
+
+build-push-cic-eth:
+    extends:
+        - .py_build_push
+        - .cic_eth_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-eth/**/*
       when: always

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a3,<0.7.0
+cic-eth-registry>=0.6.1a2,<0.7.0
 hexathon~=0.0.1a8
 chainqueue>=0.0.4a6,<0.1.0
 eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/api/api_task.py

@@ -9,7 +9,6 @@ import logging
 # external imports 
 import celery
 from chainlib.chain import ChainSpec
-from hexathon import strip_0x
 
 # local imports
 from cic_eth.api.base import ApiBase
@@ -255,8 +254,6 @@ class Api(ApiBase):
         :returns: uuid of root task
         :rtype: celery.Task
         """
-        #from_address = strip_0x(from_address)
-        #to_address = strip_0x(to_address)
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [

apps/cic-eth/cic_eth/cli/rpc.py

@@ -35,14 +35,14 @@ class RPC:
     def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label)
         if use_signer:
 
             RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
             RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
             RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
             RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
-        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc
 

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -8,8 +8,8 @@ Create Date: 2021-04-02 18:41:20.864265
 import datetime
 from alembic import op
 import sqlalchemy as sa
+from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth.db.enum import LockEnum
-from cic_eth.encode import ZERO_ADDRESS_NORMAL
 
 
 # revision identifiers, used by Alembic.
@@ -30,7 +30,7 @@ def upgrade():
             sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
             )
     op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
-    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS_NORMAL, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
+    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
 
 
 def downgrade():

apps/cic-eth/cic_eth/eth/account.py

@@ -13,7 +13,7 @@ from chainlib.eth.sign import (
         new_account,
         sign_message,
         )
-from chainlib.eth.address import to_checksum_address, is_address
+from chainlib.eth.address import to_checksum_address
 from chainlib.eth.tx import TxFormat
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
@@ -31,7 +31,6 @@ from cic_eth.eth.gas import (
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
-from cic_eth.encode import tx_normalize
 from cic_eth.error import (
         RoleMissingError,
         SignerError,
@@ -50,7 +49,6 @@ from cic_eth.queue.tx import (
 from cic_eth.encode import (
         unpack_normal,
         ZERO_ADDRESS_NORMAL,
-        tx_normalize,
         )
 
 logg = logging.getLogger()
@@ -86,7 +84,7 @@ def create(self, password, chain_spec_dict):
     # TODO: It seems infeasible that a can be None in any case, verify
     if a == None:
         raise SignerError('create account')
-    a = tx_normalize.wallet_address(a)
+        
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -177,9 +175,6 @@ def gift(self, account_address, chain_spec_dict):
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
-    if is_address(account_address):
-        account_address = tx_normalize.wallet_address(account_address)
-
     logg.debug('gift account address {} to index'.format(account_address))
     queue = self.request.delivery_info.get('routing_key')
 
@@ -253,9 +248,8 @@ def have(self, account, chain_spec_dict):
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):
-    if not is_address(address):
+    if not to_checksum_address(address):
-        raise ValueError('invalid address {}'.format(address))
+        raise ValueError('invalid checksum address {}'.format(address))
-    address = tx_normalize.wallet_address(address)
     session = SessionBase.create_session()
     role = AccountRole.set(tag, address, session=session) 
     session.add(role)
@@ -304,15 +298,13 @@ def cache_gift_data(
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
     tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = Faucet.parse_give_to_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
-    recipient_address = tx_normalize.wallet_address(tx['to'])
 
     session = self.create_session()
 
     tx_dict = {
             'hash': tx['hash'],
-            'from': sender_address,
+            'from': tx['from'],
-            'to': recipient_address,
+            'to': tx['to'],
             'source_token': ZERO_ADDRESS_NORMAL,
             'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
@@ -346,14 +338,12 @@ def cache_account_data(
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
     tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = AccountsIndex.parse_add_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
-    recipient_address = tx_normalize.wallet_address(tx['to'])
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx['hash'],
-            'from': sender_address,
+            'from': tx['from'],
-            'to': recipient_address,
+            'to': tx['to'],
             'source_token': ZERO_ADDRESS_NORMAL,
             'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,

apps/cic-eth/cic_eth/eth/erc20.py

@@ -12,10 +12,7 @@ from chainlib.eth.tx import (
         )
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
-from hexathon import (
+from hexathon import strip_0x
-        strip_0x,
-        add_0x,
-        )
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
 from chainqueue.sql.tx import cache_tx_dict
@@ -41,7 +38,6 @@ from cic_eth.task import (
         CriticalSQLAlchemyAndSignerTask,
     )
 from cic_eth.eth.nonce import CustodialTaskNonceOracle
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -66,8 +62,7 @@ def balance(tokens, holder_address, chain_spec_dict):
 
     for t in tokens:
         address = t['address']
-        logg.debug('address {} {}'.format(address, holder_address))
+        token = ERC20Token(chain_spec, rpc, address)
-        token = ERC20Token(chain_spec, rpc, add_0x(address))
         c = ERC20(chain_spec)
         o = c.balance_of(address, holder_address, sender_address=caller_address)
         r = rpc.do(o)
@@ -376,15 +371,13 @@ def cache_transfer_data(
     tx = unpack(tx_signed_raw_bytes, chain_spec)
 
     tx_data = ERC20.parse_transfer_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_data[0]
-    recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
 
-
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
-            'from': sender_address,
+            'from': tx['from'],
             'to': recipient_address,
             'source_token': tx['to'],
             'destination_token': tx['to'],
@@ -455,14 +448,13 @@ def cache_approve_data(
     tx = unpack(tx_signed_raw_bytes, chain_spec)
 
     tx_data = ERC20.parse_approve_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_data[0]
-    recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
-            'from': sender_address,
+            'from': tx['from'],
             'to': recipient_address,
             'source_token': tx['to'],
             'destination_token': tx['to'],

apps/cic-eth/cic_eth/eth/gas.py

@@ -3,17 +3,10 @@ import logging
 
 # external imports
 import celery
-from hexathon import (
+from hexathon import strip_0x
-        strip_0x,
-        add_0x,
-        )
 #from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
-from chainlib.eth.address import (
+from chainlib.eth.address import is_checksum_address
-        is_checksum_address,
-        to_checksum_address,
-        is_address
-        )
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
 from chainqueue.sql.tx import cache_tx_dict
@@ -78,6 +71,7 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
+#def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 
@@ -182,12 +176,10 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
-    rpc_format_address = None
     if address != None:
-        if not is_address(address):
+        if not is_checksum_address(address):
             raise ValueError('invalid address {}'.format(address))
         address = tx_normalize.wallet_address(address)
-        address = add_0x(address)
 
     tx_hashes = []
     txs = []
@@ -199,6 +191,7 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
         txs.append(tx)
 
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
 
     addresspass = None
     if len(txs) == 0:
@@ -214,15 +207,13 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
                 raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
             addresspass.append(address)
 
-    rpc_format_address = add_0x(to_checksum_address(address))
-
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
 
     gas_balance = 0
     try:
-        o = balance(rpc_format_address)
+        o = balance(address)
         r = conn.do(o)
         conn.disconnect()
         gas_balance = abi_decode_single(ABIContractType.UINT256, r)

apps/cic-eth/cic_eth/eth/meta.py

@@ -2,9 +2,8 @@
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.status import Status as TxStatus
 from cic_eth_registry.erc20 import ERC20Token
-from hexathon import add_0x
 
-# local impor:ts
+# local imports
 from cic_eth.ext.address import translate_address
 
 
@@ -45,8 +44,8 @@ class ExtendedTx:
             destination = source
         if destination_value == None:
             destination_value = source_value
-        st = ERC20Token(self.chain_spec, self.rpc, add_0x(source))
+        st = ERC20Token(self.chain_spec, self.rpc, source)
-        dt = ERC20Token(self.chain_spec, self.rpc, add_0x(destination))
+        dt = ERC20Token(self.chain_spec, self.rpc, destination)
         self.source_token = source
         self.source_token_symbol = st.symbol
         self.source_token_name = st.name

apps/cic-eth/cic_eth/eth/nonce.py

@@ -3,12 +3,11 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.address import is_checksum_address, is_address, strip_0x
+from chainlib.eth.address import is_checksum_address
 
 # local imports
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode import tx_normalize
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.nonce import (
         Nonce,
@@ -43,8 +42,7 @@ class CustodialTaskNonceOracle():
         :returns: Nonce
         :rtype: number
         """
-        address = tx_normalize.wallet_address(self.address)
+        r = NonceReservation.release(self.address, self.uuid, session=self.session)
-        r = NonceReservation.release(address, self.uuid, session=self.session)
         return r[1]
 
 
@@ -60,18 +58,17 @@ def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None):
         address = chained_input
         logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
     else:
-        if is_address(signer_address):
+        if is_checksum_address(signer_address):
             address = signer_address
             logg.debug('explicit address for reserve nonce {}'.format(signer_address))
         else:
             address = AccountRole.get_address(signer_address, session=session)
             logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
 
-    if not is_address(address):
+    if not is_checksum_address(address):
         raise ValueError('invalid result when resolving address for nonce {}'.format(address))
 
     root_id = self.request.root_id
-    address = tx_normalize.wallet_address(address)
     r = NonceReservation.next(address, root_id, session=session)
     logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
 

apps/cic-eth/cic_eth/ext/tx.py

@@ -32,7 +32,6 @@ from potaahto.symbols import snake_and_camel
 from cic_eth.queue.time import tx_times
 from cic_eth.task import BaseTask
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -135,7 +134,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                     tx_address = transfer_data[0]
                     tx_token_value = transfer_data[1]
                     
-                    if tx_normalize.wallet_address(address) == tx_normalize.wallet_address(tx_address):
+                    if address == tx_address:
                         status = StatusEnum.SENT
                         try:
                             o = receipt(tx['hash'])
@@ -153,8 +152,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
-                            'sender': tx_normalize.wallet_address(tx['from']),
+                            'sender': tx['from'],
-                            'recipient': tx_normalize.wallet_address(tx_address),
+                            'recipient': tx_address,
                             'source_value': tx_token_value,
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
@@ -165,12 +164,12 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             tx_r['date_created'] = times['queue']
                         else:
                             tx_r['date_created'] = times['network']
-                        txs[strip_0x(tx['hash'])] = tx_r
+                        txs[tx['hash']] = tx_r
                         break
-
     return txs
 
 
+
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
@@ -231,8 +230,6 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr
             except UnknownContractError:
                 logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
                 continue
-        tx['recipient'] = tx_normalize.wallet_address(tx['recipient'])
-        tx['sender'] = tx_normalize.wallet_address(tx['sender'])
         txs.append(tx)
 
     return txs

apps/cic-eth/cic_eth/queue/query.py

@@ -58,7 +58,6 @@ def get_tx_local(chain_spec, tx_hash, session=None):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
-    address = tx_normalize.wallet_address(address)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     return get_account_tx_local(chain_spec, address, as_sender=as_sender, as_recipient=as_recipient, counterpart=counterpart)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -21,7 +21,6 @@ from erc20_faucet import Faucet
 # local imports
 from .base import SyncFilter
 from cic_eth.eth.meta import ExtendedTx
-from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger().getChild(__name__)
 
@@ -43,9 +42,9 @@ class CallbackFilter(SyncFilter):
             return (None, None)
         r = ERC20.parse_transfer_request(tx.payload)
         transfer_data = {}
-        transfer_data['to'] = tx_normalize.wallet_address(r[0])
+        transfer_data['to'] = r[0]
         transfer_data['value'] = r[1]
-        transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
+        transfer_data['from'] = tx.outputs[0]
         transfer_data['token_address'] = tx.inputs[0]
         return ('transfer', transfer_data)
 
@@ -55,8 +54,8 @@ class CallbackFilter(SyncFilter):
             return (None, None)
         r = ERC20.parse_transfer_from_request(tx.payload)
         transfer_data = {}
-        transfer_data['from'] = tx_normalize.wallet_address(r[0])
+        transfer_data['from'] = r[0]
-        transfer_data['to'] = tx_normalize.wallet_address(r[1])
+        transfer_data['to'] = r[1]
         transfer_data['value'] = r[2]
         transfer_data['token_address'] = tx.inputs[0]
         return ('transferfrom', transfer_data)
@@ -67,9 +66,9 @@ class CallbackFilter(SyncFilter):
             return (None, None)
         r = Faucet.parse_give_to_request(tx.payload)
         transfer_data = {}
-        transfer_data['to'] = tx_normalize.wallet_address(r[0])
+        transfer_data['to'] = r[0]
         transfer_data['value'] = tx.value
-        transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
+        transfer_data['from'] = tx.outputs[0]
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -12,8 +12,7 @@ from hexathon import (
 # local imports
 from .base import SyncFilter
 
-#logg = logging.getLogger(__name__)
+logg = logging.getLogger(__name__)
-logg = logging.getLogger()
 
 account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -17,7 +17,6 @@ from cic_eth_registry import CICRegistry
 from erc20_transfer_authorization import TransferAuthorization
 
 # local imports
-from cic_eth.encode import tx_normalize
 from .base import SyncFilter
 
 
@@ -53,9 +52,9 @@ class TransferAuthFilter(SyncFilter):
 
         r = TransferAuthorization.parse_create_request_request(tx.payload) 
           
-        sender = tx_normalize.wallet_address(r[0])
+        sender = r[0]
-        recipient = tx_normalize.wallet_address(r[1])
+        recipient = r[1]
-        token = tx_normalize.executable_address(r[2])
+        token = r[2]
         value = r[3]
 
         token_data = {

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -69,6 +69,7 @@ from cic_eth.registry import (
         )
 from cic_eth.task import BaseTask
 
+
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
@@ -206,7 +207,7 @@ def main():
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
+    default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address)
     default_token.load(conn)
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name

apps/cic-eth/cic_eth/task.py

@@ -13,6 +13,7 @@ from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.gas import RPCGasOracle
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
+import liveness.linux
 
 # local imports
 from cic_eth.error import SeppukuError
@@ -47,7 +48,6 @@ class BaseTask(celery.Task):
 
     def on_failure(self, exc, task_id, args, kwargs, einfo):
         if isinstance(exc, SeppukuError):
-            import liveness.linux
             liveness.linux.reset(rundir=self.run_dir)
             logg.critical(einfo)
             msg = 'received critical exception {}, calling shutdown'.format(str(exc))

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         4,
-        'alpha.8',
+        'alpha.7',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/doc/texinfo/configuration.texi

@@ -1,6 +1,8 @@
 @node cic-eth configuration
 @section Configuration
 
+(refer to @code{cic-base} for a general overview of the config pipeline)
+
 Configuration parameters are grouped by configuration filename.
 
 
@@ -38,26 +40,7 @@ Boolean value. If set, the amount of available context for a task in the result
 
 @subsection database
 
-@table @var
+See ref cic-base when ready
-@item host
-Database host
-@item port
-Database port
-@item name
-Database name
-@item user
-Database user
-@item password
-Database password
-@item engine
-The engine part of the dsn connection string (@code{postgresql} in @code{postgresql+psycopg2})
-@item driver
-The driver part of the dsn connection string (@code{psycopg2} in @code{postgresql+psycopg2})
-@item pool_size
-Connection pool size for database drivers that provide connection pooling
-@item debug
-Output actual sql queries to logs. Potentially very verbose
-@end table
 
 
 @subsection eth

apps/cic-eth/docker/Dockerfile

@@ -14,15 +14,6 @@ ARG EXTRA_PIP_ARGS=""
 #    --force-reinstall \
 #	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
 #    -r requirements.txt 
-
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
-
-
 COPY *requirements.txt ./
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
@@ -33,19 +24,17 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     -r services_requirements.txt \
     -r admin_requirements.txt
     
-# always install the latest signer 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    crypto-dev-signer
-
 COPY . .
 RUN python setup.py install
 
 ENV PYTHONPATH .
 
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    $EXTRA_PIP_ARGS \
+    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
 
 COPY docker/entrypoints/* ./ 
 RUN chmod 755 *.sh

apps/cic-eth/docker/Dockerfile_ci

@@ -0,0 +1,71 @@
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+WORKDIR /usr/src/cic-eth
+
+# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+# do load these all into setup.py later
+# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+#COPY cic-eth/requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_ARGS=""
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#    pip install --index-url https://pypi.org/simple \
+#    --force-reinstall \
+#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+#    -r requirements.txt 
+COPY *requirements.txt . 
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    $EXTRA_PIP_ARGS \
+    -r requirements.txt \
+    -r services_requirements.txt \
+    -r admin_requirements.txt
+    
+COPY . .
+RUN python setup.py install
+
+COPY docker/entrypoints/* ./ 
+RUN chmod 755 *.sh
+
+# # ini files in config directory defines the configurable parameters for the application
+# # they can all be overridden by environment variables
+# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-eth/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
+ENTRYPOINT []
+
+# ------------------ PRODUCTION CONTAINER ----------------------
+#FROM python:3.8.6-slim-buster as prod 
+#
+#RUN apt-get update && \
+#	apt install -y gnupg libpq-dev procps
+#
+#WORKDIR /root
+#
+#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+#                          /usr/local/lib/python3.8/site-packages/
+#
+#COPY docker/entrypoints/* ./ 
+#RUN chmod 755 *.sh
+#
+## # ini files in config directory defines the configurable parameters for the application
+## # they can all be overridden by environment variables
+## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+#COPY config/ /usr/local/etc/cic-eth/
+#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+#COPY scripts/ scripts/ 
+#
+## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+##COPY util/liveness/health.sh /usr/local/bin/health.sh
+#
+#ENTRYPOINT []
+#

apps/cic-eth/docker/entrypoints/start_tasker.sh

@@ -5,27 +5,27 @@ set -e
 
 # set CONFINI_ENV_PREFIX to override the env prefix to override env vars
 
-#echo "!!! starting signer"
+echo "!!! starting signer"
-#python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
+python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
 
 echo "!!! starting taskerd"
 /usr/local/bin/cic-eth-taskerd $@
 
 # thanks! https://docs.docker.com/config/containers/multi-service_container/
-#sleep 1;
+sleep 1;
-#echo "!!! entering monitor loop"
+echo "!!! entering monitor loop"
-#while true; do
+while true; do
-#  ps aux | grep crypto-dev-daemon | grep -q -v grep
+  ps aux | grep crypto-dev-daemon | grep -q -v grep
-#  PROCESS_1_STATUS=$?
+  PROCESS_1_STATUS=$?
-#  ps aux | grep cic-eth-tasker |grep -q -v grep
+  ps aux | grep cic-eth-tasker |grep -q -v grep
-#  PROCESS_2_STATUS=$?
+  PROCESS_2_STATUS=$?
-#  # If the greps above find anything, they exit with 0 status
+  # If the greps above find anything, they exit with 0 status
-#  # If they are not both 0, then something is wrong
+  # If they are not both 0, then something is wrong
-#  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
+  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
-#    echo "One of the processes has already exited."
+    echo "One of the processes has already exited."
-#    exit 1
+    exit 1
-#  fi
+  fi
-#  sleep 15;
+  sleep 15;
-#done
+done
-#
+
 set +e

apps/cic-eth/docker/run_tests.sh

@@ -1,11 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r admin_requirements.txt \
--r services_requirements.txt \
--r test_requirements.txt  
-
-export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-

apps/cic-eth/requirements.txt

@@ -1,4 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.9rc2,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
 semver==2.13.0
-crypto-dev-signer>=0.4.15rc2,<0.5.0

apps/cic-eth/services_requirements.txt

@@ -8,7 +8,7 @@ pycryptodome==3.10.1
 liveness~=0.0.1a7
 eth-address-index>=0.2.3a4,<0.3.0
 eth-accounts-index>=0.1.2a3,<0.2.0
-cic-eth-registry>=0.6.1a3,<0.7.0
+cic-eth-registry>=0.6.1a2,<0.7.0
 erc20-faucet>=0.3.2a2,<0.4.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -18,10 +18,7 @@ from eth_erc20 import ERC20
 from sarafu_faucet import MinterFaucet
 from eth_accounts_index.registry import AccountRegistry
 from potaahto.symbols import snake_and_camel
-from hexathon import (
+from hexathon import add_0x
-        add_0x,
-        strip_0x,
-        )
 
 # local imports
 from cic_eth.runnable.daemons.filters.callback import CallbackFilter
@@ -163,7 +160,7 @@ def test_faucet_gift_to_tx(
     assert transfer_data['token_address'] == foo_token
 
 
-def test_callback_filter_filter(
+def test_callback_filter(
         default_chain_spec,
         init_database,
         eth_rpc,
@@ -216,7 +213,6 @@ def test_callback_filter_filter(
 
         def call_back(self, transfer_type, result):
             self.results[transfer_type] = result
-            logg.debug('result {}'.format(result))
             return self
 
     mock = CallbackMock()
@@ -225,4 +221,4 @@ def test_callback_filter_filter(
     fltr.filter(eth_rpc, mockblock, tx, init_database)
 
     assert mock.results.get('transfer') != None
-    assert mock.results['transfer']['destination_token'] == strip_0x(foo_token)
+    assert mock.results['transfer']['destination_token'] == foo_token

apps/cic-eth/tests/filters/test_register_filter.py

@@ -17,9 +17,6 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.address import (
-        to_checksum_address,
-        )
 from erc20_faucet import Faucet
 from hexathon import (
         strip_0x,
@@ -28,6 +25,7 @@ from hexathon import (
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+from cic_eth.encode import tx_normalize
 from cic_eth.queue.query import get_account_tx_local
 
 logg = logging.getLogger()
@@ -72,13 +70,12 @@ def test_register_filter(
     tx = Tx(tx_src, block=block, rcpt=rcpt)
     tx.apply_receipt(rcpt)
 
-    fltr = RegistrationFilter(default_chain_spec, to_checksum_address(os.urandom(20).hex()), queue=None)
+    fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
     assert t == None
 
-    fltr = RegistrationFilter(default_chain_spec, to_checksum_address(account_registry), queue=None)
+    fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    logg.debug('t {}'.format(t))
 
     t.get_leaf()
     assert t.successful()
@@ -92,4 +89,4 @@ def test_register_filter(
     gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
 
     gift = Faucet.parse_give_to_request(gift_tx['data'])
-    assert add_0x(gift[0]) == agent_roles['ALICE']
+    assert gift[0] == agent_roles['ALICE']

apps/cic-eth/tests/filters/test_transferauth_filter.py

@@ -19,7 +19,6 @@ from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
-from cic_eth.encode import tx_normalize
 
 
 def test_filter_transferauth(
@@ -67,8 +66,7 @@ def test_filter_transferauth(
     t.get_leaf()
     assert t.successful()
 
-    #approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
-    approve_txs = get_account_tx(default_chain_spec.asdict(), tx_normalize.wallet_address(agent_roles['ALICE']), as_sender=True, session=init_database)
     ks = list(approve_txs.keys())
     assert len(ks) == 1
 
@@ -78,4 +76,4 @@ def test_filter_transferauth(
 
     c = ERC20(default_chain_spec)
     approve = c.parse_approve_request(approve_tx['data']) 
-    assert approve[0] == strip_0x(agent_roles['BOB'])
+    assert approve[0] == agent_roles['BOB']

apps/cic-eth/tests/run_tests.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
--r admin_requirements.txt
--r services_requirements.txt
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests

apps/cic-eth/tests/task/api/test_admin.py

@@ -110,8 +110,8 @@ def test_tag_account(
     t = api.tag_account('bar', agent_roles['CAROL'], default_chain_spec)
     t.get()
 
-    assert AccountRole.get_address('foo', init_database) == tx_normalize.wallet_address(agent_roles['ALICE'])
+    assert AccountRole.get_address('foo', init_database) == agent_roles['ALICE']
-    assert AccountRole.get_address('bar', init_database) == tx_normalize.wallet_address(agent_roles['CAROL'])
+    assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
 
 
 def test_tx(

apps/cic-eth/tests/task/api/test_app.py

@@ -10,7 +10,6 @@ from cic_eth_registry.erc20 import ERC20Token
 from chainlib.chain import ChainSpec
 from eth_accounts_index import AccountsIndex
 from chainlib.eth.tx import (
-        receipt,
         transaction,
         )
 from chainqueue.sql.state import (
@@ -30,7 +29,6 @@ def test_account_api(
         init_database,
         init_eth_rpc,
         account_registry,
-        cic_registry,
         custodial_roles,
         celery_session_worker,
         ):
@@ -51,7 +49,6 @@ def test_account_api_register(
         eth_rpc,
         celery_session_worker,
         ):
-
     api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
     t = api.create_account('')
     register_tx_hash = t.get_leaf()
@@ -72,18 +69,12 @@ def test_account_api_register(
     r = t.get_leaf()
     assert t.successful()
 
-    o = receipt(register_tx_hash)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
     o = transaction(register_tx_hash)
     tx_src = eth_rpc.do(o)
 
     c = AccountsIndex(default_chain_spec)
     address = c.parse_add_request(tx_src['data'])
-    logg.debug('address {} '.format(address))
     o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
-    logg.debug('o {}'.format(o))
     r = eth_rpc.do(o)
     assert c.parse_have(r)
 

apps/cic-eth/tests/task/api/test_balance.py

@@ -3,22 +3,18 @@ import os
 import logging
 
 # external imports
-import pytest
 from chainlib.eth.address import to_checksum_address
-from hexathon import add_0x
 
 # local imports
 from cic_eth.api.api_task import Api
 
 logg = logging.getLogger()
 
-
 def test_balance_simple_api(
         default_chain_spec,
         init_database,
         cic_registry,
         foo_token,
-        register_lookups,
         register_tokens,
         api,
         celery_session_worker,
@@ -26,7 +22,7 @@ def test_balance_simple_api(
 
     chain_str = str(default_chain_spec)
 
-    a = add_0x(to_checksum_address(os.urandom(20).hex()))
+    a = to_checksum_address('0x' + os.urandom(20).hex())
     t = api.balance(a, 'FOO', include_pending=False)
     r = t.get_leaf()
     assert t.successful()
@@ -40,7 +36,6 @@ def test_balance_complex_api(
         init_database,
         cic_registry,
         foo_token,
-        register_lookups,
         register_tokens,
         api,
         celery_session_worker,
@@ -48,7 +43,7 @@ def test_balance_complex_api(
 
     chain_str = str(default_chain_spec)
 
-    a = add_0x(to_checksum_address(os.urandom(20).hex()))
+    a = to_checksum_address('0x' + os.urandom(20).hex())
     t = api.balance(a, 'FOO', include_pending=True)
     r = t.get_leaf()
     assert t.successful()

apps/cic-eth/tests/task/api/test_list.py

@@ -6,7 +6,6 @@ import pytest
 from chainlib.eth.nonce import RPCNonceOracle
 from eth_erc20 import ERC20
 from chainlib.eth.tx import receipt
-from hexathon import strip_0x
 
 # local imports
 from cic_eth.api.api_task import Api
@@ -24,6 +23,7 @@ from cic_eth.pytest.mock.filter import (
 logg = logging.getLogger()
 
 
+@pytest.mark.xfail()
 def test_list_tx(
         default_chain_spec,
         init_database,
@@ -34,10 +34,8 @@ def test_list_tx(
         agent_roles,
         foo_token,
         register_tokens,
-        register_lookups,
         init_eth_tester,
         celery_session_worker,
-        init_celery_tasks,
         ):
 
     tx_hashes = []
@@ -65,16 +63,13 @@ def test_list_tx(
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
     assert r['status'] == 1
-
     a = r['block_number']
-    ab = a.to_bytes(4, 'big')
+    block_filter.add(a.to_bytes(4, 'big'))
-    block_filter.add(ab)
 
-    bb = r['transaction_index'].to_bytes(4, 'big')
+    a = r['block_number'] + r['transaction_index']
-    cb = ab + bb
+    tx_filter.add(a.to_bytes(4, 'big'))
-    tx_filter.add(cb)
 
-    tx_hashes.append(strip_0x(tx_hash_hex))
+    tx_hashes.append(tx_hash_hex)
 
     # external tx two
     Nonce.next(agent_roles['ALICE'], 'foo', session=init_database)
@@ -88,29 +83,26 @@ def test_list_tx(
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
     assert r['status'] == 1
-
     a = r['block_number']
-    ab = a.to_bytes(4, 'big')
+    block_filter.add(a.to_bytes(4, 'big'))
-    block_filter.add(ab)
 
-    bb = r['transaction_index'].to_bytes(4, 'big')
+    a = r['block_number'] + r['transaction_index']
-    cb = ab + bb
+    tx_filter.add(a.to_bytes(4, 'big'))
-    tx_filter.add(cb)
 
-    tx_hashes.append(strip_0x(tx_hash_hex))
+    tx_hashes.append(tx_hash_hex)
 
     init_eth_tester.mine_blocks(28)
 
     # custodial tx 1
     api = Api(str(default_chain_spec), queue=None)
-    t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO')
+    t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO') #, 'blinky')
     r = t.get_leaf()
     assert t.successful()
     tx_hashes.append(r)
 
     # custodial tx 2
     api = Api(str(default_chain_spec), queue=None)
-    t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO')
+    t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO') #, 'blinky')
     r = t.get_leaf()
     assert t.successful()
     tx_hashes.append(r)
@@ -125,8 +117,7 @@ def test_list_tx(
     assert len(r) == 3
     logg.debug('rrrr {}'.format(r))
 
-    logg.debug('testing against hashes {}'.format(tx_hashes))
     for tx in r:
         logg.debug('have tx {}'.format(tx))
-        tx_hashes.remove(strip_0x(tx['hash']))
+        tx_hashes.remove(tx['hash'])
     assert len(tx_hashes) == 1

apps/cic-eth/tests/task/test_ext_tx.py

@@ -10,7 +10,6 @@ from chainlib.eth.tx import (
         )
 from eth_erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
-from hexathon import add_0x 
 
 # local imports
 from cic_eth.db.models.nonce import (
@@ -92,5 +91,5 @@ def test_filter_process(
 
     assert len(r) == 2
     for tx_hash in r.keys():
-        tx_hashes.remove(add_0x(tx_hash))
+        tx_hashes.remove(tx_hash)
     assert len(tx_hashes) == 0

apps/cic-eth/tests/task/test_task_gas.py

@@ -75,7 +75,7 @@ def test_task_check_gas_ok(
             'cic_eth.eth.gas.check_gas',
             [
                 [
-                    strip_0x(tx_hash_hex),
+                    tx_hash_hex,
                     ],
                 default_chain_spec.asdict(),
                 [],
@@ -283,3 +283,4 @@ def test_task_resend_explicit(
     tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
     logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
     assert tx_after['gasPrice'] > tx_before['gasPrice']
+

apps/cic-eth/tools_requirements.txt

@@ -1,6 +1,6 @@
-crypto-dev-signer>=0.4.15a7,<=0.4.15
+crypto-dev-signer>=0.4.15a1,<=0.4.15
 chainqueue>=0.0.5a1,<0.1.0
-cic-eth-registry>=0.6.1a3,<0.7.0
+cic-eth-registry>=0.6.1a2,<0.7.0
 redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1

apps/cic-meta/.gitlab-ci.yml

@@ -1,16 +1,43 @@
-build-test-cic-meta:
+
-  stage: test
+.cic_meta_variables:
-  tags:
+    variables:
-   - integration
+        APP_NAME: cic-meta
-  variables:
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-    APP_NAME: cic-meta
+        CONTEXT: apps/$APP_NAME
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+
-  script:
+build-mr-cic-meta:
-    - cd apps/cic-meta
+    extends:
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .py_build_merge_request
-    - docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh
+        - .cic_meta_variables
-  rules:
+    rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-          - apps/$APP_NAME/**/*
+          - apps/cic-meta/**/*
+      when: always
+
+test-mr-cic-meta:
+    extends:
+        - .cic_meta_variables
+    stage: test
+    image: $MR_IMAGE_TAG 
+    script:
+        - cd /root
+        - npm install --dev 
+        - npm run test
+        - npm run test:coverage
+    needs: ["build-mr-cic-meta"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-meta/**/*
+      when: always
+
+build-push-cic-meta:
+    extends:
+        - .py_build_push
+        - .cic_meta_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-meta/**/*
       when: always

apps/cic-meta/README.md

@@ -1 +0,0 @@
-# CIC-Meta

apps/cic-meta/docker/Dockerfile

@@ -15,10 +15,11 @@ RUN --mount=type=cache,mode=0755,target=/root/.npm \
 COPY webpack.config.js .
 COPY tsconfig.json . 
 ## required to build the cic-client-meta module
-COPY . .
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
 COPY tests/*.asc /root/pgp/
 
-
 ## copy runtime configs
 COPY .config/ /usr/local/etc/cic-meta/
 #

apps/cic-meta/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+#FROM node:15.3.0-alpine3.10
+FROM node:lts-alpine3.14
+
+WORKDIR /root
+
+RUN apk add --no-cache postgresql bash
+
+# copy the dependencies
+COPY package.json package-lock.json .
+RUN npm set cache /root/.npm && \
+    npm ci
+
+COPY webpack.config.js .
+COPY tsconfig.json . 
+## required to build the cic-client-meta module
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
+COPY tests/*.asc /root/pgp/
+
+## copy runtime configs
+COPY .config/ /usr/local/etc/cic-meta/
+#
+## db migrations
+COPY docker/db.sh ./db.sh
+RUN chmod 755 ./db.sh
+#
+RUN alias tsc=node_modules/typescript/bin/tsc 
+COPY docker/start_server.sh ./start_server.sh
+RUN chmod 755 ./start_server.sh
+ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/run_tests.sh

@@ -1,7 +0,0 @@
-#! /bin/bash
-
-set -e
-
-npm install --dev 
-npm run test
-npm run test:coverage

apps/cic-notify/.gitlab-ci.yml

@@ -1,17 +1,52 @@
-build-test-cic-notify:
+.cic_notify_variables:
-  stage: test
+    variables:
-  tags:
+        APP_NAME: cic-notify
-    - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-notify
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-notify:
-  script:
+    extends:
-    - cd apps/cic-notify
+        - .py_build_merge_request
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .cic_notify_variables
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-  allow_failure: true
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-  rules:
+      changes:
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+          - apps/cic-notify/**/*
-    changes:
+      when: always
-        - apps/$APP_NAME/**/*
+
-    when: always
+test-mr-cic-notify:
+    stage: test
+    extends:
+        - .cic_notify_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-notify"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
+
+build-push-cic-notify:
+    extends:
+        - .py_build_push
+        - .cic_notify_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-notify/**/*
+      when: always
+        
+

apps/cic-notify/cic_notify/tasks/sms/db.py

@@ -11,12 +11,12 @@ celery_app = celery.current_app
 
 
 @celery_app.task
-def persist_notification(message, recipient):
+def persist_notification(recipient, message):
     """
-    :param message:
-    :type message:
     :param recipient:
     :type recipient:
+    :param message:
+    :type message:
     :return:
     :rtype:
     """

apps/cic-notify/cic_notify/tasks/sms/log.py

@@ -11,13 +11,12 @@ local_logg = logging.getLogger(__name__)
 
 
 @celery_app.task
-def log(message, recipient):
+def log(recipient, message):
     """
-
-    :param message:
-    :type message:
     :param recipient:
     :type recipient:
+    :param message:
+    :type message:
     :return:
     :rtype:
     """

apps/cic-notify/docker/Dockerfile_ci

@@ -0,0 +1,27 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
+
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . .
+
+RUN python setup.py install
+
+COPY docker/*.sh .
+RUN chmod +x *.sh 
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY .config/ /usr/local/etc/cic-notify/
+COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+
+ENTRYPOINT []

apps/cic-notify/docker/run_tests.sh

@@ -1,9 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
---extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests

apps/cic-ussd/.gitlab-ci.yml

@@ -1,16 +1,52 @@
-build-test-cic-ussd:
+.cic_ussd_variables:
-  stage: test
+    variables:
-  tags:
+        APP_NAME: cic-ussd
-  - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-ussd
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-ussd:
-  script:
+    extends:
-    - cd apps/cic-ussd
+        - .py_build_merge_request
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .cic_ussd_variables
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
-    changes:
+          - apps/cic-ussd/**/*
-        - apps/$APP_NAME/**/*
+      when: always
-    when: always
+
+test-mr-cic-ussd:
+    stage: test
+    extends:
+        - .cic_ussd_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd
+    needs: ["build-mr-cic-ussd"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
+
+build-push-cic-ussd:
+    extends:
+        - .py_build_push
+        - .cic_ussd_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-ussd/**/*
+      when: always
+
+

apps/cic-ussd/cic_ussd/account/balance.py

@@ -1,12 +1,10 @@
 # standard imports
-
 import json
 import logging
 from typing import Optional
 
 # third-party imports
 from cic_eth.api import Api
-from cic_eth_aux.erc20_demurrage_token.api import Api as DemurrageApi
 
 # local imports
 from cic_ussd.account.transaction import from_wei
@@ -75,24 +73,6 @@ def calculate_available_balance(balances: dict) -> float:
     return from_wei(value=available_balance)
 
 
-def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_symbol: str):
-    """
-    :param balance:
-    :type balance:
-    :param chain_str:
-    :type chain_str:
-    :param timestamp:
-    :type timestamp:
-    :param token_symbol:
-    :type token_symbol:
-    :return:
-    :rtype:
-    """
-    logg.debug(f'retrieving adjusted balance on chain: {chain_str}')
-    demurrage_api = DemurrageApi(chain_str=chain_str)
-    return demurrage_api.get_adjusted_balance(token_symbol, balance, timestamp).result
-
-
 def get_cached_available_balance(blockchain_address: str) -> float:
     """This function attempts to retrieve balance data from the redis cache.
     :param blockchain_address: Ethereum address of an account.
@@ -101,21 +81,10 @@ def get_cached_available_balance(blockchain_address: str) -> float:
     :return: Operational balance of an account.
     :rtype: float
     """
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(blockchain_address[2:])
     key = cache_data_key(identifier, salt=':cic.balances')
     cached_balances = get_cached_data(key=key)
     if cached_balances:
         return calculate_available_balance(json.loads(cached_balances))
     else:
         raise CachedDataNotFoundError(f'No cached available balance for address: {blockchain_address}')
-
-
-def get_cached_adjusted_balance(identifier: bytes):
-    """
-    :param identifier:
-    :type identifier:
-    :return:
-    :rtype:
-    """
-    key = cache_data_key(identifier, ':cic.adjusted_balance')
-    return get_cached_data(key)

apps/cic-ussd/cic_ussd/account/metadata.py

@@ -4,6 +4,7 @@ import logging
 from typing import Optional
 
 # external imports
+from chainlib.hash import strip_0x
 from cic_types.models.person import Person
 
 # local imports
@@ -19,7 +20,7 @@ def get_cached_preferred_language(blockchain_address: str) -> Optional[str]:
     :return: Account's set preferred language | Fallback preferred language.
     :rtype: str
     """
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(strip_0x(blockchain_address))
     preferences_metadata_handler = PreferencesMetadata(identifier)
     cached_preferences_metadata = preferences_metadata_handler.get_cached_metadata()
     if cached_preferences_metadata:

apps/cic-ussd/cic_ussd/account/statement.py

@@ -86,7 +86,7 @@ def query_statement(blockchain_address: str, limit: int = 9):
     :param limit: Number of transactions to be returned.
     :type limit: int
     """
-    logg.debug(f'retrieving statement for address: {blockchain_address}')
+    logg.debug(f'retrieving balance for address: {blockchain_address}')
     chain_str = Chain.spec.__str__()
     cic_eth_api = Api(
         chain_str=chain_str,

apps/cic-ussd/cic_ussd/account/transaction.py

@@ -1,6 +1,5 @@
 # standard import
 import decimal
-import json
 import logging
 from typing import Dict, Tuple
 
@@ -9,8 +8,6 @@ from cic_eth.api import Api
 from sqlalchemy.orm.session import Session
 
 # local import
-from cic_ussd.account.chain import Chain
-from cic_ussd.account.tokens import get_cached_default_token
 from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.error import UnknownUssdRecipient
@@ -62,9 +59,7 @@ def from_wei(value: int) -> float:
     :return: SRF equivalent of value in Wei
     :rtype: float
     """
-    cached_token_data = json.loads(get_cached_default_token(Chain.spec.__str__()))
+    value = float(value) / 1e+6
-    token_decimals: int = cached_token_data.get('decimals')
-    value = float(value) / (10**token_decimals)
     return truncate(value=value, decimals=2)
 
 
@@ -75,9 +70,7 @@ def to_wei(value: int) -> int:
     :return: Wei equivalent of value in SRF
     :rtype: int
     """
-    cached_token_data = json.loads(get_cached_default_token(Chain.spec.__str__()))
+    return int(value * 1e+6)
-    token_decimals: int = cached_token_data.get('decimals')
-    return int(value * (10**token_decimals))
 
 
 def truncate(value: float, decimals: int):

apps/cic-ussd/cic_ussd/db/models/account.py

@@ -2,6 +2,7 @@
 import json
 
 # external imports
+from chainlib.hash import strip_0x
 from cic_eth.api import Api
 
 # local imports
@@ -100,7 +101,7 @@ class Account(SessionBase):
         session.add(self)
         session.flush()
         SessionBase.release_session(session=session)
-        return 'Pin reset successful.'
+        return f'Pin reset successful.'
 
     def standard_metadata_id(self) -> str:
         """This function creates an account's standard metadata identification information that contains an account owner's
@@ -108,7 +109,7 @@ class Account(SessionBase):
         :return: Standard metadata identification information | e164 formatted phone number.
         :rtype: str
         """
-        identifier = bytes.fromhex(self.blockchain_address)
+        identifier = bytes.fromhex(strip_0x(self.blockchain_address))
         key = cache_data_key(identifier, ':cic.person')
         account_metadata = get_cached_data(key)
         if not account_metadata:

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -44,7 +44,7 @@ class MetadataRequestsHandler(Metadata):
 
     def create(self, data: Union[Dict, str]):
         """"""
-        data = json.dumps(data).encode('utf-8')
+        data = json.dumps(data)
         result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
 
         error_handler(result=result)

apps/cic-ussd/cic_ussd/processor/menu.py

@@ -1,17 +1,13 @@
 # standard imports
 import json
 import logging
-from datetime import datetime, timedelta
 
 # external imports
 import i18n.config
+from sqlalchemy.orm.session import Session
 
 # local imports
-from cic_ussd.account.balance import (calculate_available_balance,
+from cic_ussd.account.balance import calculate_available_balance, get_balances, get_cached_available_balance
-                                      get_adjusted_balance,
-                                      get_balances,
-                                      get_cached_adjusted_balance,
-                                      get_cached_available_balance)
 from cic_ussd.account.chain import Chain
 from cic_ussd.account.metadata import get_cached_preferred_language
 from cic_ussd.account.statement import (
@@ -20,15 +16,14 @@ from cic_ussd.account.statement import (
     query_statement,
     statement_transaction_set
 )
-from cic_ussd.account.tokens import get_default_token_symbol
 from cic_ussd.account.transaction import from_wei, to_wei
+from cic_ussd.account.tokens import get_default_token_symbol
 from cic_ussd.cache import cache_data_key, cache_data
 from cic_ussd.db.models.account import Account
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
-from cic_ussd.processor.util import parse_person_metadata
+from cic_ussd.processor.util import latest_input, parse_person_metadata
 from cic_ussd.translation import translation_for
-from sqlalchemy.orm.session import Session
 
 logg = logging.getLogger(__name__)
 
@@ -37,7 +32,7 @@ class MenuProcessor:
     def __init__(self, account: Account, display_key: str, menu_name: str, session: Session, ussd_session: dict):
         self.account = account
         self.display_key = display_key
-        self.identifier = bytes.fromhex(self.account.blockchain_address)
+        self.identifier = bytes.fromhex(self.account.blockchain_address[2:])
         self.menu_name = menu_name
         self.session = session
         self.ussd_session = ussd_session
@@ -48,26 +43,21 @@ class MenuProcessor:
         :rtype:
         """
         available_balance = get_cached_available_balance(self.account.blockchain_address)
-        adjusted_balance = get_cached_adjusted_balance(self.identifier)
+        logg.debug('Requires call to retrieve tax and bonus amounts')
+        tax = ''
+        bonus = ''
         token_symbol = get_default_token_symbol()
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
-        with_available_balance = f'{self.display_key}.available_balance'
+        return translation_for(
-        with_fees = f'{self.display_key}.with_fees'
+            key=self.display_key,
-        if not adjusted_balance:
+            preferred_language=preferred_language,
-            return translation_for(key=with_available_balance,
+            available_balance=available_balance,
-                                   preferred_language=preferred_language,
+            tax=tax,
-                                   available_balance=available_balance,
+            bonus=bonus,
-                                   token_symbol=token_symbol)
+            token_symbol=token_symbol
-        adjusted_balance = json.loads(adjusted_balance)
+        )
-        tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
-        tax = from_wei(int(tax_wei))
-        return translation_for(key=with_fees,
-                               preferred_language=preferred_language,
-                               available_balance=available_balance,
-                               tax=tax,
-                               token_symbol=token_symbol)
 
     def account_statement(self) -> str:
         """
@@ -77,7 +67,7 @@ class MenuProcessor:
         cached_statement = get_cached_statement(self.account.blockchain_address)
         statement = json.loads(cached_statement)
         statement_transactions = parse_statement_transactions(statement)
-        transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)]
+        transaction_sets = [statement_transactions[tx:tx+3] for tx in range(0, len(statement_transactions), 3)]
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
@@ -159,22 +149,12 @@ class MenuProcessor:
         :return:
         :rtype:
         """
-        chain_str = Chain.spec.__str__()
         token_symbol = get_default_token_symbol()
         blockchain_address = self.account.blockchain_address
-        balances = get_balances(blockchain_address, chain_str, token_symbol, False)[0]
+        balances = get_balances(blockchain_address, Chain.spec.__str__(), token_symbol, False)[0]
         key = cache_data_key(self.identifier, ':cic.balances')
         cache_data(key, json.dumps(balances))
         available_balance = calculate_available_balance(balances)
-        now = datetime.now()
-        if (now - self.account.created).days >= 30:
-            if available_balance <= 0:
-                logg.info(f'Not retrieving adjusted balance, available balance: {available_balance} is insufficient.')
-            else:
-                timestamp = int((now - timedelta(30)).timestamp())
-                adjusted_balance = get_adjusted_balance(to_wei(int(available_balance)), chain_str, timestamp, token_symbol)
-                key = cache_data_key(self.identifier, ':cic.adjusted_balance')
-                cache_data(key, json.dumps(adjusted_balance))
 
         query_statement(blockchain_address)
 

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -63,7 +63,7 @@ elif ssl == 0:
 else:
     ssl = True
 
-valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
+
 def main():
 
     # TODO: improve url building 
@@ -79,9 +79,9 @@ def main():
     session = uuid.uuid4().hex
     data = {
             'sessionId': session,
-            'serviceCode': valid_service_codes[0],
+            'serviceCode': config.get('APP_SERVICE_CODE'),
             'phoneNumber': args.phone,
-            'text': "",
+            'text': config.get('APP_SERVICE_CODE'),
         }
 
     state = "_BEGIN"

apps/cic-ussd/cic_ussd/session/ussd_session.py

@@ -146,7 +146,7 @@ def create_ussd_session(
                        )
 
 
-def update_ussd_session(ussd_session: DbUssdSession,
+def update_ussd_session(ussd_session: UssdSession,
                         user_input: str,
                         state: str,
                         data: Optional[dict] = None) -> UssdSession:

apps/cic-ussd/cic_ussd/state_machine/logic/account.py

@@ -154,14 +154,15 @@ def parse_person_metadata(account: Account, metadata: dict):
     phone_number = account.phone_number
     date_registered = int(account.created.replace().timestamp())
     blockchain_address = account.blockchain_address
-    chain_str = Chain.spec.__str__()
+    chain_spec = f'{Chain.spec.common_name()}:{Chain.spec.engine()}: {Chain.spec.chain_id()}'
 
     if isinstance(metadata.get('identities'), dict):
         identities = metadata.get('identities')
     else:
         identities = manage_identity_data(
             blockchain_address=blockchain_address,
-            chain_str=chain_str
+            blockchain_type=Chain.spec.engine(),
+            chain_spec=chain_spec
         )
 
     return {

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -5,6 +5,7 @@ from datetime import timedelta
 
 # third-party imports
 import celery
+from chainlib.hash import strip_0x
 
 # local imports
 from cic_ussd.account.balance import get_balances, calculate_available_balance
@@ -54,7 +55,6 @@ def account_creation_callback(self, result: str, url: str, status_code: int):
     session.add(account)
     session.commit()
     session.close()
-    logg.debug(f'recorded account with identifier: {result}')
 
     queue = self.request.delivery_info.get('routing_key')
     s_phone_pointer = celery.signature(
@@ -86,7 +86,7 @@ def balances_callback(result: list, param: str, status_code: int):
         raise ValueError(f'Unexpected status code: {status_code}.')
 
     balances = result[0]
-    identifier = bytes.fromhex(param)
+    identifier = bytes.fromhex(strip_0x(param))
     key = cache_data_key(identifier, ':cic.balances')
     cache_data(key, json.dumps(balances))
 
@@ -113,10 +113,8 @@ def statement_callback(self, result, param: str, status_code: int):
     for transaction in statement_transactions:
         recipient_transaction, sender_transaction = transaction_actors(transaction)
         if recipient_transaction.get('blockchain_address') == param:
-            recipient_transaction['alt_blockchain_address'] = sender_transaction.get('blockchain_address')
             generate(param, queue, recipient_transaction)
         if sender_transaction.get('blockchain_address') == param:
-            sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address')
             generate(param, queue, sender_transaction)
 
 
@@ -140,14 +138,26 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     balances_data = result[0]
     available_balance = calculate_available_balance(balances_data)
     transaction = param
+    blockchain_address = transaction.get('blockchain_address')
     transaction['available_balance'] = available_balance
     queue = self.request.delivery_info.get('routing_key')
 
+    s_preferences_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.query_preferences_metadata', [blockchain_address], queue=queue
+    )
     s_process_account_metadata = celery.signature(
         'cic_ussd.tasks.processor.parse_transaction', [transaction], queue=queue
     )
     s_notify_account = celery.signature('cic_ussd.tasks.notifications.transaction', queue=queue)
-    celery.chain(s_process_account_metadata, s_notify_account).apply_async()
+
+    if transaction.get('transaction_type') == 'transfer':
+        celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async()
+
+    if transaction.get('transaction_type') == 'tokengift':
+        s_process_account_metadata = celery.signature(
+            'cic_ussd.tasks.processor.parse_transaction', [{}, transaction], queue=queue
+        )
+        celery.chain(s_process_account_metadata, s_notify_account).apply_async()
 
 
 @celery_app.task

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -3,6 +3,7 @@ import logging
 
 # third-party imports
 import celery
+from hexathon import strip_0x
 
 # local imports
 from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
@@ -20,7 +21,7 @@ def query_person_metadata(blockchain_address: str):
     :return:
     :rtype:
     """
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(strip_0x(blockchain_address))
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.query()
 
@@ -35,14 +36,14 @@ def create_person_metadata(blockchain_address: str, data: dict):
     :return:
     :rtype:
     """
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(strip_0x(blockchain_address))
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.create(data=data)
 
 
 @celery_app.task
 def edit_person_metadata(blockchain_address: str, data: dict):
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(strip_0x(blockchain_address))
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.edit(data=data)
 
@@ -50,21 +51,21 @@ def edit_person_metadata(blockchain_address: str, data: dict):
 @celery_app.task(bind=True, base=CriticalMetadataTask)
 def add_phone_pointer(self, blockchain_address: str, phone_number: str):
     identifier = phone_number.encode('utf-8')
-    stripped_address = blockchain_address
+    stripped_address = strip_0x(blockchain_address)
     phone_metadata_client = PhonePointerMetadata(identifier=identifier)
     phone_metadata_client.create(data=stripped_address)
 
 
 @celery_app.task()
 def add_custom_metadata(blockchain_address: str, data: dict):
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(strip_0x(blockchain_address))
     custom_metadata_client = CustomMetadata(identifier=identifier)
     custom_metadata_client.create(data=data)
 
 
 @celery_app.task()
 def add_preferences_metadata(blockchain_address: str, data: dict):
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(strip_0x(blockchain_address))
     preferences_metadata_client = PreferencesMetadata(identifier=identifier)
     preferences_metadata_client.create(data=data)
 
@@ -75,7 +76,7 @@ def query_preferences_metadata(blockchain_address: str):
     :param blockchain_address: Blockchain address of an account.
     :type blockchain_address: str | Ox-hex
     """
-    identifier = bytes.fromhex(blockchain_address)
+    identifier = bytes.fromhex(strip_0x(blockchain_address))
     logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
     person_metadata_client = PreferencesMetadata(identifier=identifier)
     return person_metadata_client.query()

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -24,8 +24,7 @@ def transaction(notification_data: dict):
     :rtype:
     """
     role = notification_data.get('role')
-    token_value = notification_data.get('token_value')
+    amount = from_wei(notification_data.get('token_value'))
-    amount = token_value if token_value == 0 else from_wei(token_value)
     balance = notification_data.get('available_balance')
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -5,9 +5,9 @@ import logging
 # third-party imports
 import celery
 import i18n
+from chainlib.hash import strip_0x
 
 # local imports
-from cic_ussd.account.metadata import get_cached_preferred_language
 from cic_ussd.account.statement import get_cached_statement
 from cic_ussd.account.transaction import aux_transaction_data, validate_transaction_account
 from cic_ussd.cache import cache_data, cache_data_key
@@ -23,13 +23,17 @@ logg = logging.getLogger(__file__)
 def generate_statement(self, querying_party: str, transaction: dict):
     """"""
     queue = self.request.delivery_info.get('routing_key')
+
+    s_preferences = celery.signature(
+        'cic_ussd.tasks.metadata.query_preferences_metadata', [querying_party], queue=queue
+    )
     s_parse_transaction = celery.signature(
         'cic_ussd.tasks.processor.parse_transaction', [transaction], queue=queue
     )
     s_cache_statement = celery.signature(
         'cic_ussd.tasks.processor.cache_statement', [querying_party], queue=queue
     )
-    celery.chain(s_parse_transaction, s_cache_statement).apply_async()
+    celery.chain(s_preferences, s_parse_transaction, s_cache_statement).apply_async()
 
 
 @celery_app.task
@@ -48,23 +52,25 @@ def cache_statement(parsed_transaction: dict, querying_party: str):
         statement_transactions = json.loads(cached_statement)
     statement_transactions.append(parsed_transaction)
     data = json.dumps(statement_transactions)
-    identifier = bytes.fromhex(querying_party)
+    identifier = bytes.fromhex(strip_0x(querying_party))
     key = cache_data_key(identifier, ':cic.statement')
     cache_data(key, data)
 
 
 @celery_app.task
-def parse_transaction(transaction: dict) -> dict:
+def parse_transaction(preferences: dict, transaction: dict) -> dict:
     """This function parses transaction objects and collates all relevant data for system use i.e:
     - An account's set preferred language.
     - Account identifier that facilitates notification.
     - Contextual tags i.e action and direction tags.
+    :param preferences: An account's set preferences.
+    :type preferences: dict
     :param transaction: Transaction object.
     :type transaction: dict
     :return: Transaction object with contextual data for use in the system.
     :rtype: dict
     """
-    preferred_language = get_cached_preferred_language(transaction.get('blockchain_address'))
+    preferred_language = preferences.get('preferred_language')
     if not preferred_language:
         preferred_language = i18n.config.get('fallback')
     transaction['preferred_language'] = preferred_language
@@ -77,8 +83,6 @@ def parse_transaction(transaction: dict) -> dict:
     alt_account = session.query(Account).filter_by(blockchain_address=alt_blockchain_address).first()
     if alt_account:
         transaction['alt_metadata_id'] = alt_account.standard_metadata_id()
-    else:
-        transaction['alt_metadata_id'] = 'GRASSROOTS ECONOMICS'
     transaction['metadata_id'] = account.standard_metadata_id()
     transaction['phone_number'] = account.phone_number
     session.close()

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 1, 'alpha.5')
+version = (0, 3, 1, 'alpha.4')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/config/client/app.ini

@@ -0,0 +1,2 @@
+[app]
+service_code = *483*46#

apps/cic-ussd/config/client/ussd.ini

@@ -1,4 +1,3 @@
 [ussd]
-service_code = *483*46#
 user =
 pass =

apps/cic-ussd/docker/Dockerfile

@@ -10,13 +10,6 @@ RUN mkdir -vp data
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
-
 COPY requirements.txt .
 
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \

apps/cic-ussd/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+RUN apt-get install -y redis-server
+
+# create secrets directory
+RUN mkdir -vp pgp/keys
+
+# create application directory
+RUN mkdir -vp cic-ussd
+RUN mkdir -vp data
+
+COPY requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . .
+RUN python setup.py install
+
+COPY cic_ussd/db/ussd_menu.json data/
+
+COPY docker/*.sh .
+RUN chmod +x /root/*.sh
+
+# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
+COPY config/ /usr/local/etc/cic-ussd/
+COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+
+ENTRYPOINT []

apps/cic-ussd/docker/run_tests.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
---extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd

apps/cic-ussd/requirements.txt

@@ -1,12 +1,9 @@
 alembic==1.4.2
-attrs==21.2.0
-billiard==3.6.4.0
 bcrypt==3.2.0
 celery==4.4.7
-cffi==1.14.6
 cic-eth[services]~=0.12.4a7
 cic-notify~=0.4.0a10
-cic-types~=0.1.0a15
+cic-types~=0.1.0a14
 confini>=0.4.1a1,<0.5.0
 phonenumbers==8.12.12
 psycopg2==2.8.6

apps/cic-ussd/tests/cic_ussd/account/test_balance.py

@@ -43,13 +43,10 @@ def test_sync_get_balances(activated_account,
     (5000000, 89000000, 67000000, 27.00)
 ])
 def test_calculate_available_balance(activated_account,
-                                     available_balance,
                                      balance_incoming,
                                      balance_network,
                                      balance_outgoing,
-                                     cache_balances,
+                                     available_balance):
-                                     cache_default_token_data,
-                                     load_chain_spec):
     balances = {
         'address': activated_account.blockchain_address,
         'converters': [],
@@ -60,11 +57,7 @@ def test_calculate_available_balance(activated_account,
     assert calculate_available_balance(balances) == available_balance
 
 
-def test_get_cached_available_balance(activated_account,
+def test_get_cached_available_balance(activated_account, cache_balances, balances):
-                                      balances,
-                                      cache_balances,
-                                      cache_default_token_data,
-                                      load_chain_spec):
     cached_available_balance = get_cached_available_balance(activated_account.blockchain_address)
     available_balance = calculate_available_balance(balances[0])
     assert cached_available_balance == available_balance

apps/cic-ussd/tests/cic_ussd/account/test_statement.py

@@ -27,7 +27,6 @@ def test_filter_statement_transactions(transactions_list):
 
 
 def test_generate(activated_account,
-                  cache_default_token_data,
                   cache_preferences,
                   celery_session_worker,
                   init_cache,
@@ -36,22 +35,22 @@ def test_generate(activated_account,
                   preferences,
                   preferences_metadata_url,
                   transactions_list):
-    statement_transactions = filter_statement_transactions(transactions_list)
+    with requests_mock.Mocker(real_http=False) as request_mocker:
-    for transaction in statement_transactions:
+        request_mocker.register_uri('GET', preferences_metadata_url, status_code=200, reason='OK', json=preferences)
-        querying_party = activated_account.blockchain_address
+        statement_transactions = filter_statement_transactions(transactions_list)
-        recipient_transaction, sender_transaction = transaction_actors(transaction)
+        for transaction in statement_transactions:
-        if recipient_transaction.get('blockchain_address') == querying_party:
+            querying_party = activated_account.blockchain_address
-            recipient_transaction['alt_blockchain_address'] = sender_transaction.get('blockchain_address')
+            recipient_transaction, sender_transaction = transaction_actors(transaction)
-            generate(querying_party, None, recipient_transaction)
+            if recipient_transaction.get('blockchain_address') == querying_party:
-        if sender_transaction.get('blockchain_address') == querying_party:
+                generate(querying_party, None, recipient_transaction)
-            sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address')
+            if sender_transaction.get('blockchain_address') == querying_party:
-            generate(querying_party, None, sender_transaction)
+                generate(querying_party, None, sender_transaction)
-    time.sleep(2)
+        time.sleep(2)
-    identifier = bytes.fromhex(activated_account.blockchain_address)
+        identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
-    key = cache_data_key(identifier, ':cic.statement')
+        key = cache_data_key(identifier, ':cic.statement')
-    statement = get_cached_data(key)
+        statement = get_cached_data(key)
-    statement = json.loads(statement)
+        statement = json.loads(statement)
-    assert len(statement) == 1
+        assert len(statement) == 1
 
 
 def test_get_cached_statement(activated_account, cache_statement, statement):
@@ -61,7 +60,7 @@ def test_get_cached_statement(activated_account, cache_statement, statement):
     assert cached_statement[0].get('blockchain_address') == statement[0].get('blockchain_address')
 
 
-def test_parse_statement_transactions(cache_default_token_data, statement):
+def test_parse_statement_transactions(statement):
     parsed_transactions = parse_statement_transactions(statement)
     parsed_transaction = parsed_transactions[0]
     parsed_transaction.startswith('Sent')
@@ -77,7 +76,7 @@ def test_query_statement(blockchain_address, limit, load_chain_spec, activated_a
     assert mock_transaction_list_query.get('limit') == limit
 
 
-def test_statement_transaction_set(cache_default_token_data, load_chain_spec, preferences, set_locale_files, statement):
+def test_statement_transaction_set(preferences, set_locale_files, statement):
     parsed_transactions = parse_statement_transactions(statement)
     preferred_language = preferences.get('preferred_language')
     transaction_set = statement_transaction_set(preferred_language, parsed_transactions)

apps/cic-ussd/tests/cic_ussd/account/test_transaction.py

@@ -36,19 +36,19 @@ def test_aux_transaction_data(preferences, set_locale_files, transactions_list):
     check_aux_data('helpers.sent', 'helpers.to', preferred_language, sender_tx_aux_data)
 
 
-@pytest.mark.parametrize("value, expected_result", [
+@pytest.mark.parametrize("wei, expected_result", [
     (50000000, Decimal('50.00')),
     (100000, Decimal('0.10'))
 ])
-def test_from_wei(cache_default_token_data, expected_result, value):
+def test_from_wei(wei, expected_result):
-    assert from_wei(value) == expected_result
+    assert from_wei(wei) == expected_result
 
 
 @pytest.mark.parametrize("value, expected_result", [
     (50, 50000000),
     (0.10, 100000)
 ])
-def test_to_wei(cache_default_token_data, expected_result, value):
+def test_to_wei(value, expected_result):
     assert to_wei(value) == expected_result
 
 
@@ -96,7 +96,6 @@ def test_validate_transaction_account(activated_account, init_database, transact
 @pytest.mark.parametrize("amount", [50, 0.10])
 def test_outgoing_transaction_processor(activated_account,
                                         amount,
-                                        cache_default_token_data,
                                         celery_session_worker,
                                         load_config,
                                         load_chain_spec,

apps/cic-ussd/tests/cic_ussd/processor/test_menu.py

@@ -1,6 +1,5 @@
 # standard imports
 import json
-import datetime
 
 # external imports
 from chainlib.hash import strip_0x
@@ -15,7 +14,6 @@ from cic_ussd.account.statement import (
 )
 from cic_ussd.account.tokens import get_default_token_symbol
 from cic_ussd.account.transaction import from_wei, to_wei
-from cic_ussd.cache import cache_data, cache_data_key
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
@@ -40,34 +38,24 @@ def test_menu_processor(activated_account,
                         load_chain_spec,
                         load_support_phone,
                         load_ussd_menu,
-                        mock_get_adjusted_balance,
                         mock_sync_balance_api_query,
                         mock_transaction_list_query,
                         valid_recipient):
     preferred_language = get_cached_preferred_language(activated_account.blockchain_address)
     available_balance = get_cached_available_balance(activated_account.blockchain_address)
     token_symbol = get_default_token_symbol()
-    with_available_balance = 'ussd.kenya.account_balances.available_balance'
+
-    with_fees = 'ussd.kenya.account_balances.with_fees'
+    tax = ''
+    bonus = ''
+    display_key = 'ussd.kenya.account_balances'
     ussd_menu = UssdMenu.find_by_name('account_balances')
     name = ussd_menu.get('name')
-    resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session)
+    resp = response(activated_account, display_key, name, init_database, generic_ussd_session)
-    assert resp == translation_for(with_available_balance,
+    assert resp == translation_for(display_key,
                                    preferred_language,
                                    available_balance=available_balance,
-                                   token_symbol=token_symbol)
-
-    identifier = bytes.fromhex(activated_account.blockchain_address)
-    key = cache_data_key(identifier, ':cic.adjusted_balance')
-    adjusted_balance = 45931650.64654012
-    cache_data(key, json.dumps(adjusted_balance))
-    resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session)
-    tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
-    tax = from_wei(int(tax_wei))
-    assert resp == translation_for(key=with_fees,
-                                   preferred_language=preferred_language,
-                                   available_balance=available_balance,
                                    tax=tax,
+                                   bonus=bonus,
                                    token_symbol=token_symbol)
 
     cached_statement = get_cached_statement(activated_account.blockchain_address)
@@ -108,7 +96,7 @@ def test_menu_processor(activated_account,
     display_key = 'ussd.kenya.display_user_metadata'
     ussd_menu = UssdMenu.find_by_name('display_user_metadata')
     name = ussd_menu.get('name')
-    identifier = bytes.fromhex(activated_account.blockchain_address)
+    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
     person_metadata = PersonMetadata(identifier)
     cached_person_metadata = person_metadata.get_cached_metadata()
     resp = response(activated_account, display_key, name, init_database, generic_ussd_session)
@@ -135,15 +123,6 @@ def test_menu_processor(activated_account,
                                    account_balance=available_balance,
                                    account_token_name=token_symbol)
 
-    display_key = 'ussd.kenya.start'
-    ussd_menu = UssdMenu.find_by_name('start')
-    name = ussd_menu.get('name')
-    older_timestamp = (activated_account.created - datetime.timedelta(days=35))
-    activated_account.created = older_timestamp
-    init_database.flush()
-    response(activated_account, display_key, name, init_database, generic_ussd_session)
-    assert mock_get_adjusted_balance['timestamp'] == int((datetime.datetime.now() - datetime.timedelta(days=30)).timestamp())
-
     display_key = 'ussd.kenya.transaction_pin_authorization'
     ussd_menu = UssdMenu.find_by_name('transaction_pin_authorization')
     name = ussd_menu.get('name')

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py

@@ -49,7 +49,6 @@ def test_is_valid_transaction_amount(activated_account, amount, expected_result,
 ])
 def test_has_sufficient_balance(activated_account,
                                 cache_balances,
-                                cache_default_token_data,
                                 expected_result,
                                 generic_ussd_session,
                                 init_database,

apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py

@@ -114,7 +114,6 @@ def test_statement_callback(activated_account, mocker, transactions_list):
     s_statement_callback.apply_async().get()
     statement_transactions = filter_statement_transactions(transactions_list)
     recipient_transaction, sender_transaction = transaction_actors(statement_transactions[0])
-    sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address')
     mock_statement_generate.assert_called_with(
         (activated_account.blockchain_address, sender_transaction), {}, queue='cic-ussd')
 
@@ -122,7 +121,6 @@ def test_statement_callback(activated_account, mocker, transactions_list):
 def test_transaction_balances_callback(activated_account,
                                        balances,
                                        cache_balances,
-                                       cache_default_token_data,
                                        cache_person_metadata,
                                        cache_preferences,
                                        load_chain_spec,

apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py

@@ -13,8 +13,7 @@ from cic_ussd.translation import translation_for
 # tests imports
 
 
-def test_transaction(cache_default_token_data,
+def test_transaction(celery_session_worker,
-                     celery_session_worker,
                      load_support_phone,
                      mock_notifier_api,
                      notification_data,

apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py

@@ -30,11 +30,10 @@ def test_generate_statement(activated_account,
 
 
 def test_cache_statement(activated_account,
-                         cache_default_token_data,
                          cache_person_metadata,
-                         cache_preferences,
                          celery_session_worker,
                          init_database,
+                         preferences,
                          transaction_result):
     recipient_transaction, sender_transaction = transaction_actors(transaction_result)
     identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
@@ -42,7 +41,7 @@ def test_cache_statement(activated_account,
     cached_statement = get_cached_data(key)
     assert cached_statement is None
     s_parse_transaction = celery.signature(
-        'cic_ussd.tasks.processor.parse_transaction', [sender_transaction])
+        'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction])
     result = s_parse_transaction.apply_async().get()
     s_cache_statement = celery.signature(
         'cic_ussd.tasks.processor.cache_statement', [result, activated_account.blockchain_address]
@@ -62,15 +61,15 @@ def test_cache_statement(activated_account,
 
 def test_parse_transaction(activated_account,
                            cache_person_metadata,
-                           cache_preferences,
                            celery_session_worker,
                            init_database,
+                           preferences,
                            transaction_result):
     recipient_transaction, sender_transaction = transaction_actors(transaction_result)
     assert sender_transaction.get('metadata_id') is None
     assert sender_transaction.get('phone_number') is None
     s_parse_transaction = celery.signature(
-        'cic_ussd.tasks.processor.parse_transaction', [sender_transaction])
+        'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction])
     result = s_parse_transaction.apply_async().get()
     assert result.get('metadata_id') == activated_account.standard_metadata_id()
     assert result.get('phone_number') == activated_account.phone_number

apps/cic-ussd/tests/fixtures/account.py

@@ -54,7 +54,7 @@ def cache_account_creation_data(init_cache, account_creation_data):
 
 @pytest.fixture(scope='function')
 def cache_balances(activated_account, balances, init_cache):
-    identifier = bytes.fromhex(activated_account.blockchain_address)
+    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
     balances = json.dumps(balances[0])
     key = cache_data_key(identifier, ':cic.balances')
     cache_data(key, balances)
@@ -70,7 +70,7 @@ def cache_default_token_data(default_token_data, init_cache, load_chain_spec):
 
 @pytest.fixture(scope='function')
 def cache_person_metadata(activated_account, init_cache, person_metadata):
-    identifier = bytes.fromhex(activated_account.blockchain_address)
+    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
     person = json.dumps(person_metadata)
     key = cache_data_key(identifier, ':cic.person')
     cache_data(key, person)
@@ -78,7 +78,7 @@ def cache_person_metadata(activated_account, init_cache, person_metadata):
 
 @pytest.fixture(scope='function')
 def cache_preferences(activated_account, init_cache, preferences):
-    identifier = bytes.fromhex(activated_account.blockchain_address)
+    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
     preferences = json.dumps(preferences)
     key = cache_data_key(identifier, ':cic.preferences')
     cache_data(key, preferences)
@@ -86,10 +86,10 @@ def cache_preferences(activated_account, init_cache, preferences):
 
 @pytest.fixture(scope='function')
 def cache_statement(activated_account, init_cache, statement):
-    identifier = bytes.fromhex(activated_account.blockchain_address)
+    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
-    statement = json.dumps(statement)
+    preferences = json.dumps(statement)
     key = cache_data_key(identifier, ':cic.statement')
-    cache_data(key, statement)
+    cache_data(key, preferences)
 
 
 @pytest.fixture(scope='function')

apps/cic-ussd/tests/fixtures/patches/account.py

@@ -41,22 +41,6 @@ def mock_async_balance_api_query(mocker):
     return query_args
 
 
-@pytest.fixture(scope='function')
-def mock_get_adjusted_balance(mocker, task_uuid):
-    query_args = {}
-
-    def get_adjusted_balance(self, token_symbol, balance, timestamp):
-        sync_res = mocker.patch('celery.result.AsyncResult')
-        sync_res.id = task_uuid
-        sync_res.result = 45931650.64654012
-        query_args['balance'] = balance
-        query_args['timestamp'] = timestamp
-        query_args['token_symbol'] = token_symbol
-        return sync_res
-    mocker.patch('cic_eth_aux.erc20_demurrage_token.api.Api.get_adjusted_balance', get_adjusted_balance)
-    return query_args
-
-
 @pytest.fixture(scope='function')
 def mock_notifier_api(mocker):
     sms = {}

apps/cic-ussd/tests/fixtures/transaction.py

@@ -7,7 +7,6 @@ import pytest
 # local import
 from cic_ussd.account.balance import get_cached_available_balance
 
-
 # tests imports
 
 
@@ -104,8 +103,8 @@ def transactions_list(activated_account, valid_recipient):
             'destination_token': '0x0000000000000000000000000000000000000000',
             'block_number': 80,
             'tx_index': 0,
-            'sender': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
+            'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
-            'recipient': '103d1ed6e370dBa6267045c70d4999384c18a04A',
+            'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A',
             'from_value': 0,
             'to_value': 0,
             'date_created': '2021-07-14T14:13:46.036198',
@@ -123,8 +122,8 @@ def transactions_list(activated_account, valid_recipient):
             'destination_token': '0x0000000000000000000000000000000000000000',
             'block_number': 78,
             'tx_index': 0,
-            'sender': 'b41BfEE260693A473254D62b81aE1ADCC9E51AFb',
+            'sender': '0xb41BfEE260693A473254D62b81aE1ADCC9E51AFb',
-            'recipient': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
+            'recipient': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
             'from_value': 1800000000000000,
             'to_value': 1800000000000000,
             'date_created': '2021-07-14T14:13:35.839638',
@@ -143,8 +142,8 @@ def transactions_list(activated_account, valid_recipient):
             'destination_token': '0x0000000000000000000000000000000000000000',
             'block_number': 79,
             'tx_index': 0,
-            'sender': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
+            'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
-            'recipient': '103d1ed6e370dBa6267045c70d4999384c18a04A',
+            'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A',
             'from_value': 0,
             'to_value': 0,
             'date_created': '2021-07-14T14:13:35.638355',
@@ -153,3 +152,4 @@ def transactions_list(activated_account, valid_recipient):
             'timestamp': 1626272015,
             'hash': '0x32ca3dd3bef06463b452f4d32f5f563d083cb4759219eed90f3d2a9c1791c5fc'}
     ]
+

apps/cic-ussd/tests/helpers/accounts.py

@@ -4,6 +4,7 @@ import random
 import uuid
 
 # external imports
+from chainlib.eth.address import to_checksum_address
 from faker import Faker
 from faker_e164.providers import E164Provider
 
@@ -20,7 +21,7 @@ def phone_number() -> str:
 
 
 def blockchain_address() -> str:
-    return os.urandom(20).hex().lower()
+    return to_checksum_address('0x' + os.urandom(20).hex())
 
 
 def session_id() -> str:

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -141,22 +141,12 @@ en:
         0. Back
       retry: |-
         %{retry_pin_entry}
-    account_balances:
+    account_balances: |-
-      available_balance: |-
+      CON Your balances are as follows:
-        CON Your balances are as follows:
+        balance: %{available_balance} %{token_symbol}
-          balance: %{available_balance} %{token_symbol}
+        fees: %{tax} %{token_symbol}
-        0. Back
+        rewards: %{bonus} %{token_symbol}
-      with_fees: |-
+      0. Back
-        CON Your balances are as follows:
-          balances: %{available_balance} %{token_symbol}
-          fees: %{tax} %{token_symbol}
-        0. Back
-      with_rewards: |-
-        CON Your balances are as follows:
-          balance: %{available_balance} %{token_symbol}
-          fees: %{tax} %{token_symbol}
-          rewards: %{bonus} %{token_symbol}
-        0. Back
     first_transaction_set: |-
       CON %{first_transaction_set}
       1. Next

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -140,22 +140,12 @@ sw:
         0. Nyuma
       retry: |-
         %{retry_pin_entry}
-    account_balances:
+    account_balances: |-
-      available_balance: |-
+      CON Salio zako ni zifuatazo:
-        CON Salio zako ni zifuatazo:
+        salio: %{available_balance} %{token_symbol}
-          salio: %{available_balance} %{token_symbol}
+        ushuru: %{tax} %{token_symbol}
-        0. Nyuma
+        tuzo: %{bonus} %{token_symbol}
-      with_fees: |-
+      0. Nyuma
-        CON Salio zako ni zifuatazo:
-          salio: %{available_balance} %{token_symbol}
-          ushuru: %{tax} %{token_symbol}
-        0. Nyuma
-      with_rewards: |-
-        CON Salio zako ni zifuatazo:
-          salio: %{available_balance} %{token_symbol}
-          ushuru: %{tax} %{token_symbol}
-          tuzo: %{bonus} %{token_symbol}
-        0. Nyuma
     first_transaction_set: |-
       CON %{first_transaction_set}
       1. Mbele

apps/contract-migration/.gitlab-ci.yml

@@ -1,25 +1,25 @@
-#.contract_migration_variables:
+.contract_migration_variables:
-#    variables:
+    variables:
-#        APP_NAME: contract-migration
+        APP_NAME: contract-migration
-#        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-#        CONTEXT: apps/$APP_NAME
+        CONTEXT: apps/$APP_NAME
-#
+
-#build-mr-contract-migration:
+build-mr-contract-migration:
-#    extends:
+    extends:
-#        - .py_build_merge_request
+        - .py_build_merge_request
-#        - .contract_migration_variables
+        - .contract_migration_variables
-#    rules:
+    rules:
-#    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-#      changes:
+      changes:
-#          - apps/contract-migration/**/*
+          - apps/contract-migration/**/*
-#      when: always
+      when: always
-#
+
-#build-push-contract-migration:
+build-push-contract-migration:
-#    extends:
+    extends:
-#        - .py_build_push
+        - .py_build_push
-#        - .contract_migration_variables
+        - .contract_migration_variables
-#    rules:
+    rules:
-#    - if: $CI_COMMIT_BRANCH == "master"
+    - if: $CI_COMMIT_BRANCH == "master"
-#      changes:
+      changes:
-#          - apps/contract-migration/**/*
+          - apps/contract-migration/**/*
-#      when: always
+      when: always

apps/contract-migration/config.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-
-set -a
-
-if [ -z $DEV_DATA_DIR ]; then
-	export DEV_DATA_DIR=`mktemp -d`
-else
-	mkdir -p $DEV_DATA_DIR
-fi
-
-if [ -z $DEV_CONFIG_RESET ]; then
-	if [ -f ${DEV_DATA_DIR}/env_reset ]; then
-		>&2 echo "importing existing configuration values from ${DEV_DATA_DIR}/env_reset"
-		. ${DEV_DATA_DIR}/env_reset
-	fi
-fi
-
-# Handle wallet
-export WALLET_KEY_FILE=${WALLET_KEY_FILE:-`realpath ./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c`}
-if [ ! -f $WALLET_KEY_FILE ]; then
-	>&2 echo "wallet path '$WALLET_KEY_FILE' does not point to a file"
-	exit 1
-fi
-export DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-checksum $(cat $WALLET_KEY_FILE | jq -r .address)`
-
-# Wallet dependent variable defaults
-export DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-export DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-export CIC_TRUST_ADDRESS=${CIC_TRUST_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
-export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-
-
-# Legacy variable defaults
-
-
-# Migration variable processing
-confini-dump --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset
-echo "export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS
-export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
-export WALLET_KEY_FILE=$WALLET_KEY_FILE
-" >> ${DEV_DATA_DIR}/env_reset
-
-cat ${DEV_DATA_DIR}/env_reset
-
-set +a

apps/contract-migration/config/config.ini

@@ -1,13 +0,0 @@
-[dev]
-eth_account_contract_deployer =
-eth_account_reserve_minter =
-eth_account_accounts_index_writer =
-reserve_amount = 10000000000000000000000000000000000
-faucet_amount = 0
-gas_amount = 100000000000000000000000
-token_amount = 100000000000000000000000
-eth_gas_price =
-data_dir =
-pip_extra_index_url = 
-eth_provider_host = 
-eth_provider_port = 

apps/contract-migration/docker/Dockerfile_ci

@@ -0,0 +1,44 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e
+
+WORKDIR /root
+
+RUN touch /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
+RUN cat /etc/apt/sources.list.d/ethereum.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
+
+#RUN apt-get install solc
+
+RUN mkdir -vp /usr/local/etc/cic
+
+ENV CONFINI_DIR /usr/local/etc/cic/
+
+
+COPY config_template/ /usr/local/etc/cic/
+COPY requirements.txt . 
+COPY override_requirements.txt . 
+
+ARG pip_index_url=https://pypi.org/simple
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_ARGS=""
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG pip_trusted_host=pypi.org
+RUN pip install --index-url https://pypi.org/simple \
+    pip install --index-url https://pypi.org/simple \
+    --pre \
+    --force-reinstall \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r requirements.txt 
+
+RUN pip install --index-url https://pypi.org/simple \
+    --force-reinstall \
+    --pre \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r override_requirements.txt 
+
+COPY . .
+RUN chmod +x *.sh

apps/contract-migration/envlist

@@ -56,7 +56,6 @@ ETH_PROVIDER
 ETH_ABI_DIR
 SIGNER_SOCKET_PATH
 SIGNER_SECRET
-SIGNER_PROVIDER
 CELERY_BROKER_URL
 CELERY_RESULT_URL
 META_PROVIDER

apps/contract-migration/requirements.txt

@@ -1,6 +1,6 @@
-cic-eth[tools]==0.12.4a8
+cic-eth[tools]==0.12.4a4
-chainlib-eth>=0.0.9a14,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
-eth-erc20>=0.1.2a3,<0.2.0
+eth-erc20>=0.1.2a2,<0.2.0
 erc20-demurrage-token>=0.0.5a2,<0.1.0
 eth-accounts-index>=0.1.2a2,<0.2.0
 eth-address-index>=0.2.3a4,<0.3.0
@@ -8,5 +8,3 @@ cic-eth-registry>=0.6.1a2,<0.7.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 erc20-faucet>=0.3.2a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0
-confini>=0.4.2rc3,<1.0.0
-crypto-dev-signer>=0.4.15a7,<=0.4.15