replace deploy image

docker vm builds

See merge request grassrootseconomics/cic-internal-integration!259

.gitignore

@@ -14,3 +14,4 @@ build/
 **/.venv
 .idea
 **/.vim
+**/*secret.yaml

.gitlab-ci.yml

@@ -1,14 +1,67 @@
 include:
-  - local: 'ci_templates/.cic-template.yml'
+  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
-  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  # these includes are app specific unit tests
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
+    #- local: 'apps/contract-migration/.gitlab-ci.yml'
+    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build
   - test
-  - release
+  - deploy 
+
+image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest  
+
+variables:
+  DOCKER_BUILDKIT: "1"
+  COMPOSE_DOCKER_CLI_BUILD: "1" 
+  CI_DEBUG_TRACE: "true"
+  TAG: $CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+
+     
+# runs on protected branches and pushes to repo
+build-push:
+  stage: build
+  tags:
+    - integration
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  script:
+    - TAG=$TAG sh ./scripts/build-push.sh
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+
+deploy-k8s-dev:
+  stage: deploy
+  image: line/kubectl-kustomize
+  variables:
+    CI_DEBUG_TRACE: "true"
+  script:
+    - kubectl config set-cluster k8s --server="${K8S_DEV_SERVER?dev server missing}"
+    - kubectl config set clusters.k8s.certificate-authority-data ${K8S_DEV_CERTIFICATE_AUTHORITY_DATA}
+    - kubectl config set-credentials gitlab --token="${K8S_DEV_USER_TOKEN}"
+    - kubectl config set-context grassroots --cluster=k8s --user=gitlab --namespace grassroots
+    - kubectl config use-context grassroots  
+      #- sed -i "s/<VERSION>/${CI_COMMIT_SHORT_SHA}/g" deployment.yaml
+      #- kubectl apply -f deployment.yaml
+    - echo "Wiping state..."
+    - kubectl delete jobs.batch --all 
+    - kubectl delete hr postgresql && kubectl delete pvc -l 'app.kubernetes.io/name=postgresql'
+    - kubectl delete sts,pvc -l 'app=bloxberg-validator'
+    - kubectl delete hr redis && kubectl delete pvc -l 'app=redis'
+    - kubectl apply -f kubernetes/eth-node/ -f kubernetes/postgresql/ -f kubernetes/redis/
+    - echo "deploy and run database migrations..."
+      # set image based on deploy tag
+    - bash ./scripts/set-image.sh
+    - kubectl apply -f . 
+    - echo "run contract migrations..."
+    - kubectl apply -f kubernetes/contract-migration/contract-migration-job.yaml
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+

apps/cic-base-os/Dockerfile

@@ -1,34 +0,0 @@
-# The solc image messes up the alpine environment, so we have to go all over again
-FROM python:3.8.6-slim-buster
-
-LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
-LABEL spdx-license-identifier="GPL-3.0-or-later"
-LABEL description="Base layer for buiding development images for the cic component suite"
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
-
-
-RUN echo installing nodejs tooling
-
-COPY ./dev/nvm.sh /root/
-
-# Install nvm with node and npm
-# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
-ENV NVM_DIR /root/.nvm
-ENV NODE_VERSION 15.3.0
-ENV BANCOR_NODE_VERSION 10.16.0
-
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
-	&& . $NVM_DIR/nvm.sh \
-	&& nvm install $NODE_VERSION \
-	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION \
-# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
-# owner of these files is "1001" by default - wtf
-	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
-ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
-

apps/cic-base-os/README.md

@@ -1 +0,0 @@
-## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-cache/.gitlab-ci.yml

@@ -1,52 +1,17 @@
-.cic_cache_variables:
+build-test-cic-cache:
-    variables:
+  stage: test 
-        APP_NAME: cic-cache
+  tags:
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    - integration
-        CONTEXT: apps/$APP_NAME
+  variables:
-
+    APP_NAME: cic-cache
-build-mr-cic-cache:
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-    extends:
+  script:
-        - .py_build_merge_request
+    - cd apps/cic-cache
-        - .cic_cache_variables
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-    rules:
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  allow_failure: true
-      changes:
+  rules:
-          - apps/cic-cache/**/*
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: always
+    changes:
-
+        - apps/$APP_NAME/**/*
-test-mr-cic-cache:
+    when: always
-    stage: test
-    extends:
-        - .cic_cache_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-cache"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
-
-build-push-cic-cache:
-    extends:
-        - .py_build_push
-        - .cic_cache_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-        
-

apps/cic-cache/README.md

@@ -0,0 +1 @@
+# CIC-CACHE

apps/cic-cache/cic_cache/version.py

@@ -5,7 +5,7 @@ version = (
         0,
         2,
         1,
-        'alpha.1',
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-cache/docker/Dockerfile_ci

@@ -1,38 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+

apps/cic-cache/requirements.txt

@@ -8,8 +8,8 @@ semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]>=0.0.6a1,<0.1.0
+chainsyncer[sql]>=0.0.6a3,<0.1.0
 erc20-faucet>=0.3.2a1, <0.4.0
-chainlib-eth>=0.0.9a3,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
-chainlib>=0.0.9a2,<0.1.0
+chainlib>=0.0.9a3,<0.1.0
 eth-address-index>=0.2.3a1,<0.3.0

apps/cic-eth/.gitlab-ci.yml

@@ -1,52 +1,16 @@
-.cic_eth_variables:
+build-test-cic-eth:
-    variables:
+  stage: test
-        APP_NAME: cic-eth
+  tags:
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    - integration
-        CONTEXT: apps/$APP_NAME
+  variables:
-
+    APP_NAME: cic-eth
-build-mr-cic-eth:
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-    extends:
+  script:
-        - .cic_eth_variables
+    - cd apps/cic-eth
-        - .py_build_target_dev
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-    rules:
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      #rules:
-      changes:
+      #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
-          - apps/cic-eth/**/*
+      #  changes:
-      when: always
+      #      - apps/$APP_NAME/**/*
-
+      #  when: always
-test-mr-cic-eth:
-    stage: test
-    extends:
-        - .cic_eth_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r admin_requirements.txt
-          -r services_requirements.txt
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-eth"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
-
-build-push-cic-eth:
-    extends:
-        - .py_build_push
-        - .cic_eth_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-eth/**/*
-      when: always

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
 cic-eth-registry>=0.6.1a2,<0.7.0
 hexathon~=0.0.1a8
-chainqueue>=0.0.3a2,<0.1.0
+chainqueue>=0.0.4a6,<0.1.0
 eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -4,7 +4,6 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from hexathon import (
         add_0x,
@@ -20,18 +19,17 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         )
 from cic_eth.error import LockedError
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-def normalize_address(a):
-    if a == None:
-        return None
-    return add_0x(hex_uniform(strip_0x(a)))
-
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -43,7 +41,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -53,7 +51,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -65,7 +63,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -75,7 +73,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -85,7 +83,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -93,7 +91,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -103,7 +101,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -111,7 +109,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -121,7 +119,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -129,7 +127,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -139,7 +137,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -148,12 +146,13 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    address = normalize_address(address)
+    if address != None:
+        address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
     if address != None:
         r |= Lock.check(chain_str, lock_flags, address=address, session=session)
     if r > 0:

apps/cic-eth/cic_eth/admin/nonce.py

@@ -33,6 +33,7 @@ from cic_eth.admin.ctrl import (
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
 from cic_eth.task import BaseTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -73,7 +74,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
+    query_address = tx_normalize.wallet_address(address)
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==query_address)

apps/cic-eth/cic_eth/api/admin.py

@@ -32,7 +32,6 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
 from eth_erc20 import ERC20
 
 # local imports
@@ -40,6 +39,7 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
+from cic_eth.queue.query import get_tx_local
 
 app = celery.current_app
 
@@ -284,7 +284,7 @@ class AdminApi:
         tx_hash_hex = None
         session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
+            tx_dict = get_tx_local(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
         session.close()

apps/cic-eth/cic_eth/db/models/lock.py

@@ -4,12 +4,12 @@ import logging
 
 # third-party imports
 from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode  import ZERO_ADDRESS_NORMAL
 
 logg = logging.getLogger()
 
@@ -37,7 +37,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
+    def set(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None, tx_hash=None):
         """Sets flags associated with the given address and chain.
 
         If a flags entry does not exist it is created.
@@ -90,7 +90,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def reset(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Resets flags associated with the given address and chain.
 
         If the resulting flags entry value is 0, the entry will be deleted.
@@ -134,7 +134,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def check(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Checks whether all given flags are set for given address and chain. 
 
         Does not validate the address against any other tables or components.

apps/cic-eth/cic_eth/encode.py

@@ -0,0 +1,16 @@
+# external imports
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainqueue.encode import TxHexNormalizer
+from chainlib.eth.tx import unpack
+
+tx_normalize = TxHexNormalizer()
+
+ZERO_ADDRESS_NORMAL = tx_normalize.wallet_address(ZERO_ADDRESS)
+
+
+def unpack_normal(signed_tx_bytes, chain_spec):
+    tx = unpack(signed_tx_bytes, chain_spec)
+    tx['hash'] = tx_normalize.tx_hash(tx['hash'])
+    tx['from'] = tx_normalize.wallet_address(tx['from'])
+    tx['to'] = tx_normalize.wallet_address(tx['to'])
+    return tx

apps/cic-eth/cic_eth/eth/account.py

@@ -14,10 +14,7 @@ from chainlib.eth.sign import (
         sign_message,
         )
 from chainlib.eth.address import to_checksum_address
-from chainlib.eth.tx import (
+from chainlib.eth.tx import TxFormat
-        TxFormat,
-        unpack,
-        )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
@@ -49,6 +46,10 @@ from cic_eth.eth.nonce import (
 from cic_eth.queue.tx import (
         register_tx,
         )
+from cic_eth.encode import (
+        unpack_normal,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 logg = logging.getLogger()
 celery_app = celery.current_app 
@@ -295,17 +296,17 @@ def cache_gift_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = Faucet.parse_give_to_request(tx['data'])
 
     session = self.create_session()
 
     tx_dict = {
-            'hash': tx_hash_hex,
+            'hash': tx['hash'],
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
+            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
             'to_value': 0,
             }
@@ -334,17 +335,17 @@ def cache_account_data(
     :rtype: tuple
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_dict = {
-            'hash': tx_hash_hex,
+            'hash': tx['hash'],
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
+            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
             'to_value': 0,
             }

apps/cic-eth/cic_eth/eth/gas.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import celery
 from hexathon import strip_0x
-from chainlib.eth.constant import ZERO_ADDRESS
+#from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
@@ -21,7 +21,6 @@ from chainlib.eth.error import (
 from chainlib.eth.tx import (
         TxFactory,
         TxFormat,
-        unpack,
         )
 from chainlib.eth.contract import (
         abi_decode_single,
@@ -45,6 +44,7 @@ from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.queue.tx import (
         queue_create,
         register_tx,
+        unpack,
         )
 from cic_eth.queue.query import get_tx
 from cic_eth.task import (
@@ -53,6 +53,11 @@ from cic_eth.task import (
         CriticalSQLAlchemyAndSignerTask,
         CriticalWeb3AndSignerTask,
         )
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        unpack_normal,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -66,6 +71,7 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
+#def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 
@@ -130,16 +136,16 @@ def cache_gas_data(
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
 
     session = SessionBase.create_session()
 
     tx_dict = {
-            'hash': tx_hash_hex,
+            'hash': tx['hash'],
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
+            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': tx['value'],
             'to_value': tx['value'],
             }
@@ -150,7 +156,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -170,6 +176,20 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
+    if address != None:
+        if not is_checksum_address(address):
+            raise ValueError('invalid address {}'.format(address))
+        address = tx_normalize.wallet_address(address)
+
+    tx_hashes = []
+    txs = []
+    for tx_hash in tx_hashes_hex:
+        tx_hash = tx_normalize.tx_hash(tx_hash)
+        tx_hashes.append(tx_hash)
+    for tx in txs_hex:
+        tx = tx_normalize.tx_wire(tx)
+        txs.append(tx)
+
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
 
@@ -187,9 +207,6 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
                 raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
             addresspass.append(address)
 
-    if not is_checksum_address(address):
-        raise ValueError('invalid address {}'.format(address))
-
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
@@ -304,6 +321,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # Determine value of gas tokens to send
     # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
     # however, we will perform a 0-value transaction instead
+    recipient_address = tx_normalize.wallet_address(recipient_address)
     zero_amount = False
     session = SessionBase.create_session()
     status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
@@ -378,6 +396,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     :returns: Transaction hash
     :rtype: str, 0x-hex
     """
+    txold_hash_hex = tx_normalize.tx_hash(txold_hash_hex)
     session = SessionBase.create_session()
 
     otx = Otx.load(txold_hash_hex, session)

apps/cic-eth/cic_eth/queue/balance.py

@@ -15,6 +15,7 @@ from chainqueue.db.enum import (
 # local imports
 from cic_eth.db import SessionBase
 from cic_eth.task import CriticalSQLAlchemyTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -22,6 +23,9 @@ logg = logging.getLogger()
 
 
 def __balance_outgoing_compatible(token_address, holder_address):
+    token_address = tx_normalize.executable_address(token_address)
+    holder_address = tx_normalize.wallet_address(holder_address)
+
     session = SessionBase.create_session()
     q = session.query(TxCache.from_value)
     q = q.join(Otx)
@@ -58,6 +62,9 @@ def balance_outgoing(tokens, holder_address, chain_spec_dict):
 
 
 def __balance_incoming_compatible(token_address, receiver_address):
+    token_address = tx_normalize.executable_address(token_address)
+    receiver_address = tx_normalize.wallet_address(receiver_address)
+
     session = SessionBase.create_session()
     q = session.query(TxCache.to_value)
     q = q.join(Otx)
@@ -110,7 +117,7 @@ def assemble_balances(balances_collection):
     logg.debug('received collection {}'.format(balances_collection))
     for c in balances_collection:
         for b in c:
-            address = b['address']
+            address = tx_normalize.executable_address(b['address'])
             if tokens.get(address) == None:
                 tokens[address] = {
                     'address': address,

apps/cic-eth/cic_eth/queue/lock.py

@@ -6,6 +6,7 @@ import celery
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db import SessionBase
 from cic_eth.db.models.lock import Lock
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -21,6 +22,9 @@ def get_lock(address=None):
     :returns: List of locks
     :rtype: list of dicts
     """
+    if address != None:
+        address = tx_normalize.wallet_address(address)
+
     session = SessionBase.create_session()
     q = session.query(
             Lock.date_created,

apps/cic-eth/cic_eth/queue/query.py

@@ -4,8 +4,8 @@ import datetime
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.eth.tx import unpack
 import chainqueue.sql.query
+from chainlib.eth.tx import unpack
 from chainqueue.db.enum import (
         StatusEnum,
         is_alive,
@@ -20,6 +20,10 @@ from cic_eth.db.enum import LockEnum
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.lock import Lock
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import (
+        tx_normalize,
+        unpack_normal,
+        )
 
 celery_app = celery.current_app
 
@@ -27,49 +31,76 @@ celery_app = celery.current_app
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx_cache(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    session = SessionBase.create_session()
+    return get_tx_cache_local(chain_spec, tx_hash)
+
+
+def get_tx_cache_local(chain_spec, tx_hash, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    session = SessionBase.bind_session(session)
     r = chainqueue.sql.query.get_tx_cache(chain_spec, tx_hash, session=session)
-    session.close()
+    SessionBase.release_session(session)
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    session = SessionBase.create_session()
+    return get_tx_local(chain_spec, tx_hash)
+
+
+def get_tx_local(chain_spec, tx_hash, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    session = SessionBase.bind_session(session)
     r =  chainqueue.sql.query.get_tx(chain_spec, tx_hash, session=session)
-    session.close()
+    SessionBase.release_session(session)
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    session = SessionBase.create_session()
+    return get_account_tx_local(chain_spec, address, as_sender=as_sender, as_recipient=as_recipient, counterpart=counterpart)
+
+
+def get_account_tx_local(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=None):
+    address = tx_normalize.wallet_address(address)
+    session = SessionBase.bind_session(session)
     r = chainqueue.sql.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
-    session.close()
+    SessionBase.release_session(session)
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
+def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    return get_upcoming_tx_nolock_local(chain_spec, status=status, not_status=not_status, recipient=recipient, before=before, limit=limit)
+
+
+def get_upcoming_tx_nolock_local(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
+    recipient = tx_normalize.wallet_address(recipient)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
+    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack_normal)
     session.close()
     return r
 
 
 def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
-    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
+    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack_normal)
 
 
 def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
-    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
+    sender = tx_normalize.wallet_address(sender)
+    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack_normal)
 
 
 def get_nonce_tx(chain_spec, nonce, sender):
-    return get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack)
+    sender = tx_normalize.wallet_address(sender)
+    return get_nonce_tx_local(chain_spec, nonce, sender)
+
+
+def get_nonce_tx_local(chain_spec, nonce, sender, session=None):
+    sender = tx_normalize.wallet_address(sender)
+    return chainqueue.sql.query.get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack_normal, session=session)
 
 
 def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
@@ -91,6 +122,8 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     :returns: Transactions
     :rtype: dict, with transaction hash as key, signed raw transaction as value
     """
+    if recipient != None:
+        recipient = tx_normalize.wallet_address(recipient)
     session = SessionBase.bind_session(session)
     q_outer = session.query(
             TxCache.sender,

apps/cic-eth/cic_eth/queue/state.py

@@ -6,12 +6,14 @@ import chainqueue.sql.state
 import celery
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_sent(chain_spec_dict, tx_hash, fail=False):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_sent(chain_spec, tx_hash, fail, session=session)
@@ -21,6 +23,7 @@ def set_sent(chain_spec_dict, tx_hash, fail=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
@@ -30,6 +33,7 @@ def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_cancel(chain_spec_dict, tx_hash, manual=False):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_cancel(chain_spec, tx_hash, manual, session=session)
@@ -39,6 +43,7 @@ def set_cancel(chain_spec_dict, tx_hash, manual=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_rejected(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_rejected(chain_spec, tx_hash, session=session)
@@ -48,6 +53,7 @@ def set_rejected(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_fubar(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_fubar(chain_spec, tx_hash, session=session)
@@ -57,6 +63,7 @@ def set_fubar(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_manual(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_manual(chain_spec, tx_hash, session=session)
@@ -66,6 +73,7 @@ def set_manual(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_ready(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_ready(chain_spec, tx_hash, session=session)
@@ -75,6 +83,7 @@ def set_ready(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_reserved(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_reserved(chain_spec, tx_hash, session=session)
@@ -84,6 +93,7 @@ def set_reserved(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_waitforgas(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_waitforgas(chain_spec, tx_hash, session=session)
@@ -93,6 +103,7 @@ def set_waitforgas(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_state_log(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.get_state_log(chain_spec, tx_hash, session=session)
@@ -102,6 +113,7 @@ def get_state_log(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def obsolete(chain_spec_dict, tx_hash, final):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)

apps/cic-eth/cic_eth/queue/time.py

@@ -13,6 +13,7 @@ from chainqueue.error import NotLocalTxError
 # local imports
 from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -20,6 +21,7 @@ logg = logging.getLogger()
 
 
 def tx_times(tx_hash, chain_spec, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
 
     session = SessionBase.bind_session(session)
 

apps/cic-eth/cic_eth/queue/tx.py

@@ -32,12 +32,16 @@ from cic_eth.db import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.error import LockedError
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
 def queue_create(chain_spec, nonce, holder_address, tx_hash, signed_tx, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    signed_tx = tx_normalize.tx_hash(signed_tx)
+    holder_address = tx_normalize.wallet_address(holder_address)
     session = SessionBase.bind_session(session)
 
     lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session) 
@@ -67,6 +71,8 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No
     :returns: Tuple; Transaction hash, signed raw transaction data
     :rtype: tuple
     """
+    tx_hash_hex = tx_normalize.tx_hash(tx_hash_hex)
+    tx_signed_raw_hex = tx_normalize.tx_hash(tx_signed_raw_hex)
     logg.debug('adding queue tx {}:{} -> {}'.format(chain_spec, tx_hash_hex, tx_signed_raw_hex))
     tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex))
     tx = unpack(tx_signed_raw, chain_spec)

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -10,15 +10,14 @@ from chainlib.eth.tx import unpack
 from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
-from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
 from chainlib.eth.address import to_checksum_address
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.queue.query import get_paused_tx
 from .base import SyncFilter
 
-#logg = logging.getLogger().getChild(__name__)
 logg = logging.getLogger()
 
 

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         4,
-        'alpha.4',
+        'alpha.7',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile_ci

@@ -1,71 +0,0 @@
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-WORKDIR /usr/src/cic-eth
-
-# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
-# do load these all into setup.py later
-# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
-#COPY cic-eth/requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
-#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-#    pip install --index-url https://pypi.org/simple \
-#    --force-reinstall \
-#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-#    -r requirements.txt 
-COPY *requirements.txt . 
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    -r requirements.txt \
-    -r services_requirements.txt \
-    -r admin_requirements.txt
-    
-COPY . .
-RUN python setup.py install
-
-COPY docker/entrypoints/* ./ 
-RUN chmod 755 *.sh
-
-# # ini files in config directory defines the configurable parameters for the application
-# # they can all be overridden by environment variables
-# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-ENTRYPOINT []
-
-# ------------------ PRODUCTION CONTAINER ----------------------
-#FROM python:3.8.6-slim-buster as prod 
-#
-#RUN apt-get update && \
-#	apt install -y gnupg libpq-dev procps
-#
-#WORKDIR /root
-#
-#COPY --from=dev /usr/local/bin/ /usr/local/bin/
-#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
-#                          /usr/local/lib/python3.8/site-packages/
-#
-#COPY docker/entrypoints/* ./ 
-#RUN chmod 755 *.sh
-#
-## # ini files in config directory defines the configurable parameters for the application
-## # they can all be overridden by environment variables
-## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-eth/
-#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-#COPY scripts/ scripts/ 
-#
-## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-##COPY util/liveness/health.sh /usr/local/bin/health.sh
-#
-#ENTRYPOINT []
-#

apps/cic-eth/docker/run_tests.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r admin_requirements.txt \
+-r services_requirements.txt \
+-r test_requirements.txt  
+
+export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.9a3,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
 semver==2.13.0

apps/cic-eth/services_requirements.txt

@@ -1,5 +1,5 @@
-chainqueue>=0.0.3a2,<0.1.0
+chainqueue>=0.0.5a1,<0.1.0
-chainsyncer[sql]>=0.0.6a1,<0.1.0
+chainsyncer[sql]>=0.0.6a3,<0.1.0
 alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 redis==3.5.3

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -1,7 +1,6 @@
 # external imports
 from chainlib.connection import RPCConnection
 from chainlib.eth.nonce import OverrideNonceOracle
-from chainqueue.sql.tx import create as queue_create
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -26,6 +25,8 @@ from chainqueue.db.enum import StatusBits
 # local imports
 from cic_eth.runnable.daemons.filters.gas import GasFilter
 from cic_eth.eth.gas import cache_gas_data
+from cic_eth.encode import tx_normalize
+from cic_eth.queue.tx import queue_create
 
 
 def test_filter_gas(

apps/cic-eth/tests/filters/test_register_filter.py

@@ -22,10 +22,11 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+from cic_eth.encode import tx_normalize
+from cic_eth.queue.query import get_account_tx_local
 
 logg = logging.getLogger()
 
@@ -79,7 +80,7 @@ def test_register_filter(
     t.get_leaf()
     assert t.successful()
 
-    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    gift_txs = get_account_tx_local(default_chain_spec, agent_roles['ALICE'], as_sender=True, session=init_database)
     ks = list(gift_txs.keys())
     assert len(ks) == 1
 

apps/cic-eth/tests/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+-r admin_requirements.txt
+-r services_requirements.txt
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests

apps/cic-eth/tests/task/api/test_admin.py

@@ -34,10 +34,6 @@ from chainqueue.sql.state import (
         set_ready,
         set_reserved,
         )
-from chainqueue.sql.query import (
-        get_tx,
-        get_nonce_tx_cache,
-        )
 
 # local imports
 from cic_eth.api.admin import AdminApi
@@ -46,6 +42,11 @@ from cic_eth.db.enum import LockEnum
 from cic_eth.error import InitializationError
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.queue.tx import queue_create
+from cic_eth.queue.query import (
+        get_tx,
+        get_nonce_tx_local,
+        )
+from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger()
 
@@ -286,13 +287,15 @@ def test_fix_nonce(
     assert t.successful()
 
     init_database.commit()
-    
+   
-    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
+    logg.debug('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+    txs = get_nonce_tx_local(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
     ks = txs.keys()
     assert len(ks) == 2
 
     for k in ks:
-        hsh = add_0x(k)
+        #hsh = add_0x(k)
+        hsh = tx_normalize.tx_hash(k)
         otx = Otx.load(hsh, session=init_database)
         init_database.refresh(otx)
         logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -30,7 +30,6 @@ from chainqueue.sql.state import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.query import get_nonce_tx_cache
 from eth_erc20 import ERC20
 from cic_eth_registry import CICRegistry
 
@@ -38,6 +37,7 @@ from cic_eth_registry import CICRegistry
 from cic_eth.api.admin import AdminApi
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.eth.erc20 import cache_transfer_data
+from cic_eth.queue.query import get_nonce_tx_local
 
 logg = logging.getLogger()
 
@@ -312,7 +312,7 @@ def test_resend_inplace(
     otx = Otx.load(tx_hash_hex, session=init_database)
     assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
 
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_local(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 2
 
 
@@ -363,10 +363,10 @@ def test_resend_clone(
     assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
     assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
 
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_local(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 1
 
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_local(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 1
 
     otx = Otx.load(txs[0], session=init_database)

apps/cic-eth/tests/task/test_task_gas.py

@@ -21,7 +21,6 @@ from chainlib.eth.constant import (
         MINIMUM_FEE_UNITS,
         MINIMUM_FEE_PRICE,
         )
-from chainqueue.sql.tx import create as queue_create
 from chainqueue.sql.query import get_tx
 from chainqueue.db.enum import StatusBits
 from chainqueue.sql.state import (
@@ -35,6 +34,7 @@ from hexathon import strip_0x
 # local imports
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.error import OutOfGasError
+from cic_eth.queue.tx import queue_create
 
 logg = logging.getLogger()
 

apps/cic-eth/tests/task/test_task_list.py

@@ -51,6 +51,7 @@ def test_ext_tx_collate(
         tx_hash_hex,
         tx_signed_raw_hex,
         )
+    otx.block = 666
     init_database.add(otx)
     init_database.commit()
 

apps/cic-eth/tests/unit/db/test_tx.py

@@ -46,6 +46,7 @@ def test_set(
         tx_hash_hex,
         tx_signed_raw_hex,
         )
+    otx.block = 666
     init_database.add(otx)
     init_database.commit()
 
@@ -74,7 +75,6 @@ def test_set(
     assert (tx_stored.destination_token_address == ZERO_ADDRESS)
     assert (tx_stored.from_value == tx['value'])
     assert (tx_stored.to_value == to_value)
-    assert (tx_stored.block_number == 666)
     assert (tx_stored.tx_index == 13)
 
 

apps/cic-eth/tests/unit/queue/test_balances.py

@@ -13,6 +13,7 @@ from cic_eth.queue.balance import (
         balance_incoming,
         assemble_balances,
         )
+from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger()
 
@@ -51,8 +52,8 @@ def test_assemble():
     r = assemble_balances(b)
     logg.debug('r {}'.format(r))
 
-    assert r[0]['address'] == token_foo
+    assert r[0]['address'] == tx_normalize.executable_address(token_foo)
-    assert r[1]['address'] == token_bar
+    assert r[1]['address'] == tx_normalize.executable_address(token_bar)
     assert r[0].get('balance_foo') != None
     assert r[0].get('balance_bar') != None
     assert r[1].get('balance_baz') != None
@@ -74,11 +75,11 @@ def test_outgoing_balance(
     token_address = '0x' + os.urandom(20).hex()
     sender = '0x' + os.urandom(20).hex()
     txc = TxCache(
-            tx_hash,
+            tx_normalize.tx_hash(tx_hash),
-            sender,
+            tx_normalize.wallet_address(sender),
-            recipient,
+            tx_normalize.wallet_address(recipient),
-            token_address,
+            tx_normalize.executable_address(token_address),
-            token_address,
+            tx_normalize.executable_address(token_address),
             1000,
             1000,
             session=init_database,
@@ -125,11 +126,11 @@ def test_incoming_balance(
     token_address = '0x' + os.urandom(20).hex()
     sender = '0x' + os.urandom(20).hex()
     txc = TxCache(
-            tx_hash,
+            tx_normalize.tx_hash(tx_hash),
-            sender,
+            tx_normalize.wallet_address(sender),
-            recipient,
+            tx_normalize.wallet_address(recipient),
-            token_address,
+            tx_normalize.executable_address(token_address),
-            token_address,
+            tx_normalize.executable_address(token_address),
             1000,
             1000,
             session=init_database,

apps/cic-eth/tests/unit/queue/test_query.py

@@ -21,6 +21,7 @@ from cic_eth.db.models.lock import Lock
 from cic_eth.queue.query import get_upcoming_tx
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import cache_gas_data
+from cic_eth.encode import tx_normalize
 
 # test imports
 from tests.util.nonce import StaticNonceOracle
@@ -39,8 +40,8 @@ def test_upcoming_with_lock(
     gas_oracle = RPCGasOracle(eth_rpc)
     c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
 
-    alice_normal = add_0x(hex_uniform(strip_0x(agent_roles['ALICE'])))
+    alice_normal = tx_normalize.wallet_address(agent_roles['ALICE'])
-    bob_normal = add_0x(hex_uniform(strip_0x(agent_roles['BOB'])))
+    bob_normal = tx_normalize.wallet_address(agent_roles['BOB'])
 
     (tx_hash_hex, tx_rpc) = c.create(alice_normal, bob_normal, 100 * (10 ** 6))
     tx_signed_raw_hex = tx_rpc['params'][0]

apps/cic-eth/tests/unit/queue/test_queue_lock.py

@@ -9,7 +9,7 @@ from cic_eth.db.models.lock import Lock
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import LockedError
 from cic_eth.queue.tx import queue_create
-
+from cic_eth.encode import tx_normalize
 
 def test_queue_lock(
     init_database,
@@ -21,6 +21,8 @@ def test_queue_lock(
     address = '0x' + os.urandom(20).hex()
     tx_hash = '0x' + os.urandom(32).hex()
     tx_raw = '0x' + os.urandom(128).hex()
+    address_normal = tx_normalize.wallet_address(address)
+    tx_hash_normal = tx_normalize.tx_hash(tx_hash)
 
     Lock.set(chain_str, LockEnum.QUEUE)
     with pytest.raises(LockedError):
@@ -32,7 +34,7 @@ def test_queue_lock(
                 tx_raw,
                 )
 
-    Lock.set(chain_str, LockEnum.QUEUE, address=address)
+    Lock.set(chain_str, LockEnum.QUEUE, address=address_normal)
     with pytest.raises(LockedError):
         queue_create(
                 default_chain_spec,
@@ -52,7 +54,7 @@ def test_queue_lock(
                 tx_raw,
                 )
 
-    Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
+    Lock.set(chain_str, LockEnum.QUEUE, address=address_normal, tx_hash=tx_hash_normal)
     with pytest.raises(LockedError):
         queue_create(
                 default_chain_spec,
@@ -61,5 +63,3 @@ def test_queue_lock(
                 tx_hash,
                 tx_raw,
                 )
-
-

apps/cic-eth/tools_requirements.txt

@@ -1,5 +1,5 @@
 crypto-dev-signer>=0.4.15a1,<=0.4.15
-chainqueue>=0.0.3a1,<0.1.0
+chainqueue>=0.0.5a1,<0.1.0
 cic-eth-registry>=0.6.1a2,<0.7.0
 redis==3.5.3
 hexathon~=0.0.1a8

apps/cic-meta/.gitlab-ci.yml

@@ -1,43 +1,16 @@
-
+build-test-cic-meta:
-.cic_meta_variables:
+  stage: test
-    variables:
+  tags:
-        APP_NAME: cic-meta
+   - integration
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+  variables:
-        CONTEXT: apps/$APP_NAME
+    APP_NAME: cic-meta
-
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-build-mr-cic-meta:
+  script:
-    extends:
+    - cd apps/cic-meta
-        - .py_build_merge_request
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-        - .cic_meta_variables
+    - docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh
-    rules:
+     #rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+     #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
+     #  changes:
-          - apps/cic-meta/**/*
+     #      - apps/$APP_NAME/**/*
-      when: always
+     #  when: always
-
-test-mr-cic-meta:
-    extends:
-        - .cic_meta_variables
-    stage: test
-    image: $MR_IMAGE_TAG 
-    script:
-        - cd /root
-        - npm install --dev 
-        - npm run test
-        - npm run test:coverage
-    needs: ["build-mr-cic-meta"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-meta/**/*
-      when: always
-
-build-push-cic-meta:
-    extends:
-        - .py_build_push
-        - .cic_meta_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-meta/**/*
-      when: always

apps/cic-meta/docker/Dockerfile

@@ -15,11 +15,10 @@ RUN --mount=type=cache,mode=0755,target=/root/.npm \
 COPY webpack.config.js .
 COPY tsconfig.json . 
 ## required to build the cic-client-meta module
-COPY src/ src/
+COPY . .
-COPY scripts/ scripts/
-COPY tests/ tests/
 COPY tests/*.asc /root/pgp/
 
+
 ## copy runtime configs
 COPY .config/ /usr/local/etc/cic-meta/
 #

apps/cic-meta/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-#FROM node:15.3.0-alpine3.10
-FROM node:lts-alpine3.14
-
-WORKDIR /root
-
-RUN apk add --no-cache postgresql bash
-
-# copy the dependencies
-COPY package.json package-lock.json .
-RUN npm set cache /root/.npm && \
-    npm ci
-
-COPY webpack.config.js .
-COPY tsconfig.json . 
-## required to build the cic-client-meta module
-COPY src/ src/
-COPY scripts/ scripts/
-COPY tests/ tests/
-COPY tests/*.asc /root/pgp/
-
-## copy runtime configs
-COPY .config/ /usr/local/etc/cic-meta/
-#
-## db migrations
-COPY docker/db.sh ./db.sh
-RUN chmod 755 ./db.sh
-#
-RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY docker/start_server.sh ./start_server.sh
-RUN chmod 755 ./start_server.sh
-ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/run_tests.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+set -e
+
+npm install --dev 
+npm run test
+npm run test:coverage

apps/cic-notify/.gitlab-ci.yml

@@ -1,52 +1,17 @@
-.cic_notify_variables:
+build-test-cic-notify:
-    variables:
+  stage: test
-        APP_NAME: cic-notify
+  tags:
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    - integration
-        CONTEXT: apps/$APP_NAME
+  variables:
-
+    APP_NAME: cic-notify
-build-mr-cic-notify:
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-    extends:
+  script:
-        - .py_build_merge_request
+    - cd apps/cic-notify
-        - .cic_notify_variables
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-    rules:
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  allow_failure: true
-      changes:
+  rules:
-          - apps/cic-notify/**/*
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: always
+    changes:
-
+        - apps/$APP_NAME/**/*
-test-mr-cic-notify:
+    when: always
-    stage: test
-    extends:
-        - .cic_notify_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-notify"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
-
-build-push-cic-notify:
-    extends:
-        - .py_build_push
-        - .cic_notify_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-notify/**/*
-      when: always
-        
-

apps/cic-notify/docker/Dockerfile_ci

@@ -1,27 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-COPY requirements.txt .
-
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . .
-
-RUN python setup.py install
-
-COPY docker/*.sh .
-RUN chmod +x *.sh 
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY .config/ /usr/local/etc/cic-notify/
-COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
-
-ENTRYPOINT []

apps/cic-notify/docker/run_tests.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests

apps/cic-ussd/.gitlab-ci.yml

@@ -1,52 +1,16 @@
-.cic_ussd_variables:
+build-test-cic-ussd:
-    variables:
+  stage: test
-        APP_NAME: cic-ussd
+  tags:
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+  - integration
-        CONTEXT: apps/$APP_NAME
+  variables:
-
+    APP_NAME: cic-ussd
-build-mr-cic-ussd:
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-    extends:
+  script:
-        - .py_build_merge_request
+    - cd apps/cic-ussd
-        - .cic_ussd_variables
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-    rules:
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  rules:
-      changes:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-          - apps/cic-ussd/**/*
+    changes:
-      when: always
+        - apps/$APP_NAME/**/*
-
+    when: always
-test-mr-cic-ussd:
-    stage: test
-    extends:
-        - .cic_ussd_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd
-    needs: ["build-mr-cic-ussd"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
-
-build-push-cic-ussd:
-    extends:
-        - .py_build_push
-        - .cic_ussd_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-ussd/**/*
-      when: always
-
-

apps/cic-ussd/cic_ussd/account/balance.py

@@ -20,7 +20,7 @@ def get_balances(address: str,
                  asynchronous: bool = False,
                  callback_param: any = None,
                  callback_queue='cic-ussd',
-                 callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Optional[list]:
+                 callback_task='cic_ussd.tasks.callback_handler.balances_callback') -> Optional[list]:
     """This function queries cic-eth for an account's balances, It provides a means to receive the balance either
     asynchronously or synchronously.. It returns a dictionary containing the network, outgoing and incoming balances.
     :param address: Ethereum address of an account.

apps/cic-ussd/cic_ussd/account/transaction.py

@@ -117,18 +117,18 @@ def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
     return recipient_transaction_data, sender_transaction_data
 
 
-def validate_transaction_account(session: Session, transaction: dict) -> Account:
+def validate_transaction_account(blockchain_address: str, role: str, session: Session) -> Account:
     """This function checks whether the blockchain address specified in a parsed transaction object resolves to an
     account object in the ussd system.
-    :param session: Database session object.
+    :param blockchain_address:
-    :type session: Session
+    :type blockchain_address:
-    :param transaction: Parsed transaction data object.
+    :param role:
-    :type transaction: dict
+    :type role:
+    :param session:
+    :type session:
     :return:
     :rtype:
     """
-    blockchain_address = transaction.get('blockchain_address')
-    role = transaction.get('role')
     session = SessionBase.bind_session(session)
     account = session.query(Account).filter_by(blockchain_address=blockchain_address).first()
     if not account:

apps/cic-ussd/cic_ussd/processor/util.py

@@ -67,6 +67,7 @@ def resume_last_ussd_session(last_state: str) -> Document:
         'exit',
         'exit_invalid_pin',
         'exit_invalid_new_pin',
+        'exit_invalid_recipient',
         'exit_invalid_request',
         'exit_pin_blocked',
         'exit_pin_mismatch',

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -4,6 +4,7 @@ from typing import Tuple
 
 # third party imports
 import celery
+from phonenumbers.phonenumberutil import NumberParseException
 
 # local imports
 from cic_ussd.account.balance import get_cached_available_balance
@@ -21,23 +22,21 @@ logg = logging.getLogger(__file__)
 
 
 def is_valid_recipient(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
-    """This function checks that a user exists, is not the initiator of the transaction, has an active account status
+    """This function checks that a phone number provided as the recipient of a transaction does not match the sending
-    and is authorized to perform  standard transactions.
+    party's own phone number.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
-    :return: A user's validity
+    :return: A recipient account's validity for a transaction
     :rtype: bool
     """
     user_input, ussd_session, account, session = state_machine_data
-    phone_number = process_phone_number(user_input, E164Format.region)
+    try:
-    session = SessionBase.bind_session(session=session)
+        phone_number = process_phone_number(user_input, E164Format.region)
-    recipient = Account.get_by_phone_number(phone_number=phone_number, session=session)
+    except NumberParseException:
-    SessionBase.release_session(session=session)
+        phone_number = None
     is_not_initiator = phone_number != account.phone_number
-    has_active_account_status = False
+    is_present = Account.get_by_phone_number(phone_number, session) is not None
-    if recipient:
+    return phone_number is not None and phone_number.startswith('+') and is_present and is_not_initiator
-        has_active_account_status = recipient.get_status(session) == AccountStatus.ACTIVE.name
-    return is_not_initiator and has_active_account_status and recipient is not None
 
 
 def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -138,7 +138,7 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     balances_data = result[0]
     available_balance = calculate_available_balance(balances_data)
     transaction = param
-    blockchain_address = param.get('blockchain_address')
+    blockchain_address = transaction.get('blockchain_address')
     transaction['available_balance'] = available_balance
     queue = self.request.delivery_info.get('routing_key')
 
@@ -150,10 +150,10 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     )
     s_notify_account = celery.signature('cic_ussd.tasks.notifications.transaction', queue=queue)
 
-    if param.get('transaction_type') == 'transfer':
+    if transaction.get('transaction_type') == 'transfer':
         celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async()
 
-    if param.get('transaction_type') == 'tokengift':
+    if transaction.get('transaction_type') == 'tokengift':
         s_process_account_metadata = celery.signature(
             'cic_ussd.tasks.processor.parse_transaction', [{}, transaction], queue=queue
         )
@@ -184,10 +184,11 @@ def transaction_callback(result: dict, param: str, status_code: int):
     source_token_value = result.get('source_token_value')
 
     recipient_metadata = {
-        "token_symbol": destination_token_symbol,
+        "alt_blockchain_address": sender_blockchain_address,
-        "token_value": destination_token_value,
         "blockchain_address": recipient_blockchain_address,
         "role": "recipient",
+        "token_symbol": destination_token_symbol,
+        "token_value": destination_token_value,
         "transaction_type": param
     }
 
@@ -201,10 +202,11 @@ def transaction_callback(result: dict, param: str, status_code: int):
 
     if param == 'transfer':
         sender_metadata = {
+            "alt_blockchain_address": recipient_blockchain_address,
             "blockchain_address": sender_blockchain_address,
+            "role": "sender",
             "token_symbol": source_token_symbol,
             "token_value": source_token_value,
-            "role": "sender",
             "transaction_type": param
         }
 

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -29,7 +29,8 @@ def transaction(notification_data: dict):
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')
     token_symbol = notification_data.get('token_symbol')
-    transaction_account_metadata = notification_data.get('metadata_id')
+    alt_metadata_id = notification_data.get('alt_metadata_id')
+    metadata_id = notification_data.get('metadata_id')
     transaction_type = notification_data.get('transaction_type')
     timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
 
@@ -47,7 +48,8 @@ def transaction(notification_data: dict):
                                            preferred_language=preferred_language,
                                            amount=amount,
                                            token_symbol=token_symbol,
-                                           tx_sender_information=transaction_account_metadata,
+                                           tx_recipient_information=metadata_id,
+                                           tx_sender_information=alt_metadata_id,
                                            timestamp=timestamp,
                                            balance=balance)
         if role == 'sender':
@@ -56,6 +58,7 @@ def transaction(notification_data: dict):
                                            preferred_language=preferred_language,
                                            amount=amount,
                                            token_symbol=token_symbol,
-                                           tx_recipient_information=transaction_account_metadata,
+                                           tx_recipient_information=alt_metadata_id,
+                                           tx_sender_information=metadata_id,
                                            timestamp=timestamp,
                                            balance=balance)

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -11,6 +11,7 @@ from chainlib.hash import strip_0x
 from cic_ussd.account.statement import get_cached_statement
 from cic_ussd.account.transaction import aux_transaction_data, validate_transaction_account
 from cic_ussd.cache import cache_data, cache_data_key
+from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 
 
@@ -72,13 +73,17 @@ def parse_transaction(preferences: dict, transaction: dict) -> dict:
     preferred_language = preferences.get('preferred_language')
     if not preferred_language:
         preferred_language = i18n.config.get('fallback')
-
+    transaction['preferred_language'] = preferred_language
     transaction = aux_transaction_data(preferred_language, transaction)
     session = SessionBase.create_session()
-    account = validate_transaction_account(session, transaction)
+    role = transaction.get('role')
-    metadata_id = account.standard_metadata_id()
+    alt_blockchain_address = transaction.get('alt_blockchain_address')
-    transaction['metadata_id'] = metadata_id
+    blockchain_address = transaction.get('blockchain_address')
+    account = validate_transaction_account(blockchain_address, role, session)
+    alt_account = session.query(Account).filter_by(blockchain_address=alt_blockchain_address).first()
+    if alt_account:
+        transaction['alt_metadata_id'] = alt_account.standard_metadata_id()
+    transaction['metadata_id'] = account.standard_metadata_id()
     transaction['phone_number'] = account.phone_number
-    session.commit()
     session.close()
     return transaction

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 1, 'alpha.1')
+version = (0, 3, 1, 'alpha.4')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-RUN apt-get install -y redis-server
-
-# create secrets directory
-RUN mkdir -vp pgp/keys
-
-# create application directory
-RUN mkdir -vp cic-ussd
-RUN mkdir -vp data
-
-COPY requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . .
-RUN python setup.py install
-
-COPY cic_ussd/db/ussd_menu.json data/
-
-COPY docker/*.sh .
-RUN chmod +x /root/*.sh
-
-# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY config/ /usr/local/etc/cic-ussd/
-COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
-
-ENTRYPOINT []

apps/cic-ussd/docker/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd

apps/cic-ussd/requirements.txt

@@ -1,7 +1,7 @@
 alembic==1.4.2
 bcrypt==3.2.0
 celery==4.4.7
-cic-eth[services]~=0.12.4a3
+cic-eth[services]~=0.12.4a7
 cic-notify~=0.4.0a10
 cic-types~=0.1.0a14
 confini>=0.4.1a1,<0.5.0
@@ -13,11 +13,5 @@ redis==3.5.3
 semver==2.13.0
 SQLAlchemy==1.3.20
 tinydb==4.2.0
-phonenumbers==8.12.12
-redis==3.5.3
-celery==4.4.7
-python-i18n[YAML]==0.3.9
-pyxdg==0.27
-bcrypt==3.2.0
-uWSGI==2.0.19.1
 transitions==0.8.4
+uWSGI==2.0.19.1

apps/cic-ussd/tests/cic_ussd/account/test_transaction.py

@@ -75,17 +75,21 @@ def test_transaction_actors(activated_account, transaction_result, valid_recipie
 def test_validate_transaction_account(activated_account, init_database, transactions_list):
     sample_transaction = transactions_list[0]
     recipient_transaction, sender_transaction = transaction_actors(sample_transaction)
-    recipient_account = validate_transaction_account(init_database, recipient_transaction)
+    recipient_account = validate_transaction_account(
-    sender_account = validate_transaction_account(init_database, sender_transaction)
+        recipient_transaction.get('blockchain_address'), recipient_transaction.get('role'), init_database)
+    sender_account = validate_transaction_account(
+        sender_transaction.get('blockchain_address'), sender_transaction.get('role'),  init_database)
     assert isinstance(recipient_account, Account)
     assert isinstance(sender_account, Account)
     sample_transaction = transactions_list[1]
     recipient_transaction, sender_transaction = transaction_actors(sample_transaction)
     with pytest.raises(UnknownUssdRecipient) as error:
-        validate_transaction_account(init_database, recipient_transaction)
+        validate_transaction_account(
+            recipient_transaction.get('blockchain_address'), recipient_transaction.get('role'), init_database)
     assert str(
         error.value) == f'Tx for recipient: {recipient_transaction.get("blockchain_address")} has no matching account in the system.'
-    validate_transaction_account(init_database, sender_transaction)
+    validate_transaction_account(
+        sender_transaction.get('blockchain_address'), sender_transaction.get('role'), init_database)
     assert f'Tx from sender: {sender_transaction.get("blockchain_address")} has no matching account in system.'
 
 

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py

@@ -30,8 +30,6 @@ def test_is_valid_recipient(activated_account,
                             valid_recipient):
     state_machine = ('0112365478', generic_ussd_session, valid_recipient, init_database)
     assert is_valid_recipient(state_machine) is False
-    state_machine = (pending_account.phone_number, generic_ussd_session, valid_recipient, init_database)
-    assert is_valid_recipient(state_machine) is False
     state_machine = (valid_recipient.phone_number, generic_ussd_session, activated_account, init_database)
     assert is_valid_recipient(state_machine) is True
 

apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py

@@ -24,7 +24,8 @@ def test_transaction(celery_session_worker,
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')
     token_symbol = notification_data.get('token_symbol')
-    transaction_account_metadata = notification_data.get('metadata_id')
+    alt_metadata_id = notification_data.get('alt_metadata_id')
+    metadata_id = notification_data.get('metadata_id')
     timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
     s_transaction = celery.signature(
         'cic_ussd.tasks.notifications.transaction', [notification_data]
@@ -36,7 +37,8 @@ def test_transaction(celery_session_worker,
                               preferred_language=preferred_language,
                               amount=amount,
                               token_symbol=token_symbol,
-                              tx_recipient_information=transaction_account_metadata,
+                              tx_recipient_information=alt_metadata_id,
+                              tx_sender_information=metadata_id,
                               timestamp=timestamp,
                               balance=balance)
     assert mock_notifier_api.get('message') == message
@@ -52,7 +54,8 @@ def test_transaction(celery_session_worker,
                               preferred_language=preferred_language,
                               amount=amount,
                               token_symbol=token_symbol,
-                              tx_sender_information=transaction_account_metadata,
+                              tx_recipient_information=metadata_id,
+                              tx_sender_information=alt_metadata_id,
                               timestamp=timestamp,
                               balance=balance)
     assert mock_notifier_api.get('message') == message

apps/cic-ussd/tests/fixtures/transaction.py

@@ -5,12 +5,18 @@ import random
 import pytest
 
 # local import
+from cic_ussd.account.balance import get_cached_available_balance
 
 # tests imports
 
 
 @pytest.fixture(scope='function')
-def notification_data(activated_account, cache_person_metadata, cache_preferences, preferences):
+def notification_data(activated_account,
+                      cache_person_metadata,
+                      cache_preferences,
+                      cache_balances,
+                      preferences,
+                      valid_recipient):
     return {
         'blockchain_address': activated_account.blockchain_address,
         'token_symbol': 'GFT',
@@ -18,6 +24,7 @@ def notification_data(activated_account, cache_person_metadata, cache_preference
         'role': 'sender',
         'action_tag': 'Sent',
         'direction_tag': 'To',
+        'alt_metadata_id': valid_recipient.standard_metadata_id(),
         'metadata_id': activated_account.standard_metadata_id(),
         'phone_number': activated_account.phone_number,
         'available_balance': 50.0,

apps/cic-ussd/var/lib/locale/sms.en.yml

@@ -2,9 +2,9 @@ en:
   account_successfully_created: |-
     You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
   received_tokens: |-
-    Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
+    Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp} to %{tx_recipient_information}. New balance is %{balance} %{token_symbol}.
   sent_tokens: |-
-    Successfully sent %{amount} %{token_symbol} to %{tx_recipient_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
+    Successfully sent %{amount} %{token_symbol} to %{tx_recipient_information} %{timestamp} from %{tx_sender_information}. New balance is %{balance} %{token_symbol}.
   terms: |-
     By using the service, you agree to the terms and conditions at http://grassecon.org/tos
   upsell_unregistered_recipient: |-

apps/cic-ussd/var/lib/locale/sms.sw.yml

@@ -2,9 +2,9 @@ sw:
   account_successfully_created: |-
     Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
   received_tokens: |-
-    Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
+    Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp} ikapokewa na %{tx_recipient_information}. Salio lako ni %{balance} %{token_symbol}.
   sent_tokens: |-
-    Umetuma %{amount} %{token_symbol} kwa %{tx_recipient_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
+    Umetuma %{amount} %{token_symbol} kwa %{tx_recipient_information} %{timestamp} kutoka kwa %{tx_sender_information}. Salio lako ni %{balance} %{token_symbol}.
   terms: |-
     Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos
   upsell_unregistered_recipient: |-

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -7,10 +7,8 @@ en:
       3. Help
     initial_pin_entry: |-
       CON Please enter a new four number PIN for your account.
-      0. Back
     initial_pin_confirmation: |-
       CON Enter your four number PIN again
-      0. Back
     enter_given_name: |-
       CON Enter first name
       0. Back
@@ -183,7 +181,7 @@ en:
     exit_pin_mismatch: |-
       END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}.
     exit_invalid_recipient: |-
-      CON Recipient phone number is incorrect.
+      CON Recipient's phone number is not registered or is invalid:
         00. Retry
         99. Exit
     exit_successful_transaction: |-

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -7,13 +7,10 @@ sw:
       3. Help
     initial_pin_entry: |-
       CON Tafadhali weka pin mpya yenye nambari nne kwa akaunti yako
-      0. Nyuma
     initial_pin_confirmation: |-
       CON Weka PIN yako tena
-      0. Nyuma
     enter_given_name: |-
       CON Weka jina lako la kwanza
-      0. Nyuma
     enter_family_name: |-
       CON Weka jina lako la mwisho
       0. Nyuma

apps/contract-migration/.gitlab-ci.yml

@@ -1,25 +1,25 @@
-.contract_migration_variables:
+#.contract_migration_variables:
-    variables:
+#    variables:
-        APP_NAME: contract-migration
+#        APP_NAME: contract-migration
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+#        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
+#        CONTEXT: apps/$APP_NAME
-
+#
-build-mr-contract-migration:
+#build-mr-contract-migration:
-    extends:
+#    extends:
-        - .py_build_merge_request
+#        - .py_build_merge_request
-        - .contract_migration_variables
+#        - .contract_migration_variables
-    rules:
+#    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+#    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
+#      changes:
-          - apps/contract-migration/**/*
+#          - apps/contract-migration/**/*
-      when: always
+#      when: always
-
+#
-build-push-contract-migration:
+#build-push-contract-migration:
-    extends:
+#    extends:
-        - .py_build_push
+#        - .py_build_push
-        - .contract_migration_variables
+#        - .contract_migration_variables
-    rules:
+#    rules:
-    - if: $CI_COMMIT_BRANCH == "master"
+#    - if: $CI_COMMIT_BRANCH == "master"
-      changes:
+#      changes:
-          - apps/contract-migration/**/*
+#          - apps/contract-migration/**/*
-      when: always
+#      when: always

apps/contract-migration/requirements.txt

@@ -1,5 +1,5 @@
 cic-eth[tools]==0.12.4a4
-chainlib-eth>=0.0.9a3,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
 eth-erc20>=0.1.2a2,<0.2.0
 erc20-demurrage-token>=0.0.5a2,<0.1.0
 eth-accounts-index>=0.1.2a2,<0.2.0

apps/data-seeding/requirements.txt

@@ -1,10 +1,10 @@
 sarafu-faucet~=0.0.7a1
-cic-eth[tools]~=0.12.4a4
+cic-eth[tools]~=0.12.4a7
 cic-types~=0.1.0a14
 crypto-dev-signer>=0.4.15a1,<=0.4.15
 faker==4.17.1
-chainsyncer~=0.0.6a1
+chainsyncer~=0.0.6a3
-chainlib-eth~=0.0.9a4
+chainlib-eth~=0.0.9a7
 eth-address-index~=0.2.3a4
 eth-contract-registry~=0.6.3a3
 eth-accounts-index~=0.1.2a3

ci_templates/.cic-template.yml

@@ -1,12 +1,12 @@
-image:
-  name: gcr.io/kaniko-project/executor:debug
-  entrypoint: [""]
 
 variables:
   KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
   MR_IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA
 
 .py_build_merge_request:
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
   stage: build
   script:
     - mkdir -p /kaniko/.docker
@@ -16,6 +16,9 @@ variables:
       --cache-repo $CI_REGISTRY_IMAGE --destination $MR_IMAGE_TAG 
 
 .py_build_target_dev:
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
   stage: build
   variables:
     IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA
@@ -28,6 +31,9 @@ variables:
       --destination $MR_IMAGE_TAG
 
 .py_build_push:
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
   stage: build
   variables:
       IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA

docker-compose.yml

@@ -13,20 +13,8 @@ networks:
     name: cic-network
 
 services:
-  # eth:
-  #   image: trufflesuite/ganache-cli
-  #   ports:
-  #     - ${HTTP_PORT_ETH:-8545}
-  #     - ${WS_PORT_ETH:-8546}
-  #   # Note! -e switch doesnt work, whatever you  put there, it will be 100
-  #   command: "-i 8996 -e 1000 -l 90000000 \
-  #           -m '${DEV_MNEMONIC:-\"history stumble mystery avoid embark arrive mom foil pledge keep grain dice\"}' \
-  #           -v --db /tmp/cic/ganache/ganache.db \
-  #           --noVMErrorsOnRPCResponse --allowUnlimitedContractSize"
-  #   volumes:
-  #     - ganache-db:/tmp/cic/ganache
-
   eth:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/bloxberg-node:${TAG:-latest}
     build:
       context: apps/bloxbergValidatorSetup
     restart: unless-stopped
@@ -71,6 +59,7 @@ services:
       - bee-data:/tmp/cic/bee
 
   contract-migration:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest}
     profiles:
       - migrations 
     build:
@@ -128,7 +117,9 @@ services:
     volumes:
       - contract-config:/tmp/cic/config
 
+
   cic-cache-tracker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
     profiles:
       - cache 
     build:
@@ -170,6 +161,7 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
     profiles:
       - cache 
     build:
@@ -210,6 +202,7 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
     profiles:
       - cache 
     build:
@@ -245,6 +238,7 @@ services:
 
 
   cic-eth-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build: 
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -298,6 +292,7 @@ services:
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -342,6 +337,7 @@ services:
 
 
   cic-eth-dispatcher:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -386,6 +382,7 @@ services:
 
 
   cic-eth-retrier:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -433,6 +430,7 @@ services:
 
 
   cic-notify-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:${TAG:-latest}
     build:
       context: apps/cic-notify
       dockerfile: docker/Dockerfile
@@ -461,6 +459,7 @@ services:
 
 
   cic-meta-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest}
     profiles:
       - custodial-meta
     hostname: meta
@@ -496,6 +495,7 @@ services:
     # command: "/root/start_server.sh -vv"
 
   cic-user-ussd-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
     profiles:
       - custodial-ussd
     build:
@@ -528,6 +528,7 @@ services:
     command: "/root/start_cic_user_ussd_server.sh -vv"
 
   cic-user-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
     profiles:
       - custodial-ussd
     build:
@@ -553,6 +554,7 @@ services:
     command: "/root/start_cic_user_server.sh -vv"
 
   cic-user-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
     profiles:
       - custodial-ussd
     build:

kubernetes/cic-auth-proxy/cic-auth-creds-configmap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cic-auth-proxy-credentials-configmap
+  namespace: grassroots
+data:
+    credentials.yaml: | 
+        level: 9
+        items:
+        user: 1

kubernetes/cic-auth-proxy/cic-auth-proxy-acl-configMap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cic-auth-proxy-acl-configmap
+  namespace: grassroots
+data:
+    F3FAF668E82EF5124D5187BAEF26F4682343F692: |
+        - "^/user(/.*)?$":
+            read:
+            - user

kubernetes/cic-auth-proxy/cic-auth-proxy-meta-deployment.yaml

@@ -0,0 +1,114 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-auth-proxy-meta
+  namespace: grassroots
+  labels:
+    app: cic-auth-proxy-meta
+    group: cic
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-auth-proxy-meta
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cic-auth-proxy-meta
+        group: cic
+    spec:
+      containers:
+      - name: cic-auth-proxy-meta
+        #image: registry.gitlab.com/grassrootseconomics/cic-auth-proxy:master-c05fafbf-1627493790 # {"$imagepolicy": "flux-system:cic-auth-proxy"} 
+        image: registry.gitlab.com/grassrootseconomics/cic-auth-proxy:latest
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 100m
+            memory: 200Mi
+        env:
+        - name: PROXY_HOST
+          value: cic-meta-server
+        - name: PROXY_PORT
+          value: "80"
+        - name: PROXY_PATH_PREFIX
+          value: "/"
+        - name: HTTP_AUTH_ORIGIN
+          value: https://meta-auth.dev.grassrootseconomics.net:443
+        - name: HTTP_AUTH_REALM
+          value: GE
+        - name: ACL_CREDENTIALS_ENDPOINT
+          value: http://key-server:8081/
+        - name: ACL_PATH
+          value: /data/acls/F3FAF668E82EF5124D5187BAEF26F4682343F692
+        - name: GPG_PUBLICKEYS_ENDPOINT
+          value: http://key-server:8080/.well-known/publickeys/
+        - name: GPG_SIGNATURE_ENDPOINT
+          value: http://key-server:8080/.well-known/signature/
+        - name: GPG_TRUSTED_PUBLICKEY_FINGERPRINT # fingerprint of trusted key
+          value: CCE2E1D2D0E36ADE0405E2D0995BB21816313BD5
+        - name: GPG_HOMEDIR
+          value: /usr/local/etc/cic-auth-proxy/.gnupg/
+        - name: GPG_IMPORT_DIR
+          value: /usr/local/etc/cic-auth-proxy/import/
+        - name: GPG_PUBLICKEY_FILENAME
+          value: publickeys.asc
+        - name: GPG_SIGNATURE_FILENAME
+          value: signature.asc
+        - name: GPG_TRUSTED_PUBLICKEY_MATERIAL
+          value: /usr/local/etc/cic-auth-proxy/trusted/trustedpublickey.asc
+        ports:
+        - containerPort: 8080
+          name: http
+        volumeMounts:
+        - name: acl-config
+          mountPath: /data/acls/
+          readOnly: true
+        - name: credentials-config
+          mountPath: /data/noop/
+          readOnly: true
+        - name: trusted-publickey
+          mountPath: /usr/local/etc/cic-auth-proxy/trusted/
+        - name: gpg-homedir
+          mountPath: /usr/local/etc/cic-auth-proxy/.gnupg
+        - name: pgp-meta-test
+          mountPath: /usr/local/etc/cic-auth-proxy/import
+      volumes:
+      - name: pgp-meta-test
+        configMap:
+          name: pgp-meta-test
+      - name: acl-config
+        configMap:
+          name: cic-auth-proxy-acl-configmap
+      - name: credentials-config
+        configMap:
+          name: cic-auth-proxy-credentials-configmap
+      - name: trusted-publickey
+        configMap:
+          name: pgp-trusted-publickey
+      - name: gpg-homedir
+        emptyDir: {}
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-auth-proxy-meta
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-auth-proxy-meta
+  type: ClusterIP
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 8080

kubernetes/cic-auth-proxy/cic-auth-proxy-user-deployment.yaml

@@ -0,0 +1,114 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-auth-proxy-user
+  namespace: grassroots
+  labels:
+    app: cic-auth-proxy-user
+    group: cic
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-auth-proxy-user
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cic-auth-proxy-user
+        group: cic
+    spec:
+      containers:
+      - name: cic-auth-proxy-user
+        #image: registry.gitlab.com/grassrootseconomics/cic-auth-proxy:master-c05fafbf-1627493790 # {"$imagepolicy": "flux-system:cic-auth-proxy"} 
+        image: registry.gitlab.com/grassrootseconomics/cic-auth-proxy:latest
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 100m
+            memory: 200Mi
+        env:
+        - name: PROXY_HOST
+          value: cic-user-server
+        - name: PROXY_PORT
+          value: "80"
+        - name: PROXY_PATH_PREFIX
+          value: "/"
+        - name: HTTP_AUTH_ORIGIN
+          value: https://meta-auth.dev.grassrootseconomics.net:443
+        - name: HTTP_AUTH_REALM
+          value: GE
+        - name: ACL_CREDENTIALS_ENDPOINT
+          value: http://key-server:8081/
+        - name: ACL_PATH
+          value: /data/acls/F3FAF668E82EF5124D5187BAEF26F4682343F692
+        - name: GPG_PUBLICKEYS_ENDPOINT
+          value: http://key-server:8080/.well-known/publickeys/
+        - name: GPG_SIGNATURE_ENDPOINT
+          value: http://key-server:8080/.well-known/signature/
+        - name: GPG_TRUSTED_PUBLICKEY_FINGERPRINT # fingerprint of trusted key
+          value: CCE2E1D2D0E36ADE0405E2D0995BB21816313BD5
+        - name: GPG_HOMEDIR
+          value: /usr/local/etc/cic-auth-proxy/.gnupg/
+        - name: GPG_IMPORT_DIR
+          value: /usr/local/etc/cic-auth-proxy/import/
+        - name: GPG_PUBLICKEY_FILENAME
+          value: publickeys.asc
+        - name: GPG_SIGNATURE_FILENAME
+          value: signature.asc
+        - name: GPG_TRUSTED_PUBLICKEY_MATERIAL
+          value: /usr/local/etc/cic-auth-proxy/trusted/trustedpublickey.asc
+        ports:
+        - containerPort: 8080
+          name: http
+        volumeMounts:
+        - name: acl-config
+          mountPath: /data/acls/
+          readOnly: true
+        - name: credentials-config
+          mountPath: /data/noop/
+          readOnly: true
+        - name: trusted-publickey
+          mountPath: /usr/local/etc/cic-auth-proxy/trusted/
+        - name: gpg-homedir
+          mountPath: /usr/local/etc/cic-auth-proxy/.gnupg
+        - name: pgp-meta-test
+          mountPath: /usr/local/etc/cic-auth-proxy/import
+      volumes:
+      - name: pgp-meta-test
+        configMap:
+          name: pgp-meta-test
+      - name: acl-config
+        configMap:
+          name: cic-auth-proxy-acl-configmap
+      - name: credentials-config
+        configMap:
+          name: cic-auth-proxy-credentials-configmap
+      - name: trusted-publickey
+        configMap:
+          name: pgp-trusted-publickey
+      - name: gpg-homedir
+        emptyDir: {}
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-auth-proxy-user
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-auth-proxy-user
+  type: ClusterIP
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 8080

kubernetes/cic-auth-proxy/cic-auth-proxy-ussd-deployment.yaml

@@ -0,0 +1,129 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-auth-proxy-ussd
+  namespace: grassroots
+  labels:
+    app: cic-auth-proxy-ussd
+    group: cic
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-auth-proxy-ussd
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cic-auth-proxy-ussd
+        group: cic
+    spec:
+      containers:
+      - name: cic-auth-proxy-ussd
+        #image: registry.gitlab.com/grassrootseconomics/cic-auth-proxy:master-c05fafbf-1627493790 # {"$imagepolicy": "flux-system:cic-auth-proxy"} 
+        image: registry.gitlab.com/grassrootseconomics/cic-auth-proxy:latest
+        imagePullPolicy: Always   
+        command: ["uwsgi", "--wsgi-file", "meta/scripts/proxy-ussd.py", "--http",
+          ":8080"]
+        resources:
+          requests:
+
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 100m
+            memory: 200Mi
+        env:
+        - name: PROXY_HOST
+          value: cic-user-ussd-server
+        - name: PROXY_PORT
+          value: "80"
+        - name: PROXY_PATH_PREFIX
+          value: "/"
+        - name: HTTP_AUTH_ORIGIN
+          value: https://ussd-auth.dev.grassrootseconomics.net:443
+        - name: HTTP_AUTH_REALM
+          value: GE
+        - name: ACL_CREDENTIALS_ENDPOINT
+          value: http://key-server:8081/
+        - name: ACL_PATH
+          value: /data/acls/F3FAF668E82EF5124D5187BAEF26F4682343F692
+        - name: ACL_QUERYSTRING_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: cic-ussd-querystring-creds
+              key: username
+        - name: ACL_QUERYSTRING_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: cic-ussd-querystring-creds
+              key: password
+        - name: ACL_WHITELIST
+          value: "37.188.113.15, 164.177.157.18, 5.79.0.242, 164.177.141.82, 164.177.141.83"
+        - name: GPG_PUBLICKEYS_ENDPOINT
+          value: http://key-server:8080/.well-known/publickeys/
+        - name: GPG_SIGNATURE_ENDPOINT
+          value: http://key-server:8080/.well-known/signature/
+        - name: GPG_TRUSTED_PUBLICKEY_FINGERPRINT # fingerprint of trusted key
+          value: CCE2E1D2D0E36ADE0405E2D0995BB21816313BD5
+        - name: GPG_HOMEDIR
+          value: /usr/local/etc/cic-auth-proxy/.gnupg/
+        - name: GPG_IMPORT_DIR
+          value: /usr/local/etc/cic-auth-proxy/import/
+        - name: GPG_PUBLICKEY_FILENAME
+          value: publickeys.asc
+        - name: GPG_SIGNATURE_FILENAME
+          value: signature.asc
+        - name: GPG_TRUSTED_PUBLICKEY_MATERIAL
+          value: /usr/local/etc/cic-auth-proxy/trusted/trustedpublickey.asc
+        ports:
+        - containerPort: 8080
+          name: http
+        volumeMounts:
+        - name: acl-config
+          mountPath: /data/acls/
+          readOnly: true
+        - name: credentials-config
+          mountPath: /data/noop/
+          readOnly: true
+        - name: trusted-publickey
+          mountPath: /usr/local/etc/cic-auth-proxy/trusted/
+        - name: gpg-homedir
+          mountPath: /usr/local/etc/cic-auth-proxy/.gnupg
+        - name: pgp-meta-test
+          mountPath: /usr/local/etc/cic-auth-proxy/import
+      volumes:
+      - name: pgp-meta-test
+        configMap:
+          name: pgp-meta-test
+      - name: acl-config
+        configMap:
+          name: cic-auth-proxy-acl-configmap
+      - name: credentials-config
+        configMap:
+          name: cic-auth-proxy-credentials-configmap
+      - name: trusted-publickey
+        configMap:
+          name: pgp-trusted-publickey
+      - name: gpg-homedir
+        emptyDir: {}
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-auth-proxy-ussd
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-auth-proxy-ussd
+  type: ClusterIP
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 8080

kubernetes/cic-auth-proxy/cic-ussd-querystring-creds.yaml

@@ -0,0 +1,16 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: cic-ussd-querystring-creds
+  namespace: grassroots
+spec:
+  encryptedData:
+    password: AgAO18hv8gZZRoySuTqNOg06imdnfES7io54TEGO501cq8usRcj9Bu3us+7V8zD20DQTZ7xCGCV2eyZ1kMsnFNFUBPMG0raAtwe4dAskYeQxKp0VyE1y+d9TbecvlqRXmAD3lF1exMSS3PD68VZcrDpenXE7Ag74uEJM7YmAKnUXIzBhxZlG7bIwLrRuNiTye83e/jijaPD3+66NrExBM2H//b8u1l8QVNd13XwgAWkJEW9QNMP1b2ir9VWKE4P8Da22SQRIed/Nhyc+aR3izqEpCbeRYmQqlo9r66oXK4Yr5v0IkI38gBGORrPv1OZK01plvGgMoxTe3pSiW5cyMtCXx6GgyIFKII1yYvtlI86Sf4J3DRU6kC3NSvF+2B99yFrbUPyoAGQFd6oSWAQLBI2kYqf6NXuaT3kxBNroAICMlAYygPKG0t6jEzTWk+E5l4F/PHWFD5DM+diHRqQAdDStACcMCl2i2133PPjlK1gUhQkCmeRcKeEqMT5ssBx/KM5p+v8syV4/0VlNtJpnP/aWcbwvVsiHhDE9trM/+p5E6gsVymAIiW20nRnuOSjHCnHOtjtfPtEZ3A8eHAqgJjdE6fY7DvZmwKfx2A4tMcrikz208Pa7JIFE6nj9osTz7eMrWXTmquVRotZ9we3WmGBycgVUuv9hfzUC6srkTIyT9UGHH9UNiRba/73ZDF2Zr2XvN0ofkQz8dN0dKLxy/OptCciV3Rdn/4s/IQ0usSwXLEb2tQtHgy5+twj7IQij6amjLbulRm1U2GLatmvgbjqf
+    username: AgCJQ9JiJErCGfH3pkX3I3696Garu40pGWgvcUOa9wz3r3Q+5SY0IMnroWO3z66L/HG7DW70upgfJBVyhncrUBrC/z73D++nMz43JvFc39MHcUYVmqvjIw1401705+G7UxxgcMOx09++CBZ97wbEfKc251v98s8G/bLtMcqS/12pVNfMGgjpkE6InlS0n9VD26HDs4A3uNtfN2GK2dsazV05UXs8W+6JOZsJ60QfPJylCpKh+sxPLDlYt4lRIM/6pP7kQXLn+VmWtzuo1dZTaUliMYH+DOXO2V9ePnjTUXGrMgRfuZP2PCmG3usdC45mOPpuURPFUmF8SDQ4IXKhBd7N+8pjZDiqQ2RxK61Qz6MXv851u7HABgVMhtjlZfaD4hVY+mr7KYDVQvrhJ971y84KBHuQFOxwZZnXAx5FdBWmkKkz959bjulJaRe1ZWA01k1SQHiVFeIArbbNSlvH45XoNR8rxFiQE+5Olt9UwdpXAH/sAfH7CRY1SnRrAW3LCyCVqAJcXF9kU+bnezVgWoJ/h9ff6VKRFqh3o+wa6V6J8GNbwC0/oeXo0XyUjbwAUjIRKeWbJ1obkvWDfYILpo2CVt34tzYowiVqmYYB8SqKBPgrca+Xmn9GpggOMibJKk0LC6R04iAajMy73cuxkX+PMV1Wz5xOuF60ccCsdiD15deYOlA34UdfeNQgGA7EKPhsl0kD/xm5
+  template:
+    metadata:
+      creationTimestamp: null
+      name: cic-ussd-querystring-creds
+      namespace: grassroots
+

kubernetes/cic-cache/cic-cache-server-deployment.yaml

@@ -0,0 +1,100 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-cache-server
+  namespace: grassroots
+  labels:
+    app: cic-cache-server
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-cache-server
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-cache-server
+        group: cic
+        teir: backend
+    spec:
+      containers:
+      - name: cic-cache-server
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:master-402b968b-1626300208 # {"$imagepolicy": "flux-system:cic-cache"}
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:latest
+        imagePullPolicy: Always   
+        command: ["/usr/local/bin/uwsgi", "--wsgi-file=/root/cic_cache/runnable/daemons/server.py",
+          "--http=:8000", "--pyargv", "-vv"]
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 100m
+            memory: 100Mi
+        env:
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: DATABASE_NAME
+          value: cic_cache
+        - name: SERVER_PORT
+          value: "8000"
+        - name: DATABASE_DEBUG
+          value: "0"
+        ports:
+        - containerPort: 8000
+          name: server
+      restartPolicy: Always
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-cache-svc
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-cache-server
+  type: ClusterIP
+  ports:
+  - name: server
+    protocol: TCP
+    port: 80
+    targetPort: 8000

kubernetes/cic-cache/cic-cache-tasker-deployment.yaml

@@ -0,0 +1,175 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-cache-watchers
+  namespace: grassroots
+  labels:
+    app: cic-cache-watchers
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-cache-watchers
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-cache-watchers
+        group: cic
+        tier: queue
+    spec:
+      containers:
+      - name: cic-cache-tasker
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:master-402b968b-1626300208 # {"$imagepolicy": "flux-system:cic-cache"}
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:latest
+        imagePullPolicy: Always   
+        command: ["/usr/local/bin/cic-cache-taskerd", "-vv"]
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 100m
+            memory: 100Mi
+        env:
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: DATABASE_NAME
+          value: cic_cache
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+      - name: cic-cache-tracker
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:master-402b968b-1626300208 # {"$imagepolicy": "flux-system:cic-cache"}
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:latest
+        # command: ["/usr/local/bin/cic-cache-trackerd", "-vv", "-c", "/usr/local/etc/cic-cache"]
+        command: ["./start_tracker.sh", "-c", "/usr/local/etc/cic-cache", "-vv"]
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 100m
+            memory: 100Mi
+        env:
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: DATABASE_NAME
+          value: cic_cache
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: SERVER_PORT
+          value: "8000"
+        - name: ETH_ABI_DIR
+          value: /usr/local/share/cic/solidity/abi
+        - name: DATABASE_DEBUG
+          value: "0"
+      restartPolicy: Always

kubernetes/cic-eth/cic-eth-tasker-deployment.yaml

@@ -0,0 +1,221 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-eth-tasker
+  namespace: grassroots
+  labels:
+    app: cic-eth-tasker
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-eth-tasker
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-eth-tasker
+        group: cic
+        tier: queue
+    spec:
+      containers:
+      - name: cic-eth-tasker
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:master-f1917300-1626888924 # {"$imagepolicy": "flux-system:cic-eth"}       
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:latest
+        imagePullPolicy: Always   
+        # command: ["./start_tasker.sh", "-q", "cic-eth", "-vv"]
+        command: ["/usr/local/bin/cic-eth-taskerd"]
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 250Mi
+        env:
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS # - name: ETH_GAS_PROVIDER_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: REDIS_HOST
+          value: redis-master
+        - name: REDIS_PORT
+          value: "6379"
+        - name: REDIS_DB
+          value: "0"
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: ETH_ABI_DIR
+          value: /usr/local/share/cic/solidity/abi
+        - name: DATABASE_NAME
+          value: cic_eth
+        - name: DATABASE_POOL_SIZE
+          value: "0"
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: BANCOR_DIR
+          value: /usr/local/share/cic/bancor
+        - name: SIGNER_SOCKET_PATH
+          value: ipc:///run/crypto-dev-signer/jsonrpc.ipc
+        - name: SIGNER_SECRET
+          value: deadbeef
+        - name: ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
+          value: "0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA"
+        - name: TASKS_TRACE_QUEUE_STATUS
+          value: "1"
+        - name: "DATABASE_DEBUG"
+          value: "false"
+        volumeMounts:
+        - name: socket-path
+          mountPath: /run/crypto-dev-signer/
+      - name: cic-eth-signer
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:master-f1917300-1626888924 # {"$imagepolicy": "flux-system:cic-eth"}       
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:latest
+        imagePullPolicy: Always   
+        # command: ["./start_tasker.sh", "-q", "cic-eth", "-vv"]
+        command: ["python", "/usr/local/bin/crypto-dev-daemon", "-c", "/usr/local/etc/crypto-dev-signer",
+          "-vv"]
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 250Mi
+        env:
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS # - name: ETH_GAS_PROVIDER_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: ETH_ABI_DIR
+          value: /usr/local/share/cic/solidity/abi
+        - name: DATABASE_NAME
+          value: cic_eth
+        - name: DATABASE_POOL_SIZE
+          value: "0"
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: BANCOR_DIR
+          value: /usr/local/share/cic/bancor
+        - name: SIGNER_SOCKET_PATH
+          value: ipc:///run/crypto-dev-signer/jsonrpc.ipc
+        - name: SIGNER_SECRET
+          value: deadbeef
+        - name: ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
+          value: "0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA"
+        - name: TASKS_TRACE_QUEUE_STATUS
+          value: "1"
+        - name: "DATABASE_DEBUG"
+          value: "false"
+        - name: "CIC_DEFAULT_TOKEN_SYMBOL"
+          value: GFT
+        volumeMounts:
+        - name: socket-path
+          mountPath: /run/crypto-dev-signer/
+      volumes:
+      - name: socket-path
+        emptyDir: {}
+      restartPolicy: Always

kubernetes/cic-eth/cic-eth-tracker.yaml

@@ -0,0 +1,248 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+# The guardian is composed of: 
+# cic-manager-head
+# cic-dispatch
+# cic-retrier
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-eth-tracker
+  namespace: grassroots
+  labels:
+    app: cic-eth-tracker
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-eth-tracker
+  replicas: 1 # these are all strictly 1 
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-eth-tracker
+        group: cic
+    spec:
+      containers:
+      - name: cic-eth-tracker
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:master-f1917300-1626888924 # {"$imagepolicy": "flux-system:cic-eth"}       
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:latest
+        imagePullPolicy: Always   
+        command: ["./start_tracker.sh", "-v", "-c", "/usr/local/etc/cic-eth"]
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 250Mi
+        env:
+        - name: TASKS_TRANSFER_CALLBACKS
+          value: "cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: DATABASE_NAME
+          value: cic_eth
+        - name: DATABASE_DEBUG
+          value: "0"
+        - name: ETH_ABI_DIR
+          value: /usr/local/share/cic/solidity/abi
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: REDIS_HOSTNAME
+          value: redis-master
+      - name: cic-eth-dispatcher
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:master-f1917300-1626888924 # {"$imagepolicy": "flux-system:cic-eth"}       
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:latest      
+        imagePullPolicy: Always   
+        command: ["./start_dispatcher.sh", "-q", "cic-eth", "-v"]
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 250Mi
+        env:
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: TASKS_TRANSFER_CALLBACKS
+          value: "cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: DATABASE_NAME
+          value: cic_eth
+        - name: DATABASE_DEBUG
+          value: "0"
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: REDIS_HOSTNAME
+          value: redis-master
+          # - name:  cic-eth-retrier
+          #   image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:latest
+          #   command: [ "./start_retry.sh", "-v" ]
+          #   resources:
+          #     requests:
+          #       cpu: 50m
+          #       memory: 100Mi
+          #     limits:
+          #       cpu: 500m
+          #       memory: 250Mi
+          #   env:
+          #   - name: CIC_REGISTRY_ADDRESS
+          #     valueFrom:
+          #       configMapKeyRef:
+          #         name: contract-migration-output
+          #         key: CIC_REGISTRY_ADDRESS
+          #   - name: CIC_TRUST_ADDRESS
+          #     valueFrom: 
+          #       configMapKeyRef:
+          #         name: contract-migration-output
+          #         key: CIC_TRUST_ADDRESS
+          #   - name: CIC_TX_RETRY_DELAY # TODO what is this value?
+          #     value: "15"
+          #   - name: TASKS_TRANSFER_CALLBACKS # TODO what is this value?
+          #     value: "taskcall:cic_eth.callbacks.noop.noop"
+          #   - name: ETH_PROVIDER 
+          #     value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+          #   - name: DATABASE_USER
+          #     value: grassroots
+          #   - name: DATABASE_HOST
+          #     value: postgres-helm-postgresqlsql 
+          #   - name: DATABASE_PASSWORD
+          #     value: tralala
+          #   - name: DATABASE_NAME
+          #     value: cic_eth
+          #   - name: DATABASE_PORT
+          #     value: "5432"
+          #   - name: DATABASE_ENGINE
+          #     value: postgres
+          #   - name: DATABASE_DRIVER
+          #     value: psycopg2
+          #   - name: DATABASE_DEBUG
+          #     value: "1"
+        # - name: REDIS_HOSTNAME
+        #   value: grassroots-redis-master
+      #   - name: CIC_CHAIN_SPEC
+      #     value: "evm:bloxberg:8996"
+      #   - name: CELERY_BROKER_URL
+      #     value: redis://grassroots-redis-master
+      #   - name: CELERY_RESULT_URL
+      #     value: redis://grassroots-redis-master
+      restartPolicy: Always

kubernetes/cic-meta/cic-meta-server-deployment.yaml

@@ -0,0 +1,122 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-meta-server
+  namespace: grassroots
+  labels:
+    app: cic-meta-server
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-meta-server
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-meta-server
+        group: cic
+    spec:
+      containers:
+      - name: cic-meta-server
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:master-fe017d2b-1625932004 # {"$imagepolicy": "flux-system:cic-meta"} 
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:latest
+        imagePullPolicy: Always   
+        resources:
+          requests:
+            cpu: 50m
+            memory: 250Mi
+          limits:
+            cpu: 100m
+            memory: 500Mi
+        env:
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: SCHEMA_SQL_PATH
+          value: scripts/initdb/server.postgres.sql
+        - name: DATABASE_NAME
+          value: cic_meta
+        - name: SERVER_HOST
+          value: localhost
+        - name: SERVER_PORT
+          value: "8000"
+        - name: DATABASE_SCHEMA_SQL_PATH
+          value: ""
+        - name: PGP_EXPORTS_DIR
+          value: /tmp/src/keys
+        - name: PGP_PRIVATEKEY_FILE # Private key here is for enrypting data
+          value: privatekey.asc
+        - name: PGP_PASSPHRASE
+          value: queenmarlena # TODO move to secret
+        - name: PGP_PUBLICKEY_TRUSTED_FILE
+          value: publickeys.asc
+        - name: PGP_PUBLICKEY_ACTIVE_FILE # public key here is to know who to trust
+          value: publickeys.asc
+        - name: PGP_PUBLICKEY_ENCRYPT_FILE
+          value: publickeys.asc
+        ports:
+        - containerPort: 8000
+          name: cic-meta-server
+        volumeMounts:
+        - mountPath: /tmp/src/keys
+          readOnly: true
+          name: pgp
+      volumes:
+      - name: pgp
+        configMap:
+          name: pgp-meta-test
+          items:
+      restartPolicy: Always
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-meta-server
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-meta-server
+  type: ClusterIP
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 8000

kubernetes/cic-notify/cic-notify-africastalking-sandbox-sealedsecret.yaml

@@ -0,0 +1,17 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: cic-notify-africastalking-sandbox-secret
+  namespace: grassroots
+spec:
+  encryptedData:
+    api_key: AgC1j5LwcOocVchLac0kPmCqyqljV40RzlCXVrtaxFvFvO+s2KcaUNFrIxfirZ8pbSDPKAUAgQam9tJiZFg4Wdu0x6LDDo6x2/0t/HXqb1eeH+OxO045T0AQZPpqFK+5QZ7y+c0tnUJfppCgOd+PhvB0AyHndWoWmTMGy48zpKaeRZriRRM5+rkNgGzazUNdsQoThOhEStIDCa1+zpq++KckU+VS5Qgsj7X1gOth+XFeyVFuXLY/QTTmvflgXj/YA5HMcJVCO3/l4UGaxHaUWps6IlZ5RgALamavAww7HIT7iecjXbAKx4fQt7bKuQY/FHiwDLAJmfdYUv7YK3EVsW4lSWVxgQ4RKM1dru8Kgs4nE+jmlApeuMl2Y3yT0tyCXSNPBzcsAbeumctsWO2gypQbL/0N1a9OZYgJv4/rfbonl8lK8detZDnAOqhOEiJ2fX/H9bkLc7n5hEHxnIt72sgdurD3r7WkxNsD8/laBx7hDtr7+Zv9RZ8MlyyUd+ZBVax6tO7immo8ya+BRsFKcrr3FqSuvwl3q7y/DNLMU6wqEv+/FmfSW7J1HuGlOJApHM96R6Jfqpw+ZNZ/hS6Z0CDNicLRdRfDaKglOu4UvkiWQ2F1JtqtJEtishbk3bNtZSAOO7Uan/V1XfeUvrYBAyXCoLLwK2pm/eWNZ71BW7TghCNgqRpeALPZy2JrDudnmyoXzcidJeBi5lBEnbXzIcOuos/0xgM/DiKQDO8mxh1ycbiu9XSydxdkLrRFXMtO8KK4qhIdgJp8INjZxIzgm5j3
+    api_sender_id: AgCTvYXxQNTCcRnMrfcc3D+OY+N8/mFtfI+O3xI45qxdC1n7/OE2MI0XIleYSbbAh3R7+5lNjq3pVhdKpRvCYxUGerCdSeYFQZf0RN71sAtDV8NnHCB6pRIsCWlvjDHw7XRH6PiPrr0xj4rwd1Qou8EMybVQXwT6nvV20kdKhAd6dTTgqT8x1AB5N+L0o1H/ThOKNadjN6oqlROz4in0DTTmis+Ebk4pywFroVWa5pMCjy7Ua5omKqB45lAKDp4FwogSnEUVS1b1pQTO/o0mnCQRObPcOzVjze6oqfUaEPkvtrOkmKuAxEXFKdAzG8bzPwx6EuSCbMmcUekeBGKDZBniE1YyOkPyLZelBEtyXFhQBmKbdW4qhlEJwYwflFBll8eNo2ruMVH7HTJvAqW4p2mxTIoM9Nx+UWDJjZ736aYvDfCgZX0MVKvXjrNtWEhuVBYGZxFIWfj/RqNkcJf+ZlNYK8MripzPReNWDlvhH1jzlPGJozUlYfLY8hY/NDGMbdc+EflLy8p4oRLWUo0Z0W3ARwLM1WQMk8FSXAgCjMmL7m3d310lVO5LLVdj3GTpjfCNPm2m7Re0lsOkLDE8x0vhJnlIUJ8WlY1EdezKfPWR0laNSknCCxfPYU88XiO2DVgUdWBH0Hg13yQQ3IXz49Pnf6LrAx46EgKUzoKnCgA5L/hRAIV3u8vav/5kdSF4CxY=
+    api_username: AgBQZXCDQY5B1RKfwjih04XSvdPJjmlfuA2bL6qiphYDIM223DJ5HEolHIWlCeSAjDC5JjkkesPmVzxybgD7hRwVuyOyWfmiXBG6+552v/DNv17IfSNNtZMCsEj9jyscHfvRI1NCG5pxywhBfRxhOVIYflbFI/H+13ReAs/6J/uhVfWVPMXzdg7Zj7mYsWzj3otv0npzXc+j2G4JZk53h7fDTY/XsXyZCe+Xpn9auf7Y+m+/shOiaM67CuE+eMWYhpTpTDy6oG3rjSrqhjVnE4wu8fP85nTGK2ctrQ9qgyOdcf1vPdlt1CwP5FZ+zxY8GrhUsYnunQ50znPD55e/8m5AqZhIrCjQ0lfzB2iCn0OzwOjnQpfv0W0QXpw6aDH/GYfMbDBLyXnF5yU0J+R7tv2Q45NKyRHbdO2VQXhXak95YGvK0Er3+8b0Jx4k3L+hM3OTFaJgdGOyb4IsUFG6vfQlfG3Bl4ggYqAGCJo041SpT4gc1XgGkar8WD4sOir61qKde8NnIOdRn1bhaZ8qD2eP+4p20hqsgpAA38SN6qZfrZc5sj/Re2mXxYIbmdALlP6yt1exhtU4QRzF23RDHmwiGtzh0YqIUj62+icrLcn+ZXgpufW9BgxKXilczF/bfsN2v4VFKsrkp+wRtAlFj3riDoles/IhItIJ9KpmSHEsRDRrkrUl+sFQ5xqQvfIuEoy/DHKf/Q3L
+  template:
+    metadata:
+      creationTimestamp: null
+      name: cic-notify-africastalking-sandbox-secret
+      namespace: grassroots
+

kubernetes/cic-notify/cic-notify-africastalking-sealedsecret.yaml

@@ -0,0 +1,17 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: cic-notify-africastalking-secret
+  namespace: grassroots
+spec:
+  encryptedData:
+    api_key: AgAbH1KDx0kVxtqPBvkb0w1uYJm0Vwd/eHBMhLJRC9Z5zs/YkHkV+Zrz4Pk0loSIdyV3fFdL+zcqnwWmkYjrkBYUXqB+F6HcfYPlND+zAtv8QMbpmHoF6faqx1MY981Sx7qqURlORCZzkwzrDEC4XK2nwut414PFEQ4Rn9SWe2RWXl75G3+TVkj0jd24rYYlGCmD8krjFLY9IMigKZOr0TqoPhHv9xKPg7+Xpwtd86QpiFcTR0fBtN8FLDuKlYGHvTvaUqT2kyBEXeiSn3V+rB7mD72QtNBwS9KJfQla/Gh1z4kps5DkZNoHKRz3O0SnMND1UrID713g7coYI+3q3Xk4/IFiYH4iTQwls/TZWuu/aAc74Ofs6tNwNfUeGGwa0uX/EdMpQ0cWBsGQXi3wG91kvCbnHQ1IERXgCLzGur9yB7YARy6H3tJkQrjzY2ETmMK6wj8yelKC1LLiVk/UVPiRI3O85JPxDDdtcOWkRtGso5Z1Q4D0AFWhbrhBXZc3cS2MmU1bIXoZ+fsYoZ1Mrg+uFQyuf4XwIq+HZuaYksN6xN0olId/H53Bau/z54pteFwT3VOKJNrLHCnAz0YJG2LO08Iu3FcNIUVh1tf39019QOFW2QGemhxznfPH1Mn4l9GwhE2MNfU2JqbFLnCfPZPoFSY+7YZ8I271aY4sM/OlaE2Nrk9RSsSgpS+WUCijSzZ8Crgc8dffVQryDEc/t+JIW7T4BH16lk+fgx2W1JoU/BMMvaRitHX5XsIs5xchwfuvb5+WbNnT8WbYdomGokEg
+    api_sender_id: AgBt9Sp+yb1Qek38Imhs4zW4pcujj2ExStcfLgWPXgD3WhIOfMS7AFAe5uDTICZqLEoMnluVTJn8Hk02BIU3quXWjOOgLJRmpNlgqAbchR1XA07BwGeYWMZomC4M+8nknMOHeQvo8HGN/Y5HnXw5N2Gw8Utlm/cLVVJOEYCKGiR8vWpzMMGepxIKVeex93Ev09SQsxbOJ3QO3SGdJiKFBvXgSeGnsdg+uAeC0NG1zzVBbiCgiNoIZgv6ZnwzBZSl7BYAazt4xOoHSRoicgmIX3PWdjIz+lzVHIuEuFZK9heq+MODQhZcvf54JRB1qPURGJobu3IVDAckIvnFlSCRaBp2rAqtyzAtngkau3Et/66eI3SChJFDgnbourQ1QRPzqWipzYQ11JpUbc0luXRfA+HvzrlIf5r/rSL93jepWQS0Exk2VuCrBAS+QPJccmncJYRZ98SU0fAu+5ZcHFVkEpc4iatW6j1tXyxn+tO1pl7yB/9GGfNA72XxVHwLZmTE7LN0Y4VyhdIQYtXX5ZX49bD0Lfk/m1pkwkWtj1Cas7YpFvX0JUN0bjn6JQoJAIh7fCW5O2ATp/eqpsCcfyw+Fy8kNPFVXlfvVmD9aqwYojfCE2CV0lXFGwHLkNp0nxxTPoqrx2YAa9R/nILaUawvMdMU+a8n1Ee79al/eVpqr4WdTkQikrxcoud8PDzNOyk2y8eZY+Kb2dU=
+    api_username: AgBOCaA5MnRsnGSkQw8Md2XIRsb+9Dg3XiM5yagk1GvAcO+eNnL4i2F4pbA0Ctad0LpQVXwVhppJDqxS7ffy3VJwMdEWxChKN3AiE3e8Eqx8LzD1xXU48OxAfksFKQrXNBVjfBcSgyoTwI5ohYSGxuUL3fWo4/Wp0AWdy92bwNZfhOvkLpfn+Q3nblLX9tQeu/ky6+hqkiyZSWjgykJXGnVnpPMoRS8BLbcxRD1kpJfMORfW7q1JyzheBGByStRvr7i6z3m4KVC/a9H3o0OpIQQTNtvvy4zECQ5NnjrxXYdOJfJZNq1enVqqCeA06h2e3DESfxYPPx6tL8+Ugo+4TorFb5Fw8vqFQRd/1SiGQE++W/MRY4P+fj9hQrxyzdnG5oBq1j2JTpx5VMwkOEk1yVLmRS9nYrZOfFE/6EJJSzRZMbh+xVTEvcag9ZzSzkbKoakR+VKjZ2Rpum6rwXPUUhJ5F6ohIZx2x/qE3QJlmxtnJQGfmNwe6HTiLxqjU3o0SkVQgMPN275Uydlm6Omn9eo2jkYnA5hg4TPoFjcqAmbgV5O372U6ShpNtA2hYi6CdF7K/sbKxhFtHTrPAdF5NhMGZ6N81X1AMWq3a1mojMBJbVTPVzSWPovkiqhQ62gKUUbPp191aLuQlX83dqEuRxyYTprId/ODB7RtaAddDcLsF4z/7i7yh8VtKyqHBs7hA52O1b2sQJwxmhw=
+  template:
+    metadata:
+      creationTimestamp: null
+      name: cic-notify-africastalking-secret
+      namespace: grassroots
+

kubernetes/cic-notify/cic-notify-deployment.yaml

@@ -0,0 +1,100 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-notify-tasker
+  namespace: grassroots
+  labels:
+    app: cic-notify-tasker
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-notify-tasker
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cic-notify-tasker
+        group: cic
+    spec:
+      containers:
+      - name: cic-notify-tasker
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:master-7a3cb7ab-1627053362 # {"$imagepolicy": "flux-system:cic-notify"}
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:latest
+        imagePullPolicy: Always   
+        command: ["./start_tasker.sh", "-q", "cic-notify", "-vv"]
+        resources:
+          requests:
+            cpu: 25m
+            memory: 100Mi
+          limits:
+            cpu: 50m
+            memory: 200Mi
+        env:
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: DATABASE_POOL_SIZE
+          value: "0"
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: DATABASE_NAME
+          value: cic_notify
+        - name: AFRICASTALKING_API_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: cic-notify-africastalking-sandbox-secret
+              key: api_username
+        - name: AFRICASTALKING_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: cic-notify-africastalking-sandbox-secret
+              key: api_key
+        - name: AFRICASTALKING_API_SENDER_ID
+          valueFrom:
+            secretKeyRef:
+              name: cic-notify-africastalking-sandbox-secret
+              key: api_sender_id
+        ports:
+        - containerPort: 80 # What is this value?
+          name: cic-eth-manager
+      restartPolicy: Always

kubernetes/cic-staff-client/cic-staff-client-deployment.yaml

@@ -0,0 +1,55 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-staff-client
+  namespace: grassroots
+  labels:
+    app: cic-staff-client
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-staff-client
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cic-staff-client
+        group: cic
+    spec:
+      containers:
+      - name: cicada
+        #image: registry.gitlab.com/grassrootseconomics/cic-staff-client:master-858e1e65-1627284988 # {"$imagepolicy": "flux-system:cic-staff-client"} 
+        image: registry.gitlab.com/grassrootseconomics/cic-staff-client:latest
+        imagePullPolicy: Always   
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+          limits:
+            cpu: 100m
+            memory: 100Mi
+        ports:
+        - containerPort: 80
+          name: http
+      restartPolicy: Always
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-staff-client
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-staff-client
+  type: ClusterIP
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 80

kubernetes/cic-ussd/cic-user-server-deployment.yaml

@@ -0,0 +1,113 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-user-server
+  namespace: grassroots
+  labels:
+    app: cic-user-server
+  annotations:
+    keel.sh/policy: "glob:master-*"
+    keel.sh/trigger: poll
+    keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-user-server
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-user-server
+        group: cic
+        tier: backend
+    spec:
+      containers:
+      - name: cic-user-server
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:master-fad0a4b5-1628267359 # {"$imagepolicy": "flux-system:cic-ussd"}       
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:latest
+        command: ["/root/start_cic_user_server.sh", "-vv"]
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 250Mi
+        env:
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: DATABASE_POOL_SIZE
+          value: "0"
+        - name: DATABASE_NAME
+          value: cic_ussd
+        - name: HTTP_PORT_CIC_USER_SERVER
+          value: "9500"
+        - name: PGP_KEYS_PATH
+          value: /tmp/src/keys/
+        - name: PGP_EXPORTS_DIR
+          value: /tmp/src/keys/
+        ports:
+        - containerPort: 9500
+          name: server
+        volumeMounts:
+        - mountPath: /tmp/src/keys
+          name: pgp
+          readOnly: true
+      volumes:
+      #- name: pgp
+      #  secret:
+      #    secretName: pgp
+      - name: pgp
+        configMap:
+          name: pgp-meta-test
+      restartPolicy: Always
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-user-server-svc
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-user-server
+  type: ClusterIP
+  ports:
+  - name: server
+    protocol: TCP
+    port: 80
+    targetPort: 9500

kubernetes/cic-ussd/cic-user-tasker-deployment.yaml

@@ -0,0 +1,122 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-user-tasker
+  namespace: grassroots
+  labels:
+    app: cic-user-tasker
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-user-tasker
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-user-tasker
+        group: cic
+        task: queue
+    spec:
+      containers:
+      - name: cic-user-tasker
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:master-7a3cb7ab-1627053361 # {"$imagepolicy": "flux-system:cic-ussd"}       
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:latest
+        imagePullPolicy: Always   
+        command: ["/root/start_cic_user_tasker.sh", "-q", "cic-ussd", "-vv"]
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 250Mi
+        env:
+        - name: APP_PASSWORD_PEPPER
+          valueFrom:
+            secretKeyRef:
+              name: cic-ussd-secret
+              key: app_password_pepper
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: DATABASE_NAME
+          value: cic_ussd
+        - name: DATABASE_POOL_SIZE
+          value: "0"
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: REDIS_HOST
+          value: redis-master
+        - name: REDIS_PORT
+          value: "6379"
+        - name: REDIS_DATABASE
+          value: "0"
+        - name: CIC_META_URL
+          value: http://cic-meta-server:80
+        - name: PGP_KEYS_PATH
+          value: /tmp/src/keys/
+        - name: PGP_EXPORTS_DIR
+          value: /tmp/src/keys/
+        - name: PGP_PRIVATE_KEYS
+          value: privatekey.asc
+        - name: PGP_PASSPHRASE
+          value: queenmarlena 
+        - name: CIC_META_URL
+          value: http://cic-meta-server:80
+        volumeMounts:
+        - mountPath: /tmp/src/keys
+          name: pgp
+          readOnly: true
+      volumes:
+      #- name: pgp
+      #  secret:
+      #    secretName: pgp
+      - name: pgp
+        configMap:
+          name: pgp-meta-test
+      restartPolicy: Always

kubernetes/cic-ussd/cic-user-ussd-server-deployment.yaml

@@ -0,0 +1,139 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cic-user-ussd-server
+  namespace: grassroots
+  labels:
+    app: cic-user-ussd-server
+  annotations:
+      keel.sh/policy: "glob:master-*"
+      keel.sh/trigger: poll
+      keel.sh/pollSchedule: "@every 5m"
+spec:
+  selector:
+    matchLabels:
+      app: cic-user-ussd-server
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: cic-user-ussd-server
+        group: cic
+        tier: backend
+    spec:
+      containers:
+      - name: cic-user-ussd-server
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:master-7a3cb7ab-1627053361 # {"$imagepolicy": "flux-system:cic-ussd"}       
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:latest
+        imagePullPolicy: Always   
+        command: ["/root/start_cic_user_ussd_server.sh", "-vv"]
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+          limits:
+            cpu: 500m
+            memory: 250Mi
+        env:
+        - name: APP_PASSWORD_PEPPER
+          valueFrom:
+            secretKeyRef:
+              name: cic-ussd-secret
+              key: app_password_pepper
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: DATABASE_POOL_SIZE
+          value: "0"
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: REDIS_HOST
+          value: redis-master
+        - name: REDIS_PORT
+          value: "6379"
+        - name: REDIS_DATABASE
+          value: "0"
+        - name: DATABASE_NAME
+          value: cic_ussd
+        - name: SERVER_PORT
+          value: "9000"
+        - name: APP_ALLOWED_IP
+          value: "0.0.0.0/0"
+        - name: CIC_META_URL
+          value: http://cic-meta-server:80
+        - name: PGP_KEYS_PATH
+          value: /tmp/src/keys/
+        - name: PGP_EXPORTS_DIR
+          value: /tmp/src/keys/
+        - name: PGP_PRIVATE_KEYS
+          value: privatekey.asc
+        - name: PGP_PASSPHRASE
+          value: queenmarlena # TODO move to secret
+        volumeMounts:
+        - mountPath: /tmp/src/keys
+          name: pgp
+        ports:
+        - containerPort: 9000
+          name: server
+      volumes:
+      - name: pgp
+        configMap:
+          name: pgp-meta-test
+      restartPolicy: Always
+---
+# https://kubernetes.io/docs/concepts/services-networking/service/
+apiVersion: v1
+kind: Service
+metadata:
+  name: cic-user-ussd-svc
+  namespace: grassroots
+spec:
+  selector:
+    app: cic-user-ussd-server
+  type: ClusterIP
+  ports:
+  - name: server
+    protocol: TCP
+    port: 80
+    targetPort: 9000

kubernetes/cic-ussd/cic-ussd-sealedsecret.yaml

@@ -0,0 +1,15 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: cic-ussd-secret
+  namespace: grassroots
+spec:
+  encryptedData:
+    app_password_pepper: AgCrkW9x9iLDjcmmNHTopd1CXAdamuqWkiS3Tviev5OtR5d4FBw9L9Qm8P9ZsBiH9ucQSNu4XLxUuIEgyBLEPQDM5DELl+qvCmPboQ73kPxjEH4/ppqE1QnJHBFcTqfkUVDu4dBkmGPCbYAGrWqDa1bZt1Hujl+ZpWuc919nN2yxFEArs8jt8JIP2bbGKwPl21MDNP53ITtpw1euBswxUzLRcfUrw+6ghce8faFjDEnCS9NKftCwBFMQN2ddNn8aY0K1Dl/DJLTJw9c6L+3YAHzaQAkNz2I6W6ERri6Mmkc1bE1cujvftNxrWkPqh9bJD163xhMbA6vQd8rEpQelilBQpT0plcH5dj11mhAX+n3IR4Xp5kD+fwwPPgoQ5P3TJj4nfAccM/ubVVR6vLCXrO2TV56V8YgQ2pd0H3H8IJYRdU6LE9xLWdG5J32EKrhnhTNaGcFpWq3g4UUv6Fy0z7iZ8TkBce13zCiEf7AXeXzdj2KDgVW6/FBLknJoDd2m/j4GA5UC6MiKXc/X9sgHeSIOstR5nGBuYYr/12FbAPdnvV8zQzfHQsSCysciLQBqFhC5Yeepwu+UYsYCDWKGgvmC284zl52VkK2G6cXi77qcKNc6NtViZpKkJ0vRysgLLpBiDERB/NOrXef+kA6hJXsc5hqAPJjceh0XV0iT/4FA/2qq0A7qfm4meSJAxrj2PssoVQSUQmo2jopTCsa7IEq9opqy0yJaXu+dJG32IHimDLcAm1Sp0uumcd9MOg==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: cic-ussd-secret
+      namespace: grassroots
+

kubernetes/contract-migration/contract-migration-envlist-configmap.yaml

@@ -0,0 +1,68 @@
+# https://kubernetes.io/docs/concepts/configuration/configmap/
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: contract-migration-envlist
+  namespace: grassroots
+data:
+  envlist: |
+    SYNCER_LOOP_INTERVAL
+    SSL_ENABLE_CLIENT
+    SSL_CERT_FILE
+    SSL_KEY_FILE
+    SSL_PASSWORD
+    SSL_CA_FILE
+    BANCOR_DIR
+    REDIS_HOST
+    REDIS_PORT
+    REDIS_DB
+    PGP_PRIVATEKEY_FILE
+    PGP_PASSPHRASE
+    DATABASE_USER
+    DATABASE_PASSWORD
+    DATABASE_NAME
+    DATABASE_HOST
+    DATABASE_PORT
+    DATABASE_ENGINE
+    DATABASE_DRIVER
+    DATABASE_DEBUG
+    TASKS_AFRICASTALKING
+    TASKS_SMS_DB
+    TASKS_LOG
+    TASKS_TRACE_QUEUE_STATUS
+    TASKS_TRANSFER_CALLBACKS
+    DEV_MNEMONIC
+    DEV_ETH_RESERVE_ADDRESS
+    DEV_ETH_ACCOUNTS_INDEX_ADDRESS
+    DEV_ETH_RESERVE_AMOUNT
+    DEV_ETH_ACCOUNT_BANCOR_DEPLOYER
+    DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+    DEV_ETH_ACCOUNT_GAS_PROVIDER
+    DEV_ETH_ACCOUNT_RESERVE_OWNER
+    DEV_ETH_ACCOUNT_RESERVE_MINTER
+    DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER
+    DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
+    DEV_ETH_ACCOUNT_SARAFU_OWNER
+    DEV_ETH_ACCOUNT_SARAFU_GIFTER
+    DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
+    DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
+    DEV_ETH_SARAFU_TOKEN_NAME
+    DEV_ETH_SARAFU_TOKEN_SYMBOL
+    DEV_ETH_SARAFU_TOKEN_DECIMALS
+    DEV_ETH_SARAFU_TOKEN_ADDRESS
+    DEV_PGP_PUBLICKEYS_ACTIVE_FILE
+    DEV_PGP_PUBLICKEYS_TRUSTED_FILE
+    DEV_PGP_PUBLICKEYS_ENCRYPT_FILE
+    CIC_REGISTRY_ADDRESS
+    CIC_APPROVAL_ESCROW_ADDRESS
+    CIC_TOKEN_INDEX_ADDRESS
+    CIC_ACCOUNTS_INDEX_ADDRESS
+    CIC_DECLARATOR_ADDRESS
+    CIC_CHAIN_SPEC
+    ETH_PROVIDER
+    ETH_ABI_DIR
+    SIGNER_SOCKET_PATH
+    SIGNER_SECRET
+    CELERY_BROKER_URL
+    CELERY_RESULT_URL
+    META_PROVIDER

kubernetes/contract-migration/contract-migration-job.yaml

@@ -0,0 +1,122 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/job/
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: contract-migration
+  namespace: grassroots
+  labels:
+    app: contract-migration
+spec:
+  backoffLimit: 6
+  template:
+    spec:
+      imagePullSecrets:
+      - name: gitlab-internal-integration-registry
+      # securityContext:
+      #   runAsUser: 1000
+      #   runAsGroup: 1000
+      restartPolicy: Never
+      containers:
+      - name: contract-migration
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:master-621780e9-1618865959 # {"$imagepolicy": "flux-system:cic-contract-migration"}
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
+        command: ["./run_job.sh"]
+        # command: ["sleep", "3600"]
+        env:
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS # - name: ETH_GAS_PROVIDER_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: DATABASE_NAME
+          value: cic_eth
+        - name: REDIS_HOST
+          value: redis-master
+        - name: REDIS_PORT
+          value: "6379"
+        - name: REDIS_DB
+          value: "0"
+        - name: DEV_PIP_EXTRA_INDEX_URL
+          value: https://pip.grassrootseconomics.net:8433
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: ETH_PROVIDER_HOST
+          value: bloxberg-validator.grassroots.svc.cluster.local
+        - name: ETH_PROVIDER_PORT
+          value: "8547"
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: CIC_DATA_DIR
+          value: /tmp/cic/config
+        - name: RUN_MASK
+          value: "3" # bit flags; 1: contract migrations 2: seed data
+        - name: DEV_FAUCET_AMOUNT
+          value: "50000000"
+        - name: CIC_DEFAULT_TOKEN_SYMBOL
+          value: GFT
+        - name: DEV_SARAFU_DEMURRAGE_LEVEL
+          value: "196454828847045000000000000000000"
+        - name: DEV_ETH_GAS_PRICE
+          value: "1"
+        - name: TOKEN_TYPE
+          value: giftable_erc20_token
+        - name: WALLET_KEY_FILE
+          value: /root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+        volumeMounts:
+        - mountPath: /tmp/cic/config
+          name: migration-output
+        resources:
+          requests:
+            memory: "250Mi"
+            cpu: "100m"
+          limits:
+            memory: "500Mi"
+            cpu: "250m"
+      volumes:
+      - name: migration-output
+        emptyDir: {}

kubernetes/contract-migration/contract-migration-output-configMap.yaml

@@ -0,0 +1,11 @@
+# https://kubernetes.io/docs/concepts/configuration/configmap/
+# PURPOSE: These values are *manually* populated after execution of the contract migration container
+# The contract migration pod should output these vars among the STDOUT
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: contract-migration-output
+  namespace: grassroots
+data:
+  CIC_REGISTRY_ADDRESS: "0xea6225212005e86a4490018ded4bf37f3e772161"
+  CIC_TRUST_ADDRESS: "0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C"

kubernetes/contract-migration/contract-seeding-job.yaml

@@ -0,0 +1,118 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/job/
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: contract-seeding
+  namespace: grassroots
+  labels:
+    app: contract-seeding
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      imagePullSecrets:
+      - name: gitlab-internal-integration-registry
+      # securityContext:
+      #   runAsUser: 1000
+      #   runAsGroup: 1000
+      restartPolicy: Never
+      containers:
+      - name: registry-seeder
+        #image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:master-621780e9-1618865959 # {"$imagepolicy": "flux-system:cic-contract-migration"}
+        image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest 
+        command: ["./run_job.sh"]
+        # command: ["sleep", "3600"]
+        env:
+        - name: CIC_REGISTRY_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_REGISTRY_ADDRESS
+        - name: CIC_TRUST_ADDRESS # - name: ETH_GAS_PROVIDER_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: contract-migration-output
+              key: CIC_TRUST_ADDRESS
+        - name: DATABASE_USER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_USER
+        - name: DATABASE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_HOST
+        - name: DATABASE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PORT
+        - name: DATABASE_ENGINE
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_ENGINE
+        - name: DATABASE_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_DRIVER
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            configMapKeyRef:
+              name: postgresql-conn-common
+              key: DATABASE_PASSWORD
+        - name: CELERY_BROKER_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_BROKER_URL
+        - name: CELERY_RESULT_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-conn-common
+              key: CELERY_RESULT_URL
+        - name: DATABASE_NAME
+          value: cic_eth
+        - name: REDIS_HOST
+          value: redis-master
+        - name: REDIS_PORT
+          value: "6379"
+        - name: REDIS_DB
+          value: "0"
+        - name: DEV_PIP_EXTRA_INDEX_URL
+          value: https://pip.grassrootseconomics.net:8433
+        - name: ETH_PROVIDER
+          value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+        - name: ETH_PROVIDER_HOST
+          value: bloxberg-validator.grassroots.svc.cluster.local
+        - name: ETH_PROVIDER_PORT
+          value: "8547"
+        - name: CIC_CHAIN_SPEC
+          value: "evm:bloxberg:8996"
+        - name: CIC_DATA_DIR
+          value: /tmp/cic/config
+        - name: RUN_MASK
+          value: "2" # bit flags; 1: contract migrations 2: seed data
+        - name: DEV_FAUCET_AMOUNT
+          value: "50000000"
+        - name: CIC_DEFAULT_TOKEN_SYMBOL
+          value: GFT
+        - name: DEV_SARAFU_DEMURRAGE_LEVEL
+          value: "196454828847045000000000000000000"
+        - name: DEV_ETH_GAS_PRICE
+          value: "1"
+        volumeMounts:
+        - mountPath: /tmp/cic/config
+          name: migration-output
+        resources:
+          requests:
+            memory: "250Mi"
+            cpu: "100m"
+          limits:
+            memory: "500Mi"
+            cpu: "250m"
+      volumes:
+      - name: migration-output
+        emptyDir: {}

kubernetes/contract-migration/data-seeding-deployment.yaml

@@ -0,0 +1,229 @@
+# https://kubernetes.io/docs/concepts/workloads/pods/
+apiVersion: v1
+kind: Deployment 
+metadata:
+  name: data-seeding 
+  namespace: grassroots
+  labels:
+    app: data-seeding 
+    group: cic
+spec:
+  containers:
+    # This container should stay up for interactive use for now.
+  - name: data-seeding 
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:latest
+    command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
+    resources:
+      requests:
+        cpu: 50m
+        memory: 200Mi
+      limits:
+        cpu: 100m
+        memory: 400Mi
+    env:
+      - name: DATABASE_USER
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_USER
+      - name: DATABASE_HOST
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_HOST
+      - name: DATABASE_PORT
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_PORT
+      - name: DATABASE_ENGINE
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_ENGINE
+      - name: DATABASE_DRIVER
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_DRIVER
+      - name: DATABASE_PASSWORD
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_PASSWORD
+      - name: CELERY_BROKER_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_BROKER_URL
+      - name: CELERY_RESULT_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_RESULT_URL
+      - name: ETH_PROVIDER
+        value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+      - name: CIC_REGISTRY_ADDRESS
+        valueFrom:
+          configMapKeyRef:
+            name: contract-migration-output
+            key: CIC_REGISTRY_ADDRESS
+      - name: CIC_TRUST_ADDRESS
+        valueFrom:
+          configMapKeyRef:
+            name: contract-migration-output
+            key: CIC_TRUST_ADDRESS
+      - name: CELERY_BROKER_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_BROKER_URL
+      - name: CELERY_RESULT_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_RESULT_URL
+      - name: DATABASE_NAME
+        value: cic_eth
+      - name: META_URL
+        value: http://cic-meta-server:80
+      - name: META_HOST
+        value: cic-meta-server
+      - name: META_PORT
+        value: "80"
+      - name: PGP_PRIVATE_KEY_FILE # Private key here is for enrypting data
+        value: privatekey.asc
+      - name: PGP_PUBLIC_KEY_FILE
+        value: publickeys.asc
+      - name: PGP_PASSPHRASE
+        value: queenmarlena # TODO move to secret
+      - name: REDIS_HOST
+        value: redis-master
+      - name: REDIS_PORT
+        value: "6379"
+      - name: TOKEN_SYMBOL
+        value: "GFT"
+      - name: USER_USSD_HOST
+        value: cic-user-ussd-svc 
+      - name: USER_USSD_PORT
+        value: "80"
+      - name: KEYSTORE_FILE_PATH
+        value: /root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+    volumeMounts:
+    - mountPath: /tmp/src/keys
+      readOnly: true
+      name: pgp
+    - moutPath: /root/out
+      name: out-dir
+  - name: data-seeding-tasker 
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:latest
+    command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
+    resources:
+      requests:
+        cpu: 50m
+        memory: 200Mi
+      limits:
+        cpu: 100m
+        memory: 400Mi
+    env:
+      - name: DATABASE_USER
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_USER
+      - name: DATABASE_HOST
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_HOST
+      - name: DATABASE_PORT
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_PORT
+      - name: DATABASE_ENGINE
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_ENGINE
+      - name: DATABASE_DRIVER
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_DRIVER
+      - name: DATABASE_PASSWORD
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_PASSWORD
+      - name: CELERY_BROKER_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_BROKER_URL
+      - name: CELERY_RESULT_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_RESULT_URL
+      - name: ETH_PROVIDER
+        value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+      - name: CIC_REGISTRY_ADDRESS
+        valueFrom:
+          configMapKeyRef:
+            name: contract-migration-output
+            key: CIC_REGISTRY_ADDRESS
+      - name: CIC_TRUST_ADDRESS
+        valueFrom:
+          configMapKeyRef:
+            name: contract-migration-output
+            key: CIC_TRUST_ADDRESS
+      - name: CELERY_BROKER_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_BROKER_URL
+      - name: CELERY_RESULT_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_RESULT_URL
+      - name: DATABASE_NAME
+        value: cic_eth
+      - name: META_URL
+        value: http://cic-meta-server:80
+      - name: META_HOST
+        value: cic-meta-server
+      - name: META_PORT
+        value: "80"
+      - name: PGP_PRIVATE_KEY_FILE # Private key here is for enrypting data
+        value: privatekey.asc
+      - name: PGP_PUBLIC_KEY_FILE
+        value: publickeys.asc
+      - name: PGP_PASSPHRASE
+        value: queenmarlena # TODO move to secret
+      - name: REDIS_HOST
+        value: redis-master
+      - name: REDIS_PORT
+        value: "6379"
+      - name: TOKEN_SYMBOL
+        value: "GFT"
+      - name: USER_USSD_HOST
+        value: cic-user-ussd-svc 
+      - name: USER_USSD_PORT
+        value: "80"
+      - name: KEYSTORE_FILE_PATH
+        value: /root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+    volumeMounts:
+    - mountPath: /tmp/src/keys
+      readOnly: true
+      name: pgp
+    - moutPath: /root/out
+      name: out-dir
+  volumes:
+  - name: out-dir
+    emptyDir: {}
+  - name: pgp
+    configMap:
+      name: pgp-meta-test
+  restartPolicy: Never 

kubernetes/contract-migration/data-seeding-pod.yaml

@@ -0,0 +1,127 @@
+# https://kubernetes.io/docs/concepts/workloads/pods/
+apiVersion: v1
+kind: Pod
+metadata:
+  name: data-seeding 
+  namespace: grassroots
+  labels:
+    app: data-seeding 
+    group: cic
+spec:
+  imagePullSecrets: 
+  - name: gitlab-grassroots-registry
+  containers:
+  - name: data-seeding 
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:latest
+    command: ["sleep", "360000"]
+    resources:
+      requests:
+        cpu: 50m
+        memory: 200Mi
+      limits:
+        cpu: 100m
+        memory: 400Mi
+    env:
+      - name: DATABASE_USER
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_USER
+      - name: DATABASE_HOST
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_HOST
+      - name: DATABASE_PORT
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_PORT
+      - name: DATABASE_ENGINE
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_ENGINE
+      - name: DATABASE_DRIVER
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_DRIVER
+      - name: DATABASE_PASSWORD
+        valueFrom:
+          configMapKeyRef:
+            name: postgresql-conn-common
+            key: DATABASE_PASSWORD
+      - name: CELERY_BROKER_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_BROKER_URL
+      - name: CELERY_RESULT_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_RESULT_URL
+      - name: ETH_PROVIDER
+        value: http://bloxberg-validator.grassroots.svc.cluster.local:8547
+      - name: CIC_REGISTRY_ADDRESS
+        valueFrom:
+          configMapKeyRef:
+            name: contract-migration-output
+            key: CIC_REGISTRY_ADDRESS
+      - name: CIC_TRUST_ADDRESS
+        valueFrom:
+          configMapKeyRef:
+            name: contract-migration-output
+            key: CIC_TRUST_ADDRESS
+      - name: CELERY_BROKER_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_BROKER_URL
+      - name: CELERY_RESULT_URL
+        valueFrom:
+          configMapKeyRef:
+            name: redis-conn-common
+            key: CELERY_RESULT_URL
+      - name: DATABASE_NAME
+        value: cic_eth
+      - name: META_URL
+        value: http://cic-meta-server:80
+      - name: META_HOST
+        value: cic-meta-server
+      - name: META_PORT
+        value: "80"
+      - name: PGP_PRIVATE_KEY_FILE # Private key here is for enrypting data
+        value: privatekey.asc
+      - name: PGP_PUBLIC_KEY_FILE
+        value: publickeys.asc
+      - name: PGP_PASSPHRASE
+        value: queenmarlena # TODO move to secret
+      - name: REDIS_HOST
+        value: redis-master
+      - name: REDIS_PORT
+        value: "6379"
+      - name: TOKEN_SYMBOL
+        value: "GFT"
+      - name: USER_USSD_HOST
+        value: cic-user-ussd-svc 
+      - name: USER_USSD_PORT
+        value: "80"
+      - name: KEYSTORE_FILE_PATH
+        value: /root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+      - name: PGP_EXPORTS_DIR
+        value: /tmp/src/keys
+      - name: OUT_DIR
+        value: /root/out
+      - name: NUMBER_OF_USERS
+        value: "10"
+    volumeMounts:
+    - mountPath: /tmp/src/keys
+      readOnly: true
+      name: pgp
+  volumes:
+  - name: pgp
+    configMap:
+      name: pgp-meta-test
+  restartPolicy: Never