Revert bump in contract migration dockerfile

add permissions and move some pip installs

See merge request grassrootseconomics/cic-internal-integration!69

apps/cic-cache/.config/database.ini

@@ -1,5 +1,5 @@
 [database]
-NAME=cic-eth
+NAME=cic_cache
 USER=postgres
 PASSWORD=
 HOST=localhost

apps/cic-cache/cic_cache/db/migrations/default/alembic.ini

@@ -36,7 +36,7 @@ script_location = .
 # output_encoding = utf-8
 
 #sqlalchemy.url = driver://user:pass@localhost/dbname
-sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic-cache
+sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic_cache
 
 
 [post_write_hooks]

apps/cic-cache/db/initdb_files/create_all_db.sql

@@ -1,5 +0,0 @@
-CREATE DATABASE "cic-cache";
-CREATE DATABASE "cic-eth";
-CREATE DATABASE "cic-notify";
-CREATE DATABASE "cic-meta";
-CREATE DATABASE "cic-signer";

apps/cic-cache/setup.cfg

@@ -25,6 +25,7 @@ licence_files =
 python_requires = >= 3.6
 packages = 
 	cic_cache
+	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
 	cic_cache.runnable
@@ -33,5 +34,6 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-cache-tracker = cic_cache.runnable.tracker:main
-	cic-cache-server = cic_cache.runnable.server:main
+	cic-cache-trackerd = cic_cache.runnable.tracker:main
+	cic-cache-serverd = cic_cache.runnable.server:main
+	cic-cache-taskerd = cic_cache.runnable.tasker:main

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -10,12 +10,15 @@ from cic_registry import zero_address
 from cic_eth.db.enum import LockEnum
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.lock import Lock
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        )
 from cic_eth.error import LockedError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
@@ -33,7 +36,7 @@ def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
@@ -51,7 +54,7 @@ def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set send lock
 
@@ -67,7 +70,7 @@ def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock_send(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset send lock
 
@@ -83,7 +86,7 @@ def unlock_send(chained_input, chain_str, address=zero_address):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set queue direct lock
 
@@ -99,7 +102,7 @@ def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock_queue(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset queue direct lock
 
@@ -115,7 +118,7 @@ def unlock_queue(chained_input, chain_str, address=zero_address):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_str, lock_flags, address=None):
     session = SessionBase.create_session()
     r = Lock.check(chain_str, lock_flags, address=zero_address, session=session)

apps/cic-eth/cic_eth/api/api_task.py

@@ -82,6 +82,7 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -143,6 +144,7 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -340,11 +342,6 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_nonce = celery.signature(
-                'cic_eth.eth.tx.reserve_nonce',
-                [],
-                queue=self.queue,
-                )
         s_account = celery.signature(
                 'cic_eth.eth.account.create',
                 [
@@ -352,12 +349,18 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_nonce.link(s_account)
-        s_check.link(s_nonce)
+        s_check.link(s_account)
         if self.callback_param != None:
             s_account.link(self.callback_success)
 
         if register:
+            s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
+                [
+                    'ACCOUNTS_INDEX_WRITER',
+                    ],
+                queue=self.queue,
+                )
             s_register = celery.signature(
                 'cic_eth.eth.account.register', 
                 [
@@ -365,7 +368,8 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-            s_account.link(s_register)
+            s_nonce.link(s_register)
+            s_account.link(s_nonce)
 
         t = s_check.apply_async(queue=self.queue)
         return t
@@ -390,7 +394,9 @@ class Api:
                 )
         s_nonce = celery.signature(
                 'cic_eth.eth.tx.reserve_nonce',
-                [],
+                [
+                    'GAS_GIFTER',
+                    ],
                 queue=self.queue,
                 )
         s_refill = celery.signature(

apps/cic-eth/cic_eth/error.py

@@ -73,3 +73,10 @@ class SignerError(Exception):
 
     """
     pass
+
+
+class EthError(Exception):
+    """Exception raised when unspecified error from evm node is encountered
+
+    """
+    pass

apps/cic-eth/cic_eth/eth/gas.py

@@ -54,6 +54,7 @@ class GasOracle():
         """
         session = SessionBase.create_session()
         a = AccountRole.get_address('GAS_GIFTER', session)
+        logg.debug('gasgifter {}'.format(a))
         session.close()
         return a
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -14,6 +14,7 @@ from cic_eth.db import Otx, SessionBase
 from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.nonce import NonceReservation
 from cic_eth.db.models.lock import Lock
+from cic_eth.db.models.role import AccountRole
 from cic_eth.db.enum import (
         LockEnum,
         StatusBits,
@@ -38,6 +39,7 @@ from cic_eth.task import (
         CriticalWeb3Task,
         CriticalWeb3AndSignerTask,
         CriticalSQLAlchemyAndSignerTask,
+        CriticalSQLAlchemyAndWeb3Task,
         )
 
 celery_app = celery.current_app
@@ -47,7 +49,7 @@ MAX_NONCE_ATTEMPTS = 3
 
 
 # TODO this function is too long
-@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
 def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=None):
     """Check the gas level of the sender address of a transaction.
 
@@ -75,6 +77,9 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
             if address == None:
                 address = o['address']
 
+    if not web3.Web3.isChecksumAddress(address):
+        raise ValueError('invalid address {}'.format(address))
+
     chain_spec = ChainSpec.from_chain_str(chain_str)
 
     queue = self.request.delivery_info['routing_key']
@@ -83,7 +88,12 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
     c = RpcClient(chain_spec)
 
     # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
-    balance = c.w3.eth.getBalance(address) 
+    balance = 0
+    try:
+        balance = c.w3.eth.getBalance(address) 
+    except ValueError as e:
+        raise EthError('balance call for {}'.format())
+
     logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required))
 
     if gas_required > balance:
@@ -590,11 +600,23 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
 
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
-def reserve_nonce(self, chained_input, address=None):
+def reserve_nonce(self, chained_input, signer=None):
     session = SessionBase.create_session()
 
-    if address == None:
+    address = None
+    if signer == None:
         address = chained_input
+        logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
+    else:
+        if web3.Web3.isChecksumAddress(signer):
+            address = signer
+            logg.debug('explicit address for reserve nonce {}'.format(signer))
+        else:
+            address = AccountRole.get_address(signer, session=session)
+            logg.debug('role for reserve nonce {} -> {}'.format(signer, address))
+
+    if not web3.Web3.isChecksumAddress(address):
+        raise ValueError('invalid result when resolving address for nonce {}'.format(address))
 
     root_id = self.request.root_id
     nonce = NonceReservation.next(address, root_id)

apps/cic-eth/cic_eth/runnable/create.py

@@ -5,6 +5,7 @@ import os
 import logging
 import uuid
 import json
+from xdg.BaseDirectory import xdg_config_home
 
 import celery
 from cic_eth.api import Api

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -25,7 +25,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
 
 
 default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')

apps/cic-eth/cic_eth/runnable/resend.py

@@ -22,7 +22,7 @@ logg = logging.getLogger()
 logging.getLogger('web3').setLevel(logging.WARNING)
 logging.getLogger('urllib3').setLevel(logging.WARNING)
 
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 
 argparser = argparse.ArgumentParser()

apps/cic-eth/cic_eth/runnable/view.py

@@ -37,7 +37,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
 
 
 default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)')

apps/cic-eth/cic_eth/task.py

@@ -6,7 +6,10 @@ import celery
 import sqlalchemy
 
 # local imports
-from cic_eth.error import SignerError
+from cic_eth.error import (
+        SignerError,
+        EthError,
+        )
 
 
 class CriticalTask(celery.Task):
@@ -33,6 +36,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         requests.exceptions.ConnectionError,
+        EthError,
         )
 
 class CriticalSQLAlchemyAndSignerTask(CriticalTask):

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         0,
-        'alpha.38',
+        'alpha.41',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
-cic-base~=0.1.1a10
+cic-base~=0.1.1a20
 web3==5.12.2
 celery==4.4.7
 crypto-dev-signer~=0.4.13rc4

apps/cic-notify/cic_notify/version.py

@@ -10,7 +10,7 @@ from cic_notify.error import PleaseCommitFirstError
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.2')
+version = (0, 4, 0, 'alpha.3')
 
 version_object = semver.VersionInfo(
         major=version[0],
@@ -24,9 +24,6 @@ version_string = str(version_object)
 
 def git_hash():
     import subprocess
-    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
-    if len(git_diff.stdout) > 0:
-        raise PleaseCommitFirstError()
 
     git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
     git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
@@ -35,7 +32,7 @@ def git_hash():
 
 try:
     version_git = git_hash()
-    version_string += '.build.{}'.format(version_git)
+    version_string += '+build.{}'.format(version_git)
 except FileNotFoundError:
     time_string_pair = str(time.time()).split('.')
     version_string += '+build.{}{:<09d}'.format(

apps/cic-notify/requirements.txt

@@ -1,5 +1,5 @@
-celery~=4.4.7
-confini~=0.3.6a1
 alembic~=1.4.2
+celery~=4.4.7
+confini~=0.3.6rc3
 redis~=3.5.3
 semver==2.13.0

apps/cic-notify/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-notify
-version= attr: cic_notify.version.__version_string__
+version= 0.4.0a2
 description = CIC notifications service
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-ussd/.config_client/app.ini

@@ -0,0 +1,2 @@
+[app]
+service_code = *483*46#

apps/cic-ussd/.config_client/client.ini

@@ -0,0 +1,4 @@
+[client]
+host = 
+port = 
+ssl =

apps/cic-ussd/.config_client/ussd.ini

@@ -0,0 +1,3 @@
+[ussd]
+user =
+pass =

apps/cic-ussd/cic_ussd/operations.py

@@ -14,7 +14,7 @@ from cic_ussd.db.models.user import User
 from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.menu.ussd_menu import UssdMenu
-from cic_ussd.processor import custom_display_text, process_request
+from cic_ussd.processor import custom_display_text, process_request, retrieve_most_recent_ussd_session
 from cic_ussd.redis import InMemoryStore
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.validator import check_known_user, validate_response_type
@@ -60,7 +60,8 @@ def create_ussd_session(
         phone: str,
         service_code: str,
         user_input: str,
-        current_menu: str) -> InMemoryUssdSession:
+        current_menu: str,
+        session_data: Optional[dict] = None) -> InMemoryUssdSession:
     """
     Creates a new ussd session
     :param external_session_id: Session id value provided by AT
@@ -73,6 +74,8 @@ def create_ussd_session(
     :type user_input: str
     :param current_menu: Menu name that is currently being displayed on the ussd session
     :type current_menu: str
+    :param session_data: Any additional data that was persisted during the user's interaction with the system.
+    :type session_data: dict.
     :return: ussd session object
     :rtype: Session
     """
@@ -81,7 +84,8 @@ def create_ussd_session(
         msisdn=phone,
         user_input=user_input,
         state=current_menu,
-        service_code=service_code
+        service_code=service_code,
+        session_data=session_data
     )
     return session
 
@@ -126,7 +130,9 @@ def create_or_update_session(
             phone=phone,
             service_code=service_code,
             user_input=user_input,
-            current_menu=current_menu)
+            current_menu=current_menu,
+            session_data=session_data
+        )
     return ussd_session
 
 
@@ -338,14 +344,26 @@ def process_menu_interaction_requests(chain_str: str,
                 user_input=user_input
             )
 
-        # create or update the ussd session as appropriate
-        ussd_session = create_or_update_session(
-            external_session_id=external_session_id,
-            phone=phone_number,
-            service_code=service_code,
-            user_input=user_input,
-            current_menu=current_menu.get('name')
-        )
+        last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
+
+        if last_ussd_session:
+            # create or update the ussd session as appropriate
+            ussd_session = create_or_update_session(
+                external_session_id=external_session_id,
+                phone=phone_number,
+                service_code=service_code,
+                user_input=user_input,
+                current_menu=current_menu.get('name'),
+                session_data=last_ussd_session.session_data
+            )
+        else:
+            ussd_session = create_or_update_session(
+                external_session_id=external_session_id,
+                phone=phone_number,
+                service_code=service_code,
+                user_input=user_input,
+                current_menu=current_menu.get('name')
+            )
 
         # define appropriate response
         response = custom_display_text(

apps/cic-ussd/cic_ussd/processor.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 # third party imports
 import celery
-from cic_types.models.person import Person
+from sqlalchemy import desc
 from tinydb.table import Document
 
 # local imports
@@ -315,6 +315,16 @@ def process_start_menu(display_key: str, user: User):
     )
 
 
+def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession:
+    # get last ussd session based on user phone number
+    last_ussd_session = UssdSession.session\
+        .query(UssdSession)\
+        .filter_by(msisdn=phone_number)\
+        .order_by(desc(UssdSession.created))\
+        .first()
+    return last_ussd_session
+
+
 def process_request(user_input: str, user: User, ussd_session: Optional[dict] = None) -> Document:
     """This function assesses a request based on the user from the request comes, the session_id and the user's
     input. It determines whether the request translates to a return to an existing session by checking whether the
@@ -337,7 +347,23 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] =
             return UssdMenu.find_by_name(name=successive_state)
     else:
         if user.has_valid_pin():
-            return UssdMenu.find_by_name(name='start')
+            last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
+
+            key = create_cached_data_key(
+                identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
+                salt='cic.person'
+            )
+            user_metadata = get_cached_data(key=key)
+
+            if last_ussd_session:
+                # get last state
+                last_state = last_ussd_session.state
+                logg.debug(f'LAST USSD SESSION STATE: {last_state}')
+                # if last state is account_creation_prompt and metadata exists, show start menu
+                if last_state == 'account_creation_prompt' and user_metadata is not None:
+                    return UssdMenu.find_by_name(name='start')
+                else:
+                    return UssdMenu.find_by_name(name=last_state)
         else:
             if user.failed_pin_attempts >= 3 and user.get_account_status() == AccountStatus.LOCKED.name:
                 return UssdMenu.find_by_name(name='exit_pin_blocked')

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+
+# Author: Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
+# Description: interactive console for Sempo USSD session
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# standard imports
+import os
+import sys
+import uuid
+import json
+import argparse
+import logging
+import urllib
+from xdg.BaseDirectory import xdg_config_home
+from urllib import request
+
+# third-party imports
+from confini import Config
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+argparser = argparse.ArgumentParser(description='CLI tool to interface a Sempo USSD session')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+#argparser.add_argument('-d', type=str, default='local', help='deployment name to interface (config root subdirectory)')
+argparser.add_argument('--host', type=str, default='localhost')
+argparser.add_argument('--port', type=int, default=9000)
+argparser.add_argument('--nossl', help='do not use ssl (careful)', action='store_true')
+argparser.add_argument('phone', help='phone number for USSD session')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+#config_dir = os.path.join(args.c, args.d)
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+
+config = Config(config_dir)
+config.process()
+logg.debug('config loaded from {}'.format(config_dir))
+
+host = config.get('CLIENT_HOST')
+port = config.get('CLIENT_PORT')
+ssl = config.get('CLIENT_SSL')
+
+if host == None:
+    host = args.host
+if port == None:
+    port = args.port
+if ssl == None:
+    ssl = not args.nossl
+elif ssl == 0:
+    ssl = False
+else:
+    ssl = True
+
+
+def main():
+
+    # TODO: improve url building 
+    url = 'http'
+    if ssl:
+        url += 's'
+    url += '://{}:{}'.format(host, port)
+    url += '/?username={}&password={}'.format(config.get('USSD_USER'), config.get('USSD_PASS'))
+
+    logg.info('service url {}'.format(url))
+    logg.info('phone {}'.format(args.phone))
+
+    session = uuid.uuid4().hex
+    data = {
+            'sessionId': session,
+            'serviceCode': config.get('APP_SERVICE_CODE'),
+            'phoneNumber': args.phone,
+            'text': config.get('APP_SERVICE_CODE'),
+        }
+
+    state = "_BEGIN"
+    while state != "END":
+
+        if state != "_BEGIN":
+            user_input = input('next> ')
+            data['text'] = user_input
+
+        req = urllib.request.Request(url)
+        data_str = json.dumps(data)
+        data_bytes = data_str.encode('utf-8')
+        req.add_header('Content-Type', 'application/json')
+        req.data = data_bytes
+        response = urllib.request.urlopen(req)
+        response_data = response.read().decode('utf-8')
+        state = response_data[:3]
+        out = response_data[4:]
+        print(out)
+
+
+if __name__ == "__main__":
+    main()

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.1')
+version = (0, 3, 0, 'alpha.5')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/docker/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.8.5-alpine
+# FROM python:3.8.5-alpine
+FROM python:3.8.6-slim-buster
 
 # set working directory
 WORKDIR /usr/src
@@ -6,10 +7,8 @@ WORKDIR /usr/src
 # add args for installing from self-hosted packages
 ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
 
-# add alpine sys packages
-RUN apk update && \
-        apk add git linux-headers postgresql-dev gnupg bash
-RUN apk add --update musl-dev gcc libffi-dev
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
 # create secrets directory
 RUN mkdir -vp pgp/keys

apps/cic-ussd/emulator/browser/index.html

@@ -16,9 +16,16 @@ div#session {
 		<textarea id="monitor" disabled="1"></textarea>
 		<div id="login">
 			<label for="user">API username</label>
-			<input type="text" id="user" name="user" type="text" /><br/>
+			<input type="text" id="user" name="user" />  
 			<label for="user">API password</label>
-			<input type="text" id="pass" name="pass" type="text" /><br/>
+			<input type="text" id="pass" name="pass" /> <br/>
+
+			<label for="host">API host</label>
+			 <input type="text" id="host" name="host" />
+			<label for="host">API port</label>
+			 <input type="text" id="port" name="port" />
+			<label for="host">SSL</label>
+			<input type="checkbox" id="ssl" name="ssl" checked="1"/> <br/>
 			<hr/>
 			<input type="text" id="phone" /> <button onclick="setPhone(document.getElementById('phone').value);" id="send_phone">set phone number</button> 
 		</div>

apps/cic-ussd/emulator/browser/ussd.js

@@ -1,10 +1,12 @@
-//var proto = 'http';
-//var host = 'localhost:9000';
+var ssl = false;
+var host = 'localhost';
+var port = 9000;
 var proto = 'https';
 var host = 'staging.sarafu.network';
-var user = 'admin_bert_token_inc.';
-var pass = '197781ed60bf16d5dc12d84e3df37e35';
-var serviceCode = '*483*061#';
+var user = 'foo';
+var pass = 'bar';
+var path = '/';
+var serviceCode = '*483*46#';
 
 // cheekily stolen from https://www.tutorialspoint.com/how-to-create-guid-uuid-in-javascript
 function createUUID() {
@@ -23,9 +25,17 @@ function send(s) {
 	document.getElementById('send_input').disabled = true;
 	var xhr = new XMLHttpRequest();
 	xhr.responseType = 'text';
-	current_user = document.getElementById('user').value;
-	current_pass = document.getElementById('pass').value;
-	xhr.open('POST', proto + '://' + host + '/api/v1/ussd/kenya?username=' + current_user + '&password=' + current_pass, true);
+	const current_user = document.getElementById('user').value;
+	const current_pass = document.getElementById('pass').value;
+	const current_host = document.getElementById('host').value;
+	const current_port = document.getElementById('port').value;
+	let current_scheme = 'http';
+	if (document.getElementById('ssl').checked) {
+		current_scheme += 's';
+	}
+	const url = current_scheme + '://' + current_host + ':' + current_port + '?username=' + current_user + '&password=' + current_pass
+	console.debug('connecting to', url);
+	xhr.open('POST', url, true);
 	xhr.setRequestHeader('Content-Type', 'application/json');
 	data = {
 		sessionId: uuid,
@@ -106,6 +116,8 @@ function abort() {
 window.addEventListener('load', () => {
 	document.getElementById('user').value = user;
 	document.getElementById('pass').value = pass;
+	document.getElementById('host').value = host;
+	document.getElementById('port').value = port;
 	document.getElementById('phone').addEventListener('keyup', (e) => {
 		if (e.keyCode == '13') {
 			document.getElementById('input').value = '';

apps/cic-ussd/requirements.txt

@@ -6,12 +6,11 @@ betterpath==0.2.2
 billiard==3.6.3.0
 celery==4.4.7
 cffi==1.14.3
-chainlib~=0.0.1a15
-cic-eth==0.10.0a38
-cic-notify==0.3.1
-cic-types==0.1.0a8
+cic-eth~=0.10.0a41
+cic-notify~=0.4.0a3
+cic-types~=0.1.0a8
 click==7.1.2
-confini==0.3.5
+confini~=0.3.6rc3
 cryptography==3.2.1
 faker==4.17.1
 iniconfig==1.1.1
@@ -36,7 +35,6 @@ python-i18n==0.3.9
 pytz==2020.1
 PyYAML==5.3.1
 redis==3.5.3
-requests==2.24.0
 semver==2.13.0
 six==1.15.0
 SQLAlchemy==1.3.20
@@ -46,4 +44,4 @@ transitions==0.8.4
 uWSGI==2.0.19.1
 vcversioner==2.16.0.0
 vine==1.3.0
-zope.interface==5.1.2
+zope.interface==5.1.2

apps/cic-ussd/setup.cfg

@@ -44,4 +44,5 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-ussd-tasker = cic_ussd.runnable.tasker:main
+	cic-ussd-tasker = cic_ussd.runnable.tasker:main
+	cic-ussd-client = cic_ussd.runnable.client:main

apps/contract-migration/docker/Dockerfile

@@ -1,14 +1,22 @@
-#FROM ethereum/solc:0.6.12
-FROM ethereum/solc:0.8.0
+# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster as compile-image
 
-# The solc image messes up the alpine environment, so we have to go all over again
-FROM alpine
-COPY --from=0 /usr/bin/solc /usr/bin/solc
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
 
-RUN apk update && \
-	apk add make git
-	
-WORKDIR /usr/src
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository ppa:ethereum/ethereum
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C52189C923F6CA9
+RUN apt-get update
+RUN apt-get install solc
+RUN pip install --upgrade pip
+
+WORKDIR /root
+RUN mkdir -vp /usr/local/etc/cic
+
+COPY contract-migration/nvm.sh .
+ENV CONFINI_DIR /usr/local/etc/cic/
+RUN mkdir -vp $CONFINI_DIR
 
 ARG cic_config_commit=35c69ba75f00c8147150acf325565d5391cf25bf
 ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
@@ -16,11 +24,8 @@ RUN echo Install confini schema files && \
 	git clone --depth 1 $cic_config_url cic-config && \
 	cd cic-config && \
 	git fetch --depth 1 origin $cic_config_commit && \
-	git checkout $cic_config_commit && \
-	mkdir -vp /usr/local/etc/cic && \
-	cp -v *.ini /usr/local/etc/cic/
-ENV CONFINI_DIR /usr/local/etc/cic
-
+	git checkout $cic_config_commit && \	
+	cp -v *.ini $CONFINI_DIR
 
 ARG cic_contracts_commit=698ef3a30fde8d7f2c498f1208fb0ff45d665501
 ARG cic_contracts_url=https://gitlab.com/grassrootseconomics/cic-contracts.git/
@@ -31,30 +36,6 @@ RUN echo Install ABI collection for solidity interfaces used across all componen
 	git checkout $cic_contracts_commit && \
 	make install
 
-#COPY ./Makefile ./cic-contracts/Makefile
-#COPY ./*.sol ./cic-contracts/
-
-#RUN cd cic-contracts && \
-#	make -B && make install -B 
-
-FROM python:3.8.6-slim-buster
-
-COPY --from=1 /usr/local/share/cic/ /usr/local/share/cic/
-COPY --from=1 /usr/local/etc/ /usr/local/etc/
-
-LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
-LABEL spdx-license-identifier="GPL-3.0-or-later"
-LABEL description="Base layer for buiding development images for the cic component suite"
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
-
-
-RUN echo installing nodejs tooling
-
-COPY contract-migration/nvm.sh /root/
-
 # Install nvm with node and npm
 # https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
 ENV NVM_DIR /root/.nvm
@@ -65,67 +46,100 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh |
 	&& . $NVM_DIR/nvm.sh \
 	&& nvm install $NODE_VERSION \
 	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION \
-# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
-# owner of these files is "1001" by default - wtf
-	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
+	&& nvm use $NODE_VERSION
+# && chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
 
 ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
 ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
 
+# RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version
+
+RUN useradd --create-home grassroots
+WORKDIR /home/grassroots
+USER grassroots
 
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
+ARG cic_base_version=0.1.1a23
+ARG cic_registry_version=0.5.3a24
+ARG cic_eth_version=0.10.0a41
+ARG chainlib_version=0.0.1a21
+ARG cic_contracts_version=0.0.2a2
+RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
+                                                              cic-registry==$cic_registry_version \
+                                                              cic-eth==$cic_eth_version \
+                                                              chainlib==$chainlib_version \
+															  cic-contracts==$cic_contracts_version
+
+# ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/
+# ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity
+# RUN echo Compile and install bancor protocol contracts && \
+# 	git clone --depth 1 $cic_bancor_url cic-bancor && \
+# 	cd cic-bancor
+
+# RUN cd cic-bancor/python && \
+# 	pip install --extra-index-url $pip_extra_index_url .
 
 # This is a temporary solution for building the Bancor contracts using the bancor protocol repository truffle setup
 # We should instead flatten the files ourselves and build them with solc in the first image layer in this file
-ARG cic_bancor_commit=a04c7ae6882ea515938d852cc861d59a35070094
-ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/
-ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity
-RUN echo Compile and install bancor protocol contracts && \
-	git clone --depth 1 $cic_bancor_url cic-bancor && \
-	cd cic-bancor && \
-	git fetch --depth 1 origin $cic_bancor_commit && \
-	git checkout $cic_bancor_commit && \
-	# Apparently the git version here doesn't have set-url as a command. *sigh*
-	#if [ ! -z $cic_bancor_contracts_url ]; then
-	#	git submodule set-url bancor $cic_bancor_contracts_url
-	#fi
-	git submodule init && \
-	git submodule update
-RUN cd root && \
-	. $NVM_DIR/nvm.sh &&\
-	nvm install $BANCOR_NODE_VERSION && \
-	nvm use $BANCOR_NODE_VERSION && \
-	cd - && \
-	cd cic-bancor/bancor && \
-	npm install --python=/usr/bin/python2 && \
-	node_modules/truffle/build/cli.bundled.js compile && \
-	mkdir -vp /usr/local/share/cic/bancor/solidity/build  && \
-	cp -vR solidity/build/contracts /usr/local/share/cic/bancor/solidity/build/
-RUN cd cic-bancor/python && \
-	pip install --extra-index-url $pip_extra_index_url .
+# ARG cic_bancor_commit=a04c7ae6882ea515938d852cc861d59a35070094
+# ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/
+# ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity
+# RUN echo Compile and install bancor protocol contracts && \
+# 	git clone --depth 1 $cic_bancor_url cic-bancor && \
+# 	cd cic-bancor && \
+# 	git fetch --depth 1 origin $cic_bancor_commit && \
+# 	git checkout $cic_bancor_commit && \
+# 	# Apparently the git version here doesn't have set-url as a command. *sigh*
+# 	#if [ ! -z $cic_bancor_contracts_url ]; then
+# 	#	git submodule set-url bancor $cic_bancor_contracts_url
+# 	#fi
+# 	git submodule init && \
+# 	git submodule update
+# RUN cd root && \
+# 	. $NVM_DIR/nvm.sh &&\
+# 	nvm install $BANCOR_NODE_VERSION && \
+# 	nvm use $BANCOR_NODE_VERSION && \
+# 	cd - && \
+# 	cd cic-bancor/bancor && \
+# 	npm install --python=/usr/bin/python2 && \
+# 	node_modules/truffle/build/cli.bundled.js compile && \
+# 	mkdir -vp /usr/local/share/cic/bancor/solidity/build  && \
+# 	cp -vR solidity/build/contracts /usr/local/share/cic/bancor/solidity/build/
+# RUN cd cic-bancor/python && \
+# 	pip install --extra-index-url $pip_extra_index_url .
 
+FROM python:3.8.6-slim-buster as runtime-image
 
-RUN apt-get install -y cargo
-ARG cic_base_version=0.1.1a10
-RUN pip install  --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends gnupg libpq-dev
 
-ARG cic_registry_version=0.5.3a22
-RUN pip install  --extra-index-url $pip_extra_index_url  cic-registry==$cic_registry_version
+COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
+COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
 
-WORKDIR /root
+RUN useradd --create-home grassroots
+WORKDIR /home/grassroots
+# COPY python dependencies to user dir
+COPY --from=compile-image /home/grassroots/.local .local
+ENV PATH=/home/grassroots/.local/bin:$PATH
 
 COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/wait-for-it.sh .
-RUN chmod +x ./wait-for-it.sh
-
-# COPY contract-migration/.env_config_template .env_config_template
-# COPY contract-migration/.env_dockercompose_template .env_dockercompose_template
-
 COPY contract-migration/reset.sh reset.sh
 COPY contract-migration/from_env.sh from_env.sh
 COPY contract-migration/seed_cic_eth.sh seed_cic_eth.sh
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
+COPY contract-migration/envlist .
 
-ENTRYPOINT [ "/bin/bash" ]
+# RUN chown grassroots:grassroots .local/
+RUN chown grassroots:grassroots ./
+RUN chmod gu+x *.sh
+
+RUN mkdir -p /tmp/cic/config
+RUN chown grassroots:grassroots /tmp/cic/config
+# A shared output dir for environment configs
+RUN chmod a+rwx /tmp/cic/config 
+
+USER grassroots
+
+ENTRYPOINT [ ]

apps/contract-migration/envlist

@@ -0,0 +1,61 @@
+SYNCER_LOOP_INTERVAL
+SSL_ENABLE_CLIENT
+SSL_CERT_FILE
+SSL_KEY_FILE
+SSL_PASSWORD
+SSL_CA_FILE
+BANCOR_DIR
+REDIS_HOST
+REDIS_PORT
+REDIS_DB
+PGP_EXPORTS_DIR
+PGP_PRIVATEKEY_FILE
+PGP_PASSPHRASE
+DATABASE_USER
+DATABASE_PASSWORD
+DATABASE_NAME
+DATABASE_HOST
+DATABASE_PORT
+DATABASE_ENGINE
+DATABASE_DRIVER
+DATABASE_DEBUG
+TASKS_AFRICASTALKING
+TASKS_SMS_DB
+TASKS_LOG
+TASKS_TRACE_QUEUE_STATUS
+TASKS_TRANSFER_CALLBACKS
+DEV_MNEMONIC
+DEV_ETH_RESERVE_ADDRESS
+DEV_ETH_ACCOUNTS_INDEX_ADDRESS
+DEV_ETH_RESERVE_AMOUNT
+DEV_ETH_ACCOUNT_BANCOR_DEPLOYER
+DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+DEV_ETH_ACCOUNT_GAS_PROVIDER
+DEV_ETH_ACCOUNT_RESERVE_OWNER
+DEV_ETH_ACCOUNT_RESERVE_MINTER
+DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER
+DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
+DEV_ETH_ACCOUNT_SARAFU_OWNER
+DEV_ETH_ACCOUNT_SARAFU_GIFTER
+DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
+DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
+DEV_ETH_SARAFU_TOKEN_NAME
+DEV_ETH_SARAFU_TOKEN_SYMBOL
+DEV_ETH_SARAFU_TOKEN_DECIMALS
+DEV_ETH_SARAFU_TOKEN_ADDRESS
+DEV_PGP_PUBLICKEYS_ACTIVE_FILE
+DEV_PGP_PUBLICKEYS_TRUSTED_FILE
+DEV_PGP_PUBLICKEYS_ENCRYPT_FILE
+CIC_REGISTRY_ADDRESS
+CIC_APPROVAL_ESCROW_ADDRESS
+CIC_TOKEN_INDEX_ADDRESS
+CIC_ACCOUNTS_INDEX_ADDRESS
+CIC_DECLARATOR_ADDRESS
+CIC_CHAIN_SPEC
+ETH_PROVIDER
+ETH_ABI_DIR
+SIGNER_SOCKET_PATH
+SIGNER_SECRET
+CELERY_BROKER_URL
+CELERY_RESULT_URL
+META_PROVIDER

apps/contract-migration/reset.sh

@@ -61,10 +61,10 @@ export DEV_ETH_ACCOUNTS_INDEX_ADDRESS=$CIC_ACCOUNTS_INDEX_ADDRESS
 export BANCOR_REGISTRY_ADDRESS=$BANCOR_REGISTRY_ADDRESS
 export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
 export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
-
+export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
 EOF
 
-cat $CIC_DATA_DIR/envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
+cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
 # popd
 
 set +a

apps/contract-migration/scripts/create_import_users.py

@@ -191,9 +191,16 @@ if __name__ == '__main__':
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
 
-    for i in range(user_count):
-    
-        (eth, phone, o) = gen()
+    i = 0
+    while i < user_count:
+        eth = None
+        phone = None
+        o = None
+        try:
+            (eth, phone, o) = gen()
+        except Exception as e:
+            logg.warning('generate failed, trying anew: {}'.format(e))
+            continue
         uid = eth[2:].upper()
 
         print(o)
@@ -212,5 +219,7 @@ if __name__ == '__main__':
         amount = genAmount()
         fa.write('{},{}\n'.format(eth,amount))
         logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
+        
+        i += 1
 
     fa.close()

apps/contract-migration/scripts/import_meta.js

@@ -74,7 +74,7 @@ new cic.PGPKeyStore(
 	importMeta,
 );
 
-const batchSize = 50;
+const batchSize = 16;
 const batchDelay = 1000;
 const total = parseInt(process.argv[3]);
 const workDir = path.join(process.argv[2], 'meta');

apps/contract-migration/scripts/import_users.py

@@ -38,7 +38,7 @@ argparser.add_argument('--redis-host-callback', dest='redis_host_callback', defa
 argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
 argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
 argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches')
-argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
+argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout')
 argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
 argparser.add_argument('-v', action='store_true', help='Be verbose')
 argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@@ -97,6 +97,7 @@ def register_eth(i, u):
         callback_queue=args.q,
         )
     t = api.create_account(register=True)
+    logg.debug('register {} -> {}'.format(u, t))
 
     while True:
         ps.get_message()
@@ -112,11 +113,11 @@ def register_eth(i, u):
             r = json.loads(m['data'])
             address = r['result']
             break
-        except TypeError as e:
+        except Exception as e:
             if m == None:
-                logg.critical('empty response from redis callback (did the service crash?)')
+                logg.critical('empty response from redis callback (did the service crash?) {}'.format(e))
             else:
-                logg.critical('unexpected response from redis callback: {}'.format(m))
+                logg.critical('unexpected response from redis callback: {} {}'.format(m, e))
             sys.exit(1)
         logg.debug('[{}] register eth {} {}'.format(i, u, address))
 

apps/contract-migration/scripts/requirements.txt

@@ -1,3 +1,3 @@
-cic-base[full_graph]==0.1.1a12
-cic-eth==0.10.0a38
+cic-base[full_graph]==0.1.1a23
+cic-eth==0.10.0a41
 cic-types==0.1.0a8

apps/contract-migration/seed_cic_eth.sh

@@ -26,14 +26,11 @@ env_out_file=${CIC_DATA_DIR}/.env_seed
 init_level_file=${CIC_DATA_DIR}/.init
 truncate $env_out_file -s 0
 
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 chainlib==0.0.1a22
 
 set -e
 set -a
 
-# We need to not install these here...
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a38 chainlib==0.0.1a19 cic-contracts==0.0.2a2
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10
-
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
 DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
@@ -129,6 +126,7 @@ export CIC_TOKEN_INDEX_ADDRESS=$CIC_TOKEN_INDEX_ADDRESS
 >&2 echo "add declarations for sarafu token"
 token_description_one=`sha256sum sarafu_declaration.json | awk '{ print $1; }'`
 token_description_two=0x54686973206973207468652053617261667520746f6b656e0000000000000000
+echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo $CIC_DECLARATOR_ADDRESSh"
 >&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_one
 >&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_two
 

ci_templates/.cic-template.yml

@@ -1,54 +1,35 @@
-image: docker:19.03.13
+image:
+  name: gcr.io/kaniko-project/executor:debug
+  entrypoint: [""]
 
 variables:
-    # docker host
-    DOCKER_HOST: tcp://docker:2376
-    # container, thanks to volume mount from config.toml
-    DOCKER_TLS_CERTDIR: "/certs"
-    # These are usually specified by the entrypoint, however the
-    # Kubernetes executor doesn't run entrypoints
-    # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125
-    DOCKER_TLS_VERIFY: 1
-    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-    # We are building these from the apps dir to easily share the requirements file there.
-    # It would be nicer to build from the app dir context. TODO figure out a nice way to do this in local DOCKER_TLS_VERIFY
-    CONTEXT: apps/
-
-services:
-  - docker:19.03.13-dind
-
-before_script:
-  - docker info
+  KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
+  CONTEXT: $CI_PROJECT_DIR/apps/
 
 .py_build_merge_request:
   stage: build
-  before_script:
-    - cd $CONTEXT
   variables:
-    CI_DEBUG_TRACE: "true"
-    IMAGE_TAG: $APP_NAME:$CI_COMMIT_SHORT_SHA
+    - CI_DEBUG_TRACE: "true"
   script:
-    - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
 
 .py_build_push:
-    stage: build
-    before_script:
-      - cd $CONTEXT
-      - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" $CI_REGISTRY --password-stdin
-    variables:
-        CI_DEBUG_TRACE: "true"
-        IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
-        LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
-    script:
-      - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
-      - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
-      - docker push $IMAGE_TAG
-      - docker tag $IMAGE_TAG $LATEST_TAG
-      - docker push $LATEST_TAG
-    rules:
+  stage: build
+  variables:
+      IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
+      LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
+  script:
+    - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
+  rules:
       - if: $CI_COMMIT_BRANCH == "master"
         when: always
 

docker-compose.yml

@@ -54,6 +54,7 @@ services:
     volumes:
       - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql 
       - ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh
+      - ./apps/cic-cache/db/psycopg2/db.sql:/docker-entrypoint-initdb.d/3-init-cic-meta.sql
       - postgres-db:/var/lib/postgresql/data
 
   redis:
@@ -162,7 +163,43 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        /usr/local/bin/cic-cache-tracker -vv
+        /usr/local/bin/cic-cache-trackerd -vv
+    volumes:
+      - contract-config:/tmp/cic/config/:ro
+
+  cic-cache-tasker:
+    build:
+      context: apps
+      dockerfile: cic-cache/docker/Dockerfile
+    environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
+      ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
+      DATABASE_USER: ${DATABASE_USER:-grassroots}
+      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
+      DATABASE_HOST: ${DATABASE_HOST:-postgres}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
+      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
+      DATABASE_DEBUG: 1
+      ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
+      CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
+      CELERY_BROKER_URL: redis://redis:6379
+      CELERY_RESULT_URL: redis://redis:6379
+    deploy: 
+      restart_policy:
+        condition: on-failure
+    depends_on:
+      - redis
+      - postgres
+      - eth
+    command:
+      - /bin/bash
+      - -c
+      - |
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
+        /usr/local/bin/cic-cache-taskerd -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -192,7 +229,7 @@ services:
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         "/usr/local/bin/uwsgi" \
-        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \
+        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \
         --http :8000 \
         --pyargv -vv
 
@@ -238,7 +275,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_tasker.sh -q cic-eth 
+        ./start_tasker.sh -q cic-eth  -vv
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker: