Reinstate skipped admin noncritical tests

.gitlab-ci.yml

@@ -1,44 +1,14 @@
-#include:
+include:
-#  - local: 'ci_templates/.cic-template.yml'
+  - local: 'ci_templates/.cic-template.yml'
-#  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  - local: 'apps/contract-migration/.gitlab-ci.yml'
-#  - local: 'apps/cic-eth/.gitlab-ci.yml'
+  - local: 'apps/cic-eth/.gitlab-ci.yml'
-#  - local: 'apps/cic-ussd/.gitlab-ci.yml'
+  - local: 'apps/cic-ussd/.gitlab-ci.yml'
-#  - local: 'apps/cic-notify/.gitlab-ci.yml'
+  - local: 'apps/cic-notify/.gitlab-ci.yml'
-#  - local: 'apps/cic-meta/.gitlab-ci.yml'
+  - local: 'apps/cic-meta/.gitlab-ci.yml'
-#  - local: 'apps/cic-cache/.gitlab-ci.yml'
+  - local: 'apps/cic-cache/.gitlab-ci.yml'
-#  - local: 'apps/data-seeding/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'
-
-image: tiangolo/docker-with-compose
-
-before_script:
-  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
 
 stages:
   - build
-  - deploy
+  - test
-
+  - release
-variables:
-  DOCKER_BUILDKIT: 1
-  COMPOSE_DOCKER_CLI_BUILD: 1 
-
-  #services:
-  #  - docker:stable-dind
-
-build-prod:
-  tags:
-    - integration
-  stage: build
-  script:
-    -  docker-compose build
-  only:
-    - bvander/integration-tests-on-docker
-
-deploy-prod:
-  tags:
-    - integration
-  stage: deploy
-  script:
-    - docker stack deploy -c docker-compose.yml --with-registry-auth my-stack
-  only:
-    - bvander/integration-tests-on-docker
-

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -34,8 +34,6 @@ elif args.v:
 config = confini.Config(args.c, args.env_prefix)
 config.process()
 config.add(args.q, '_CELERY_QUEUE', True)
-config.censor('API_KEY', 'AFRICASTALKING')
-config.censor('API_USERNAME', 'AFRICASTALKING')
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 

apps/cic-notify/docker/Dockerfile

@@ -13,10 +13,13 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     -r requirements.txt
 
 COPY . . 
+
 RUN python setup.py install
 
+# TODO please review..can this go into requirements?
+RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
+
 COPY docker/*.sh .
-RUN chmod +x *.sh 
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
@@ -24,5 +27,4 @@ RUN chmod +x *.sh
 COPY .config/ /usr/local/etc/cic-notify/
 COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
 
-
 ENTRYPOINT []

apps/cic-notify/docker/Dockerfile_ci

@@ -15,8 +15,10 @@ COPY . .
 
 RUN python setup.py install
 
+# TODO please review..can this go into requirements?
+RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
+
 COPY docker/*.sh .
-RUN chmod +x *.sh 
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables

apps/cic-notify/docker/db.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-set -e
+
-python scripts/migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv
+migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv

apps/cic-notify/docker/start_tasker.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-. /root/db.sh
+. ./db.sh
 
 /usr/local/bin/cic-notify-tasker -vv $@

apps/cic-notify/scripts/migrate.py

@@ -35,10 +35,9 @@ elif args.v:
 
 config = confini.Config(args.c, args.env_prefix)
 config.process()
-config.censor('API_KEY', 'AFRICASTALKING')
-config.censor('API_USERNAME', 'AFRICASTALKING')
 config.censor('PASSWORD', 'DATABASE')
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
+#config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
 
 migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
 if not os.path.isdir(migrations_dir):

apps/cic-notify/setup.cfg

@@ -29,11 +29,18 @@ packages =
 	cic_notify.db
 	cic_notify.db.models
 	cic_notify.ext
-	cic_notify.tasks
 	cic_notify.tasks.sms
 	cic_notify.runnable
 scripts =
-	./scripts/migrate.py
+	scripts/migrate.py
+[options.extras_require]
+africastalking = africastalking==1.2.3
+notifylog = psycopg2==2.8.6
+testing =
+	pytest==6.0.1
+	pytest-celery==0.0.0a1
+	pytest-mock==3.3.1
+	pysqlite3==0.4.3
 
 [options.entry_points]
 console_scripts =

apps/cic-ussd/.config/app.ini

@@ -1,6 +1,6 @@
 [app]
 ALLOWED_IP=0.0.0.0/0
-LOCALE_FALLBACK=sw
+LOCALE_FALLBACK=en
 LOCALE_PATH=var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=

apps/cic-ussd/cic_ussd/balance.py

@@ -15,46 +15,44 @@ from cic_ussd.conversions import from_wei
 logg = logging.getLogger()
 
 
-def get_balances(
+class BalanceManager:
-        address: str,
+
-        chain_str: str,
+    def __init__(self, address: str, chain_str: str, token_symbol: str):
-        token_symbol: str,
+        """
-        asynchronous: bool = False,
+        :param address: Ethereum address of account whose balance is being queried
-        callback_param: any = None,
+        :type address: str, 0x-hex
-        callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Union[celery.Task, dict]:
+        :param chain_str: The chain name and network id.
+        :type chain_str: str
+        :param token_symbol: ERC20 token symbol of whose balance is being queried
+        :type token_symbol: str
+        """
+        self.address = address
+        self.chain_str = chain_str
+        self.token_symbol = token_symbol
+
+    def get_balances(self, asynchronous: bool = False) -> Union[celery.Task, dict]:
         """
         This function queries cic-eth for an account's balances, It provides a means to receive the balance either
         asynchronously or synchronously depending on the provided value for teh asynchronous parameter. It returns a
         dictionary containing network, outgoing and incoming balances.
-    :param address: Ethereum address of the recipient
+        :param asynchronous: Boolean value checking whether to return balances asynchronously
-    :type address: str, 0x-hex
-    :param chain_str: The chain name and network id.
-    :type chain_str: str
-    :param callback_param:
-    :type callback_param:
-    :param callback_task:
-    :type callback_task:
-    :param token_symbol: ERC20 token symbol of the account whose balance is being queried.
-    :type token_symbol: str
-    :param asynchronous: Boolean value checking whether to return balances asynchronously.
         :type asynchronous: bool
         :return:
         :rtype:
         """
-    logg.debug(f'Retrieving balance for address: {address}')
         if asynchronous:
             cic_eth_api = Api(
-            chain_str=chain_str,
+                chain_str=self.chain_str,
                 callback_queue='cic-ussd',
-            callback_task=callback_task,
+                callback_task='cic_ussd.tasks.callback_handler.process_balances_callback',
-            callback_param=callback_param
+                callback_param=''
             )
-        cic_eth_api.balance(address=address, token_symbol=token_symbol)
+            cic_eth_api.balance(address=self.address, token_symbol=self.token_symbol)
         else:
-        cic_eth_api = Api(chain_str=chain_str)
+            cic_eth_api = Api(chain_str=self.chain_str)
             balance_request_task = cic_eth_api.balance(
-            address=address,
+                address=self.address,
-            token_symbol=token_symbol)
+                token_symbol=self.token_symbol)
             return balance_request_task.get()[0]
 
 

apps/cic-ussd/cic_ussd/error.py

@@ -48,6 +48,3 @@ class InitializationError(Exception):
     pass
 
 
-class UnknownUssdRecipient(Exception):
-    """Raised when a recipient of a transaction is not known to the ussd application."""
-

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -127,19 +127,14 @@ class MetadataRequestsHandler(Metadata):
         if not isinstance(result_data, dict):
             raise ValueError(f'Invalid result data object: {result_data}.')
 
-        if result.status_code == 200:
+        if result.status_code == 200 and self.cic_type == ':cic.person':
-            if self.cic_type == ':cic.person':
             # validate person metadata
             person = Person()
             person_data = person.deserialize(person_data=result_data)
 
             # format new person data for caching
-                serialized_person_data = person_data.serialize()
+            data = json.dumps(person_data.serialize())
-                data = json.dumps(serialized_person_data)
-            else:
-                data = json.dumps(result_data)
 
             # cache metadata
             cache_data(key=self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
-        return result_data

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -1,7 +1,6 @@
 # standard imports
 
 # external imports
-import celery
 
 # local imports
 from .base import MetadataRequestsHandler

apps/cic-ussd/cic_ussd/processor.py

@@ -12,7 +12,7 @@ from tinydb.table import Document
 
 # local imports
 from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement
-from cic_ussd.balance import compute_operational_balance, get_balances, get_cached_operational_balance
+from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
@@ -182,6 +182,7 @@ def process_transaction_pin_authorization(account: Account, display_key: str, se
     token_symbol = retrieve_token_symbol()
     user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
+    logg.debug('Requires integration to determine user tokens.')
     return process_pin_authorization(
         account=account,
         display_key=display_key,
@@ -379,9 +380,12 @@ def process_start_menu(display_key: str, user: Account):
     token_symbol = retrieve_token_symbol()
     chain_str = Chain.spec.__str__()
     blockchain_address = user.blockchain_address
+    balance_manager = BalanceManager(address=blockchain_address,
+                                     chain_str=chain_str,
+                                     token_symbol=token_symbol)
 
     # get balances synchronously for display on start menu
-    balances_data = get_balances(address=blockchain_address, chain_str=chain_str, token_symbol=token_symbol)
+    balances_data = balance_manager.get_balances()
 
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py

@@ -8,16 +8,13 @@ import tempfile
 import celery
 import i18n
 import redis
-from chainlib.chain import ChainSpec
 from confini import Config
 
 # local imports
-from cic_ussd.chain import Chain
 from cic_ussd.db import dsn_from_config
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.metadata.signer import Signer
 from cic_ussd.metadata.base import Metadata
-from cic_ussd.phone_number import Support
 from cic_ussd.redis import InMemoryStore
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.validator import validate_presence
@@ -85,15 +82,6 @@ Signer.key_file_path = key_file_path
 i18n.load_path.append(config.get('APP_LOCALE_PATH'))
 i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
 
-chain_spec = ChainSpec(
-    common_name=config.get('CIC_COMMON_NAME'),
-    engine=config.get('CIC_ENGINE'),
-    network_id=config.get('CIC_NETWORK_ID')
-)
-
-Chain.spec = chain_spec
-Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER')
-
 # set up celery
 current_app = celery.Celery(__name__)
 

apps/cic-ussd/cic_ussd/tasks/__init__.py

@@ -1,4 +1,8 @@
 # standard import
+import os
+import logging
+import urllib
+import json
 
 # third-party imports
 # this must be included for the package to be recognized as a tasks package
@@ -10,5 +14,3 @@ from .logger import *
 from .ussd_session import *
 from .callback_handler import *
 from .metadata import *
-from .notifications import *
-from .processor import *

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -7,14 +7,14 @@ from datetime import datetime, timedelta
 import celery
 
 # local imports
-from cic_ussd.balance import compute_operational_balance, get_balances
-from cic_ussd.chain import Chain
 from cic_ussd.conversions import from_wei
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.account import Account
+from cic_ussd.account import define_account_tx_metadata
 from cic_ussd.error import ActionDataNotFoundError
-from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key, get_cached_data
+from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key
 from cic_ussd.tasks.base import CriticalSQLAlchemyTask
+from cic_ussd.transactions import IncomingTransactionProcessor
 
 logg = logging.getLogger(__file__)
 celery_app = celery.current_app
@@ -87,106 +87,59 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
     session.close()
 
 
-@celery_app.task(bind=True)
+@celery_app.task
-def process_transaction_callback(self, result: dict, param: str, status_code: int):
+def process_incoming_transfer_callback(result: dict, param: str, status_code: int):
+    session = SessionBase.create_session()
     if status_code == 0:
-        chain_str = Chain.spec.__str__()
 
-        # collect transaction metadata
+        # collect result data
-        destination_token_symbol = result.get('destination_token_symbol')
-        destination_token_value = result.get('destination_token_value')
         recipient_blockchain_address = result.get('recipient')
         sender_blockchain_address = result.get('sender')
-        source_token_symbol = result.get('source_token_symbol')
+        token_symbol = result.get('destination_token_symbol')
-        source_token_value = result.get('source_token_value')
+        value = result.get('destination_token_value')
 
-        # build stakeholder callback params
+        # try to find users in system
-        recipient_metadata = {
+        recipient_user = session.query(Account).filter_by(blockchain_address=recipient_blockchain_address).first()
-            "token_symbol": destination_token_symbol,
+        sender_user = session.query(Account).filter_by(blockchain_address=sender_blockchain_address).first()
-            "token_value": destination_token_value,
-            "blockchain_address": recipient_blockchain_address,
-            "tag": "recipient",
-            "tx_param": param
-        }
 
-        # retrieve account balances
+        # check whether recipient is in the system
-        get_balances(
+        if not recipient_user:
-            address=recipient_blockchain_address,
+            session.close()
-            callback_param=recipient_metadata,
+            raise ValueError(
-            chain_str=chain_str,
+                f'Tx for recipient: {recipient_blockchain_address} was received but has no matching user in the system.'
-            callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback',
+            )
-            token_symbol=destination_token_symbol,
-            asynchronous=True)
 
-        # only retrieve sender if transaction is a transfer
+        # process incoming transactions
-        if param == 'transfer':
+        incoming_tx_processor = IncomingTransactionProcessor(phone_number=recipient_user.phone_number,
-            sender_metadata = {
+                                                             preferred_language=recipient_user.preferred_language,
-                "blockchain_address": sender_blockchain_address,
+                                                             token_symbol=token_symbol,
-                "token_symbol": source_token_symbol,
+                                                             value=value)
-                "token_value": source_token_value,
-                "tag": "sender",
-                "tx_param": param
-            }
 
-            get_balances(
+        if param == 'tokengift':
-                address=sender_blockchain_address,
+            incoming_tx_processor.process_token_gift_incoming_transactions()
-                callback_param=sender_metadata,
+        elif param == 'transfer':
-                chain_str=chain_str,
+            if sender_user:
-                callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback',
+                sender_information = define_account_tx_metadata(user=sender_user)
-                token_symbol=source_token_symbol,
+                incoming_tx_processor.process_transfer_incoming_transaction(
-                asynchronous=True)
+                    sender_information=sender_information,
-    else:
+                    recipient_blockchain_address=recipient_blockchain_address
-        raise ValueError(f'Unexpected status code: {status_code}.')
+                )
-
-
-@celery_app.task(bind=True)
-def process_transaction_balances_callback(self, result: list, param: dict, status_code: int):
-    queue = self.request.delivery_info.get('routing_key')
-    if status_code == 0:
-        # retrieve balance data
-        balances_data = result[0]
-        operational_balance = compute_operational_balance(balances=balances_data)
-
-        # retrieve account's address
-        blockchain_address = param.get('blockchain_address')
-
-        # append balance to transaction metadata
-        transaction_metadata = param
-        transaction_metadata['operational_balance'] = operational_balance
-
-        # retrieve account's preferences
-        s_preferences_metadata = celery.signature(
-            'cic_ussd.tasks.metadata.query_preferences_metadata',
-            [blockchain_address],
-            queue=queue
-        )
-
-        # parse metadata and run validations
-        s_process_account_metadata = celery.signature(
-            'cic_ussd.tasks.processor.process_tx_metadata_for_notification',
-            [transaction_metadata],
-            queue=queue
-        )
-
-        # issue notification of transaction
-        s_notify_account = celery.signature(
-            'cic_ussd.tasks.notifications.notify_account_of_transaction',
-            queue=queue
-        )
-
-        if param.get('tx_param') == 'transfer':
-            celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async()
-
-        if param.get('tx_param') == 'tokengift':
-            s_process_account_metadata = celery.signature(
-                'cic_ussd.tasks.processor.process_tx_metadata_for_notification',
-                [{}, transaction_metadata],
-                queue=queue
-            )
-            celery.chain(s_process_account_metadata, s_notify_account).apply_async()
             else:
+                logg.warning(
+                    f'Tx with sender: {sender_blockchain_address} was received but has no matching user in the system.'
+                )
+                incoming_tx_processor.process_transfer_incoming_transaction(
+                    sender_information='GRASSROOTS ECONOMICS',
+                    recipient_blockchain_address=recipient_blockchain_address
+                )
+        else:
+            session.close()
+            raise ValueError(f'Unexpected transaction param: {param}.')
+    else:
+        session.close()
         raise ValueError(f'Unexpected status code: {status_code}.')
 
+    session.close()
 
 @celery_app.task
 def process_balances_callback(result: list, param: str, status_code: int):
@@ -198,7 +151,6 @@ def process_balances_callback(result: list, param: str, status_code: int):
             salt=':cic.balances_data'
         )
         cache_data(key=key, data=json.dumps(balances_data))
-        logg.debug(f'caching: {balances_data} with key: {key}')
     else:
         raise ValueError(f'Unexpected status code: {status_code}.')
 

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -26,7 +26,6 @@ def query_person_metadata(blockchain_address: str):
     :rtype:
     """
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
-    logg.debug(f'Retrieving person metadata for address: {blockchain_address}.')
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.query()
 
@@ -73,15 +72,3 @@ def add_preferences_metadata(blockchain_address: str, data: dict):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
     custom_metadata_client = PreferencesMetadata(identifier=identifier)
     custom_metadata_client.create(data=data)
-
-
-@celery_app.task()
-def query_preferences_metadata(blockchain_address: str):
-    """This method retrieves preferences metadata based on an account's blockchain address.
-    :param blockchain_address: Blockchain address of an account.
-    :type blockchain_address: str | Ox-hex
-    """
-    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
-    logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
-    person_metadata_client = PreferencesMetadata(identifier=identifier)
-    return person_metadata_client.query()

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -1,70 +0,0 @@
-# standard imports
-import datetime
-import logging
-
-# third-party imports
-import celery
-
-# local imports
-from cic_ussd.notifications import Notifier
-from cic_ussd.phone_number import Support
-
-celery_app = celery.current_app
-logg = logging.getLogger(__file__)
-notifier = Notifier()
-
-
-@celery_app.task
-def notify_account_of_transaction(notification_data: dict):
-    """
-    :param notification_data:
-    :type notification_data:
-    :return:
-    :rtype:
-    """
-
-    account_tx_role = notification_data.get('account_tx_role')
-    amount = notification_data.get('amount')
-    balance = notification_data.get('balance')
-    phone_number = notification_data.get('phone_number')
-    preferred_language = notification_data.get('preferred_language')
-    token_symbol = notification_data.get('token_symbol')
-    transaction_account_metadata = notification_data.get('transaction_account_metadata')
-    transaction_type = notification_data.get('transaction_type')
-
-    timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
-
-    if transaction_type == 'tokengift':
-        support_phone = Support.phone_number
-        notifier.send_sms_notification(
-            key='sms.account_successfully_created',
-            phone_number=phone_number,
-            preferred_language=preferred_language,
-            balance=balance,
-            support_phone=support_phone,
-            token_symbol=token_symbol
-        )
-
-    if transaction_type == 'transfer':
-        if account_tx_role == 'recipient':
-            notifier.send_sms_notification(
-                key='sms.received_tokens',
-                phone_number=phone_number,
-                preferred_language=preferred_language,
-                amount=amount,
-                token_symbol=token_symbol,
-                tx_sender_information=transaction_account_metadata,
-                timestamp=timestamp,
-                balance=balance
-            )
-        else:
-            notifier.send_sms_notification(
-                key='sms.sent_tokens',
-                phone_number=phone_number,
-                preferred_language=preferred_language,
-                amount=amount,
-                token_symbol=token_symbol,
-                tx_recipient_information=transaction_account_metadata,
-                timestamp=timestamp,
-                balance=balance
-            )

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -1,88 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-from i18n import config
-
-# local imports
-from cic_ussd.account import define_account_tx_metadata
-from cic_ussd.db.models.account import Account
-from cic_ussd.db.models.base import SessionBase
-from cic_ussd.error import UnknownUssdRecipient
-from cic_ussd.transactions import from_wei
-
-
-celery_app = celery.current_app
-logg = logging.getLogger(__file__)
-
-
-@celery_app.task
-def process_tx_metadata_for_notification(result: celery.Task, transaction_metadata: dict):
-    """
-    :param result:
-    :type result:
-    :param transaction_metadata:
-    :type transaction_metadata:
-    :return:
-    :rtype:
-    """
-    notification_data = {}
-
-    # get preferred language
-    preferred_language = result.get('preferred_language')
-    if not preferred_language:
-        preferred_language = config.get('fallback')
-    notification_data['preferred_language'] = preferred_language
-
-    # validate account information against present ussd storage data.
-    session = SessionBase.create_session()
-    blockchain_address = transaction_metadata.get('blockchain_address')
-    tag = transaction_metadata.get('tag')
-    account = session.query(Account).filter_by(blockchain_address=blockchain_address).first()
-    if not account and tag == 'recipient':
-        session.close()
-        raise UnknownUssdRecipient(
-            f'Tx for recipient: {blockchain_address} was received but has no matching user in the system.'
-        )
-
-    # get phone number associated with account
-    phone_number = account.phone_number
-    notification_data['phone_number'] = phone_number
-
-    # get account's role in transaction i.e sender / recipient
-    tx_param = transaction_metadata.get('tx_param')
-    notification_data['transaction_type'] = tx_param
-
-    # get token amount and symbol
-    if tag == 'recipient':
-        account_tx_role = tag
-        amount = transaction_metadata.get('token_value')
-        amount = from_wei(value=amount)
-        token_symbol = transaction_metadata.get('token_symbol')
-    else:
-        account_tx_role = tag
-        amount = transaction_metadata.get('token_value')
-        amount = from_wei(value=amount)
-        token_symbol = transaction_metadata.get('token_symbol')
-    notification_data['account_tx_role'] = account_tx_role
-    notification_data['amount'] = amount
-    notification_data['token_symbol'] = token_symbol
-
-    # get account's standard ussd identification pattern
-    if tx_param == 'transfer':
-        tx_account_metadata = define_account_tx_metadata(user=account)
-        notification_data['transaction_account_metadata'] = tx_account_metadata
-
-        if tag == 'recipient':
-            notification_data['notification_key'] = 'sms.received_tokens'
-        else:
-            notification_data['notification_key'] = 'sms.sent_tokens'
-
-    if tx_param == 'tokengift':
-        notification_data['notification_key'] = 'sms.account_successfully_created'
-
-    # get account's balance
-    notification_data['balance'] = transaction_metadata.get('operational_balance')
-
-    return notification_data

apps/cic-ussd/cic_ussd/transactions.py

@@ -7,9 +7,9 @@ from datetime import datetime
 from cic_eth.api import Api
 
 # local imports
-from cic_ussd.balance import get_balances, get_cached_operational_balance
+from cic_ussd.balance import get_cached_operational_balance
 from cic_ussd.notifications import Notifier
-from cic_ussd.phone_number import Support
+
 
 logg = logging.getLogger()
 notifier = Notifier()
@@ -50,6 +50,61 @@ def to_wei(value: int) -> int:
     return int(value * 1e+6)
 
 
+class IncomingTransactionProcessor:
+
+    def __init__(self, phone_number: str, preferred_language: str, token_symbol: str, value: int):
+        """
+        :param phone_number: The recipient's phone number.
+        :type phone_number: str
+        :param preferred_language: The user's preferred language.
+        :type preferred_language: str
+        :param token_symbol: The symbol for the token the recipient receives.
+        :type token_symbol: str
+        :param value: The amount of tokens received in the transactions.
+        :type value: int
+        """
+        self.phone_number = phone_number
+        self.preferred_language = preferred_language
+        self.token_symbol = token_symbol
+        self.value = value
+
+    def process_token_gift_incoming_transactions(self):
+        """This function processes incoming transactions with a "tokengift" param, it collects all appropriate data to
+        send out notifications to users when their accounts are successfully created.
+
+        """
+        balance = from_wei(value=self.value)
+        key = 'sms.account_successfully_created'
+        notifier.send_sms_notification(key=key,
+                                       phone_number=self.phone_number,
+                                       preferred_language=self.preferred_language,
+                                       balance=balance,
+                                       token_symbol=self.token_symbol)
+
+    def process_transfer_incoming_transaction(self, sender_information: str, recipient_blockchain_address: str):
+        """This function processes incoming transactions with the "transfer" param and issues notifications to users
+        about reception of funds into their accounts.
+        :param sender_information: A string with a user's full name and phone number.
+        :type sender_information: str
+        :param recipient_blockchain_address:
+        type recipient_blockchain_address: str
+        """
+        key = 'sms.received_tokens'
+        amount = from_wei(value=self.value)
+        timestamp = datetime.now().strftime('%d-%m-%y, %H:%M %p')
+
+        operational_balance = get_cached_operational_balance(blockchain_address=recipient_blockchain_address)
+
+        notifier.send_sms_notification(key=key,
+                                       phone_number=self.phone_number,
+                                       preferred_language=self.preferred_language,
+                                       amount=amount,
+                                       token_symbol=self.token_symbol,
+                                       tx_sender_information=sender_information,
+                                       timestamp=timestamp,
+                                       balance=operational_balance)
+
+
 class OutgoingTransactionProcessor:
 
     def __init__(self, chain_str: str, from_address: str, to_address: str):
@@ -61,7 +116,6 @@ class OutgoingTransactionProcessor:
         :param to_address: Ethereum address of the recipient
         :type to_address: str, 0x-hex
         """
-        self.chain_str = chain_str
         self.cic_eth_api = Api(chain_str=chain_str)
         self.from_address = from_address
         self.to_address = to_address

apps/cic-ussd/requirements.txt

@@ -10,7 +10,7 @@ tinydb==4.2.0
 phonenumbers==8.12.12
 redis==3.5.3
 celery==4.4.7
-python-i18n[YAML]==0.3.9
+python-i18n==0.3.9
 pyxdg==0.27
 bcrypt==3.2.0
 uWSGI==2.0.19.1

apps/cic-ussd/var/lib/locale/sms.en.yml

@@ -3,7 +3,5 @@ en:
     You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
   received_tokens: |-
     Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
-  sent_tokens: |-
-    Successfully sent %{amount} %{token_symbol} to %{tx_recipient_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
   terms: |-
     By using the service, you agree to the terms and conditions at http://grassecon.org/tos

apps/cic-ussd/var/lib/locale/sms.sw.yml

@@ -2,8 +2,6 @@ sw:
   account_successfully_created: |-
     Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
   received_tokens: |-
-    Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
+    Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}.
-  sent_tokens: |-
-    Umetuma %{amount} %{token_symbol} kwa %{tx_recipient_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
   terms: |-
     Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos

apps/contract-migration/.dockerignore

@@ -2,5 +2,3 @@
 .cache
 .dot
 **/doc
-**/.venv
-**/venv

apps/contract-migration/reset.sh

@@ -158,7 +158,6 @@ export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
 EOF
 
 cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
-cat ./envlist
 # popd
 
 set +a

apps/data-seeding/docker/.env

@@ -1,14 +0,0 @@
-ETH_PROVIDER=http://eth:8545
-CELERY_BROKER_URL=redis://redis:6379
-CELERY_RESULT_URL=redis://redis:6379
-CIC_REGISTRY_ADDRESS=0xea6225212005e86a4490018ded4bf37f3e772161
-TOKEN_SYMBOL=GFT
-REDIS_HOST=redis
-REDIS_PORT=6379
-USER_USSD_HOST=cic-user-ussd-server
-USER_USSD_PORT=9000
-KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
-CIC_DATA_DIR=config/
-OUT_DIR=out
-NUMBER_OF_USERS=10
-META_URL=http://cic-meta-server:63380

apps/data-seeding/docker/Dockerfile

@@ -9,7 +9,7 @@ COPY package.json \
      package-lock.json \
      .
 
-RUN npm ci --production 
+RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
 
 COPY requirements.txt . 
 
@@ -20,5 +20,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
     --extra-index-url $EXTRA_INDEX_URL -r requirements.txt
 
 COPY . . 
-RUN chmod +x scripts/*
+
 ENTRYPOINT [ ]

apps/data-seeding/scripts/run_job.sh

@@ -1,36 +0,0 @@
-#! /bin/bash
-
-if [[ $((RUN_MASK & 3)) -eq 3 ]]
-then
-	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 3 - data seeding"
-
-    contract_migration_complete=0
-    retry_count=0
-    retry_sleep=30 #seconds
-    retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))"
-    while [[ $contract_migration_complete -ne 1 ]]
-    do
-      if [[ -f "$CIC_DATA_DIR/.env"  ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env
-      then
-        echo "🤜💥🤛 data-seeding found the output of contract-migration!"
-        source /tmp/cic/config/.env
-        env
-        contract_migration_complete=1
-        ./scripts/run_ussd_user_imports.sh
-      elif [[ $retry_count -ge $retry_limit ]] 
-      then
-        echo "😢 data-seeding timeout waiting for contract migration to finish." >&2
-        exit 1
-      else
-        echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..."
-        ((retry_count= $retry_count + $retry_sleep))
-        sleep $retry_sleep 
-      fi
-    done
-  
-	if [ $? -ne "0" ]; then
-		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 3 - data seeding"
-		exit 1;
-	fi
-	>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 3 - data seeding"
-fi

apps/data-seeding/scripts/run_tasker.sh

@@ -1,3 +0,0 @@
-#! /bin/bash 
-
-python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR

apps/data-seeding/scripts/run_ussd_user_imports.sh

@@ -1,65 +0,0 @@
-#! /bin/bash
-
-set -u
-set -e
-
-
-while getopts ":n:o:g:" opt; do
-  case $opt in
-    n) NUMBER_OF_USERS="$OPTARG"
-    ;;
-    o) OUT_DIR="$OPTARG"
-    ;;
-    \?) echo "Invalid option -$OPTARG" >&2
-    ;;
-  esac
-done
-
-# using timeout because the timeout flag for celery inspect does not work
-timeout 5 celery inspect ping -b $CELERY_BROKER_URL
-if [[ $? -eq 124 ]] 
-then
-  >&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?"
-  exit 1
-fi
-
-if [[ -d $OUT_DIR ]]
-then
-  echo "found existing OUT_DIR cleaning up..."
-  rm -rf $OUT_DIR
-  mkdir -p $OUT_DIR
-else
-  echo "OUT_DIR does not exist creating it."
-  mkdir -p $OUT_DIR
-fi
-
-
-echo "creating accounts"
-
-python create_import_users.py --dir $OUT_DIR $NUMBER_OF_USERS
-
-echo "purging existing ussd tasks..."
-
-celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f
-
-echo "running import_balance in the background..."
-
-#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
-#  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR > nohup.out 2> nohup.err < /dev/null & 
-
-#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
-#  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 
-
-echo "import_balance pid: $!" 
-
-echo "importing accounts"
-
-python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out
-
-sleep 3600
-
-echo "data importing finished!"
-
-echo "run verify manually:"
-echo "python verify.py -v -c config -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER --token-symbol $TOKEN_SYMBOL $OUT_DIR"
-

apps/data-seeding/verify.py

@@ -154,26 +154,22 @@ for t in custodial_tests:
         logg.info('activating custodial module'.format(t))
         break
 
-#cols = os.get_terminal_size().columns
+cols = os.get_terminal_size().columns
 
-##disabled because:
+
-#data-seeding_1          | Traceback (most recent call last):
+def to_terminalwidth(s):
-#data-seeding_1          |   File "verify.py", line 157, in <module>
+    ss = s.ljust(int(cols)-1)
-#data-seeding_1          |     cols = os.get_terminal_size().columns
+    ss += "\r"
-#data-seeding_1          | OSError: [Errno 25] Inappropriate ioctl for device
+    return ss
-#contract-migration_1    | DEBUG:create_account_script:register False
+
-#def to_terminalwidth(s):
+def default_outfunc(s):
-#    ss = s.ljust(int(cols)-1)
+    ss = to_terminalwidth(s)
-#    ss += "\r"
+    sys.stdout.write(ss)
-#    return ss
+outfunc = default_outfunc
-#
+if logg.isEnabledFor(logging.DEBUG):
-#def default_outfunc(s):
-#    ss = to_terminalwidth(s)
-#    sys.stdout.write(ss)
-#outfunc = default_outfunc
-#if logg.isEnabledFor(logging.DEBUG):
     outfunc = logg.debug
 
+
 def send_ussd_request(address, data_dir):
     upper_address = strip_0x(address).upper()
     f = open(os.path.join(

docker-compose.test.yml

@@ -1,120 +0,0 @@
-version: "3"
-
-services:
-  contract-migration:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration
-    build:
-      context: apps/contract-migration
-      dockerfile: docker/Dockerfile
-      args:
-        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
-        pip_extra_args: $PIP_EXTRA_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
-      ETH_PROVIDER: http://eth:8545
-      # And these two are for wait-for-it (could parse this)
-      DEV_USE_DOCKER_WAIT_SCRIPT: 1
-      ETH_PROVIDER_HOST: eth 
-      ETH_PROVIDER_PORT: 8545
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
-      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
-      DATABASE_HOST: ${DATABASE_HOST:-postgres}
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
-      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
-      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_USER: ${DATABASE_USER:-postgres}
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_DB: ${REDIS_DB:-0}
-      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
-      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
-      RUN_MASK: ${RUN_MASK:-0} 
-      DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
-      #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
-      DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
-      CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
-      TOKEN_NAME: $TOKEN_NAME
-      TOKEN_DECIMALS: $TOKEN_DECIMALS
-      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
-      TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
-      TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
-      TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
-      TOKEN_TYPE: $TOKEN_TYPE
-        #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
-    command: ["./run_job.sh"]
-        #command: ["./reset.sh"]
-    volumes:
-      - contract-config:/tmp/cic/config
-
-
-  data-seeding-tasker:
-    build:
-      context: apps/data-seeding
-      dockerfile: docker/Dockerfile
-      args:
-        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
-        pip_extra_args: $PIP_EXTRA_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
-      ETH_PROVIDER: http://eth:8545
-      # And these two are for wait-for-it (could parse this)
-      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
-      DEV_USE_DOCKER_WAIT_SCRIPT: 1
-      ETH_PROVIDER_HOST: eth 
-      ETH_PROVIDER_PORT: 8545
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_DB: ${REDIS_DB:-0}
-      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
-      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
-      RUN_MASK: ${RUN_MASK:-0} # 7
-      TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
-      USER_USSD_HOST: cic-user-ussd-server
-      USER_USSD_PORT: 9000
-      KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
-      OUT_DIR: out
-      NUMBER_OF_USERS: 10
-      #command: "python cic_ussd/import_balance.py -v -c config -p $$ETH_PROVIDER -r $$CIC_REGISTRY_ADDRESS --token-symbol $$TOKEN_SYMBOL -y $$KEYSTORE_FILE_PATH $$OUT_DIR"
-    command: "bash scripts/run_tasker.sh"
-    volumes:
-      - contract-config:/tmp/cic/config
-
-  data-seeding:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding
-    build:
-      context: apps/data-seeding
-      dockerfile: docker/Dockerfile
-      args:
-        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
-        pip_extra_args: $PIP_EXTRA_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
-      ETH_PROVIDER: http://eth:8545
-      # And these two are for wait-for-it (could parse this)
-      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
-      DEV_USE_DOCKER_WAIT_SCRIPT: 1
-      ETH_PROVIDER_HOST: eth 
-      ETH_PROVIDER_PORT: 8545
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_DB: ${REDIS_DB:-0}
-      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
-      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
-      RUN_MASK: ${RUN_MASK:-0} # 7
-      TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
-      USER_USSD_HOST: cic-user-ussd-server
-      USER_USSD_PORT: 9000
-      KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
-      OUT_DIR: out
-      NUMBER_OF_USERS: 10
-    command: ["bash", "./scripts/run_job.sh"]
-    volumes:
-      - contract-config:/tmp/cic/config
-

docker-compose.yml

@@ -8,10 +8,25 @@ volumes:
   bloxberg-data: {}
   contract-config: {} 
 
+networks:
+  default:
+    name: cic-network
 
 services:
+  # eth:
+  #   image: trufflesuite/ganache-cli
+  #   ports:
+  #     - ${HTTP_PORT_ETH:-8545}
+  #     - ${WS_PORT_ETH:-8546}
+  #   # Note! -e switch doesnt work, whatever you  put there, it will be 100
+  #   command: "-i 8996 -e 1000 -l 90000000 \
+  #           -m '${DEV_MNEMONIC:-\"history stumble mystery avoid embark arrive mom foil pledge keep grain dice\"}' \
+  #           -v --db /tmp/cic/ganache/ganache.db \
+  #           --noVMErrorsOnRPCResponse --allowUnlimitedContractSize"
+  #   volumes:
+  #     - ganache-db:/tmp/cic/ganache
+
   eth:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/bloxberg-node
     build:
       context: apps/bloxbergValidatorSetup
     restart: unless-stopped
@@ -55,8 +70,59 @@ services:
     volumes:
       - bee-data:/tmp/cic/bee
 
+  contract-migration:
+    build:
+      context: apps/contract-migration
+      dockerfile: docker/Dockerfile
+      args:
+        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
+        pip_extra_args: $PIP_EXTRA_ARGS
+    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
+    environment:
+      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
+      ETH_PROVIDER: http://eth:8545
+      # And these two are for wait-for-it (could parse this)
+      DEV_USE_DOCKER_WAIT_SCRIPT: 1
+      ETH_PROVIDER_HOST: eth 
+      ETH_PROVIDER_PORT: 8545
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
+      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
+      DATABASE_HOST: ${DATABASE_HOST:-postgres}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+      DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
+      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
+      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
+      DATABASE_USER: ${DATABASE_USER:-postgres}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_DB: ${REDIS_DB:-0}
+      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
+      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
+      DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
+      RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
+      DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
+      #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
+      DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
+      CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
+      TOKEN_NAME: $TOKEN_NAME
+      TOKEN_DECIMALS: $TOKEN_DECIMALS
+      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
+      TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
+      TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
+      TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
+      TOKEN_TYPE: $TOKEN_TYPE
+        #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
+    command: ["./run_job.sh"]
+        #command: ["./reset.sh"]
+    depends_on:
+      - eth
+      - postgres
+      - redis
+      - cic-eth-tasker
+    volumes:
+      - contract-config:/tmp/cic/config
+
   cic-cache-tracker:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -93,7 +159,6 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-tasker:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -131,7 +196,6 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-server:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -163,10 +227,11 @@ services:
 
 
   cic-eth-tasker:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
+    # image: grassrootseconomics:cic-eth-service
     build: 
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
+        target: dev
     environment:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
       ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER
@@ -213,10 +278,10 @@ services:
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
+        target: dev
     environment:
       ETH_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
@@ -252,10 +317,10 @@ services:
 
 
   cic-eth-dispatcher:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
+        target: dev
     environment:
       ETH_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
@@ -273,6 +338,7 @@ services:
       TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
       DATABASE_DEBUG: ${DATABASE_DEBUG:-false}
         #DATABASE_DEBUG: 1
+
     depends_on:
       - eth
       - postgres
@@ -292,10 +358,10 @@ services:
 
 
   cic-eth-retrier:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
+        target: dev
     environment:
       ETH_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
@@ -335,7 +401,6 @@ services:
 
 
   cic-notify-tasker:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify
     build:
       context: apps/cic-notify
       dockerfile: docker/Dockerfile
@@ -364,7 +429,6 @@ services:
 
 
   cic-meta-server:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta
     hostname: meta
     build:
       context: apps/cic-meta
@@ -398,7 +462,6 @@ services:
     # command: "/root/start_server.sh -vv"
 
   cic-user-ussd-server:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd
     build:
       context: apps/cic-ussd
       dockerfile: docker/Dockerfile
@@ -428,7 +491,6 @@ services:
     command: "/root/start_cic_user_ussd_server.sh -vv"
 
   cic-user-server:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd
     build:
       context: apps/cic-ussd
       dockerfile: docker/Dockerfile
@@ -451,7 +513,6 @@ services:
     command: "/root/start_cic_user_server.sh -vv"
 
   cic-user-tasker:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd
     build:
       context: apps/cic-ussd/
       dockerfile: docker/Dockerfile