Restore session worker in gas task tests

.gitignore

@@ -14,4 +14,3 @@ build/
 **/.venv
 .idea
 **/.vim
-**/*secret.yaml

.gitlab-ci.yml

@@ -12,13 +12,13 @@ include:
 stages:
   - build
   - test
-  - deploy
+  - deploy 
 
-image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
+image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest  
 
 variables:
   DOCKER_BUILDKIT: "1"
-  COMPOSE_DOCKER_CLI_BUILD: "1"
+  COMPOSE_DOCKER_CLI_BUILD: "1" 
   CI_DEBUG_TRACE: "true"
 
 before_script:
@@ -29,15 +29,8 @@ build-push:
   stage: build
   tags:
     - integration
-  #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
   script:
-    - TAG=latest sh ./scripts/build-push.sh
+    - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
   rules:
   - if: $CI_COMMIT_REF_PROTECTED == "true"
     when: always
-
-deploy-dev:
-  stage: deploy
-  trigger: grassrootseconomics/devops
-  when: manual

README.md

@@ -26,7 +26,7 @@ docker-compose down
 
 stop cluster and delete data
 ```
-docker-compose down -v --remove-orphans
+docker-compose down -v
 ```
 
 rebuild an images
@@ -34,7 +34,4 @@ rebuild an images
 docker-compose up --build <service_name>
 ```
 
-to delete the buildkit cache
-```
-docker builder prune --filter type=exec.cachemount
-```
+

apps/cic-cache/cic_cache/cli/rpc.py

@@ -27,11 +27,11 @@ class RPC:
     @staticmethod
     def from_config(config):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
         if config.get('SIGNER_PROVIDER'):
             RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
             RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
-        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc
 

apps/cic-cache/requirements.txt

@@ -9,6 +9,7 @@ psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
 chainsyncer[sql]>=0.0.6a3,<0.1.0
-erc20-faucet>=0.3.2a2, <0.4.0
-chainlib-eth>=0.0.9a14,<0.1.0
-eth-address-index>=0.2.3a4,<0.3.0
+erc20-faucet>=0.3.2a1, <0.4.0
+chainlib-eth>=0.0.9a7,<0.1.0
+chainlib>=0.0.9a3,<0.1.0
+eth-address-index>=0.2.3a1,<0.3.0

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,4 +1,5 @@
 celery==4.4.7
 erc20-demurrage-token~=0.0.3a1
-cic-eth-registry>=0.6.1a2,<0.7.0
-cic-eth[services]~=0.12.4a8
+cic-eth-registry~=0.5.8a1
+chainlib~=0.0.7a1
+cic_eth~=0.12.2a4

apps/cic-eth/.gitlab-ci.yml

@@ -9,8 +9,8 @@ build-test-cic-eth:
     - cd apps/cic-eth
     - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
     - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
+      #rules:
+      #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      #  changes:
+      #      - apps/$APP_NAME/**/*
+      #  when: always

apps/cic-eth/cic_eth/cli/rpc.py

@@ -35,14 +35,14 @@ class RPC:
     def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label)
         if use_signer:
 
             RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
             RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
             RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
             RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
-        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc
 

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -69,6 +69,7 @@ from cic_eth.registry import (
         )
 from cic_eth.task import BaseTask
 
+
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
@@ -203,9 +204,6 @@ def main():
     argv.append(config.get('CELERY_QUEUE'))
     argv.append('-n')
     argv.append(config.get('CELERY_QUEUE'))
-    argv.append('--config')
-    argv.append('celeryconfig')
-
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)

apps/cic-eth/crypto_dev_signer_config/config.ini

@@ -0,0 +1,4 @@
+[signer]
+secret = deadbeef
+#database = crypto-dev-signer
+socket_path = ipc:///run/crypto-dev-signer/jsonrpc.ipc

apps/cic-eth/crypto_dev_signer_config/database.ini

@@ -0,0 +1,8 @@
+[database]
+NAME=cic-signer
+USER=
+PASSWORD=
+HOST=
+PORT=
+ENGINE=
+DRIVER=

apps/cic-eth/docker/Dockerfile

@@ -13,16 +13,7 @@ ARG EXTRA_PIP_ARGS=""
 #    pip install --index-url https://pypi.org/simple \
 #    --force-reinstall \
 #	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-#    -r requirements.txt
-
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
-
-
+#    -r requirements.txt 
 COPY *requirements.txt ./
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
@@ -31,21 +22,19 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
-    -r admin_requirements.txt 
-   
-# always install the latest signer 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    crypto-dev-signer
-
+    -r admin_requirements.txt
+    
 COPY . .
 RUN python setup.py install
 
 ENV PYTHONPATH .
 
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    $EXTRA_PIP_ARGS \
+    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
 
 COPY docker/entrypoints/* ./ 
 RUN chmod 755 *.sh

apps/cic-eth/docker/entrypoints/start_tasker.sh

@@ -5,27 +5,27 @@ set -e
 
 # set CONFINI_ENV_PREFIX to override the env prefix to override env vars
 
-#echo "!!! starting signer"
-#python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
+echo "!!! starting signer"
+python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
 
 echo "!!! starting taskerd"
 /usr/local/bin/cic-eth-taskerd $@
 
 # thanks! https://docs.docker.com/config/containers/multi-service_container/
-#sleep 1;
-#echo "!!! entering monitor loop"
-#while true; do
-#  ps aux | grep crypto-dev-daemon | grep -q -v grep
-#  PROCESS_1_STATUS=$?
-#  ps aux | grep cic-eth-tasker |grep -q -v grep
-#  PROCESS_2_STATUS=$?
-#  # If the greps above find anything, they exit with 0 status
-#  # If they are not both 0, then something is wrong
-#  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
-#    echo "One of the processes has already exited."
-#    exit 1
-#  fi
-#  sleep 15;
-#done
-#
+sleep 1;
+echo "!!! entering monitor loop"
+while true; do
+  ps aux | grep crypto-dev-daemon | grep -q -v grep
+  PROCESS_1_STATUS=$?
+  ps aux | grep cic-eth-tasker |grep -q -v grep
+  PROCESS_2_STATUS=$?
+  # If the greps above find anything, they exit with 0 status
+  # If they are not both 0, then something is wrong
+  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
+    echo "One of the processes has already exited."
+    exit 1
+  fi
+  sleep 15;
+done
+
 set +e

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.9a14,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
 semver==2.13.0

apps/cic-eth/tools_requirements.txt

@@ -1,4 +1,4 @@
-crypto-dev-signer>=0.4.15a7,<=0.4.15
+crypto-dev-signer>=0.4.15a1,<=0.4.15
 chainqueue>=0.0.5a1,<0.1.0
 cic-eth-registry>=0.6.1a2,<0.7.0
 redis==3.5.3

apps/cic-meta/.gitlab-ci.yml

@@ -9,8 +9,8 @@ build-test-cic-meta:
     - cd apps/cic-meta
     - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
     - docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
+     #rules:
+     #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
+     #  changes:
+     #      - apps/$APP_NAME/**/*
+     #  when: always

apps/cic-meta/README.md

@@ -1 +0,0 @@
-# CIC-Meta

apps/cic-ussd/cic_ussd/account/balance.py

@@ -1,12 +1,10 @@
 # standard imports
-
 import json
 import logging
 from typing import Optional
 
 # third-party imports
 from cic_eth.api import Api
-from cic_eth_aux.erc20_demurrage_token.api import Api as DemurrageApi
 
 # local imports
 from cic_ussd.account.transaction import from_wei
@@ -75,24 +73,6 @@ def calculate_available_balance(balances: dict) -> float:
     return from_wei(value=available_balance)
 
 
-def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_symbol: str):
-    """
-    :param balance:
-    :type balance:
-    :param chain_str:
-    :type chain_str:
-    :param timestamp:
-    :type timestamp:
-    :param token_symbol:
-    :type token_symbol:
-    :return:
-    :rtype:
-    """
-    logg.debug(f'retrieving adjusted balance on chain: {chain_str}')
-    demurrage_api = DemurrageApi(chain_str=chain_str)
-    return demurrage_api.get_adjusted_balance(token_symbol, balance, timestamp).result
-
-
 def get_cached_available_balance(blockchain_address: str) -> float:
     """This function attempts to retrieve balance data from the redis cache.
     :param blockchain_address: Ethereum address of an account.
@@ -108,14 +88,3 @@ def get_cached_available_balance(blockchain_address: str) -> float:
         return calculate_available_balance(json.loads(cached_balances))
     else:
         raise CachedDataNotFoundError(f'No cached available balance for address: {blockchain_address}')
-
-
-def get_cached_adjusted_balance(identifier: bytes):
-    """
-    :param identifier:
-    :type identifier:
-    :return:
-    :rtype:
-    """
-    key = cache_data_key(identifier, ':cic.adjusted_balance')
-    return get_cached_data(key)

apps/cic-ussd/cic_ussd/processor/menu.py

@@ -1,17 +1,13 @@
 # standard imports
 import json
 import logging
-from datetime import datetime, timedelta
 
 # external imports
 import i18n.config
+from sqlalchemy.orm.session import Session
 
 # local imports
-from cic_ussd.account.balance import (calculate_available_balance,
-                                      get_adjusted_balance,
-                                      get_balances,
-                                      get_cached_adjusted_balance,
-                                      get_cached_available_balance)
+from cic_ussd.account.balance import calculate_available_balance, get_balances, get_cached_available_balance
 from cic_ussd.account.chain import Chain
 from cic_ussd.account.metadata import get_cached_preferred_language
 from cic_ussd.account.statement import (
@@ -20,15 +16,14 @@ from cic_ussd.account.statement import (
     query_statement,
     statement_transaction_set
 )
-from cic_ussd.account.tokens import get_default_token_symbol
 from cic_ussd.account.transaction import from_wei, to_wei
+from cic_ussd.account.tokens import get_default_token_symbol
 from cic_ussd.cache import cache_data_key, cache_data
 from cic_ussd.db.models.account import Account
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
-from cic_ussd.processor.util import parse_person_metadata
+from cic_ussd.processor.util import latest_input, parse_person_metadata
 from cic_ussd.translation import translation_for
-from sqlalchemy.orm.session import Session
 
 logg = logging.getLogger(__name__)
 
@@ -48,26 +43,21 @@ class MenuProcessor:
         :rtype:
         """
         available_balance = get_cached_available_balance(self.account.blockchain_address)
-        adjusted_balance = get_cached_adjusted_balance(self.identifier)
+        logg.debug('Requires call to retrieve tax and bonus amounts')
+        tax = ''
+        bonus = ''
         token_symbol = get_default_token_symbol()
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
-        with_available_balance = f'{self.display_key}.available_balance'
-        with_fees = f'{self.display_key}.with_fees'
-        if not adjusted_balance:
-            return translation_for(key=with_available_balance,
-                                   preferred_language=preferred_language,
-                                   available_balance=available_balance,
-                                   token_symbol=token_symbol)
-        adjusted_balance = json.loads(adjusted_balance)
-        tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
-        tax = from_wei(int(tax_wei))
-        return translation_for(key=with_fees,
-                               preferred_language=preferred_language,
-                               available_balance=available_balance,
-                               tax=tax,
-                               token_symbol=token_symbol)
+        return translation_for(
+            key=self.display_key,
+            preferred_language=preferred_language,
+            available_balance=available_balance,
+            tax=tax,
+            bonus=bonus,
+            token_symbol=token_symbol
+        )
 
     def account_statement(self) -> str:
         """
@@ -77,7 +67,7 @@ class MenuProcessor:
         cached_statement = get_cached_statement(self.account.blockchain_address)
         statement = json.loads(cached_statement)
         statement_transactions = parse_statement_transactions(statement)
-        transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)]
+        transaction_sets = [statement_transactions[tx:tx+3] for tx in range(0, len(statement_transactions), 3)]
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
@@ -159,22 +149,12 @@ class MenuProcessor:
         :return:
         :rtype:
         """
-        chain_str = Chain.spec.__str__()
         token_symbol = get_default_token_symbol()
         blockchain_address = self.account.blockchain_address
-        balances = get_balances(blockchain_address, chain_str, token_symbol, False)[0]
+        balances = get_balances(blockchain_address, Chain.spec.__str__(), token_symbol, False)[0]
         key = cache_data_key(self.identifier, ':cic.balances')
         cache_data(key, json.dumps(balances))
         available_balance = calculate_available_balance(balances)
-        now = datetime.now()
-        if (now - self.account.created).days >= 30:
-            if available_balance <= 0:
-                logg.info(f'Not retrieving adjusted balance, available balance: {available_balance} is insufficient.')
-            else:
-                timestamp = int((now - timedelta(30)).timestamp())
-                adjusted_balance = get_adjusted_balance(to_wei(int(available_balance)), chain_str, timestamp, token_symbol)
-                key = cache_data_key(self.identifier, ':cic.adjusted_balance')
-                cache_data(key, json.dumps(adjusted_balance))
 
         query_statement(blockchain_address)
 

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -63,7 +63,7 @@ elif ssl == 0:
 else:
     ssl = True
 
-valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
+
 def main():
 
     # TODO: improve url building 
@@ -79,9 +79,9 @@ def main():
     session = uuid.uuid4().hex
     data = {
             'sessionId': session,
-            'serviceCode': valid_service_codes[0],
+            'serviceCode': config.get('APP_SERVICE_CODE'),
             'phoneNumber': args.phone,
-            'text': "",
+            'text': config.get('APP_SERVICE_CODE'),
         }
 
     state = "_BEGIN"

apps/cic-ussd/cic_ussd/state_machine/logic/account.py

@@ -154,14 +154,15 @@ def parse_person_metadata(account: Account, metadata: dict):
     phone_number = account.phone_number
     date_registered = int(account.created.replace().timestamp())
     blockchain_address = account.blockchain_address
-    chain_str = Chain.spec.__str__()
+    chain_spec = f'{Chain.spec.common_name()}:{Chain.spec.engine()}: {Chain.spec.chain_id()}'
 
     if isinstance(metadata.get('identities'), dict):
         identities = metadata.get('identities')
     else:
         identities = manage_identity_data(
             blockchain_address=blockchain_address,
-            chain_str=chain_str
+            blockchain_type=Chain.spec.engine(),
+            chain_spec=chain_spec
         )
 
     return {

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -24,8 +24,7 @@ def transaction(notification_data: dict):
     :rtype:
     """
     role = notification_data.get('role')
-    token_value = notification_data.get('token_value')
-    amount = token_value if token_value == 0 else from_wei(token_value)
+    amount = from_wei(notification_data.get('token_value'))
     balance = notification_data.get('available_balance')
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 1, 'alpha.5')
+version = (0, 3, 1, 'alpha.4')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/config/client/app.ini

@@ -0,0 +1,2 @@
+[app]
+service_code = *483*46#

apps/cic-ussd/config/client/ussd.ini

@@ -1,4 +1,3 @@
 [ussd]
-service_code = *483*46#
 user =
 pass =

apps/cic-ussd/docker/Dockerfile

@@ -10,13 +10,6 @@ RUN mkdir -vp data
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
-
 COPY requirements.txt .
 
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \

apps/cic-ussd/requirements.txt

@@ -1,12 +1,9 @@
 alembic==1.4.2
-attrs==21.2.0
-billiard==3.6.4.0
 bcrypt==3.2.0
 celery==4.4.7
-cffi==1.14.6
 cic-eth[services]~=0.12.4a7
 cic-notify~=0.4.0a10
-cic-types~=0.1.0a15
+cic-types~=0.1.0a14
 confini>=0.4.1a1,<0.5.0
 phonenumbers==8.12.12
 psycopg2==2.8.6

apps/cic-ussd/tests/cic_ussd/account/test_balance.py

@@ -43,13 +43,10 @@ def test_sync_get_balances(activated_account,
     (5000000, 89000000, 67000000, 27.00)
 ])
 def test_calculate_available_balance(activated_account,
-                                     available_balance,
                                      balance_incoming,
                                      balance_network,
                                      balance_outgoing,
-                                     cache_balances,
-                                     cache_default_token_data,
-                                     load_chain_spec):
+                                     available_balance):
     balances = {
         'address': activated_account.blockchain_address,
         'converters': [],
@@ -60,11 +57,7 @@ def test_calculate_available_balance(activated_account,
     assert calculate_available_balance(balances) == available_balance
 
 
-def test_get_cached_available_balance(activated_account,
-                                      balances,
-                                      cache_balances,
-                                      cache_default_token_data,
-                                      load_chain_spec):
+def test_get_cached_available_balance(activated_account, cache_balances, balances):
     cached_available_balance = get_cached_available_balance(activated_account.blockchain_address)
     available_balance = calculate_available_balance(balances[0])
     assert cached_available_balance == available_balance

apps/cic-ussd/tests/cic_ussd/account/test_statement.py

@@ -27,8 +27,6 @@ def test_filter_statement_transactions(transactions_list):
 
 
 def test_generate(activated_account,
-                  cache_default_token_data,
-                  cache_statement,
                   cache_preferences,
                   celery_session_worker,
                   init_cache,
@@ -62,7 +60,7 @@ def test_get_cached_statement(activated_account, cache_statement, statement):
     assert cached_statement[0].get('blockchain_address') == statement[0].get('blockchain_address')
 
 
-def test_parse_statement_transactions(cache_default_token_data, statement):
+def test_parse_statement_transactions(statement):
     parsed_transactions = parse_statement_transactions(statement)
     parsed_transaction = parsed_transactions[0]
     parsed_transaction.startswith('Sent')
@@ -78,7 +76,7 @@ def test_query_statement(blockchain_address, limit, load_chain_spec, activated_a
     assert mock_transaction_list_query.get('limit') == limit
 
 
-def test_statement_transaction_set(cache_default_token_data, load_chain_spec, preferences, set_locale_files, statement):
+def test_statement_transaction_set(preferences, set_locale_files, statement):
     parsed_transactions = parse_statement_transactions(statement)
     preferred_language = preferences.get('preferred_language')
     transaction_set = statement_transaction_set(preferred_language, parsed_transactions)

apps/cic-ussd/tests/cic_ussd/account/test_transaction.py

@@ -36,19 +36,19 @@ def test_aux_transaction_data(preferences, set_locale_files, transactions_list):
     check_aux_data('helpers.sent', 'helpers.to', preferred_language, sender_tx_aux_data)
 
 
-@pytest.mark.parametrize("value, expected_result", [
+@pytest.mark.parametrize("wei, expected_result", [
     (50000000, Decimal('50.00')),
     (100000, Decimal('0.10'))
 ])
-def test_from_wei(cache_default_token_data, expected_result, value):
-    assert from_wei(value) == expected_result
+def test_from_wei(wei, expected_result):
+    assert from_wei(wei) == expected_result
 
 
 @pytest.mark.parametrize("value, expected_result", [
     (50, 50000000),
     (0.10, 100000)
 ])
-def test_to_wei(cache_default_token_data, expected_result, value):
+def test_to_wei(value, expected_result):
     assert to_wei(value) == expected_result
 
 
@@ -96,7 +96,6 @@ def test_validate_transaction_account(activated_account, init_database, transact
 @pytest.mark.parametrize("amount", [50, 0.10])
 def test_outgoing_transaction_processor(activated_account,
                                         amount,
-                                        cache_default_token_data,
                                         celery_session_worker,
                                         load_config,
                                         load_chain_spec,

apps/cic-ussd/tests/cic_ussd/processor/test_menu.py

@@ -1,6 +1,5 @@
 # standard imports
 import json
-import datetime
 
 # external imports
 from chainlib.hash import strip_0x
@@ -15,7 +14,6 @@ from cic_ussd.account.statement import (
 )
 from cic_ussd.account.tokens import get_default_token_symbol
 from cic_ussd.account.transaction import from_wei, to_wei
-from cic_ussd.cache import cache_data, cache_data_key
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
@@ -40,34 +38,24 @@ def test_menu_processor(activated_account,
                         load_chain_spec,
                         load_support_phone,
                         load_ussd_menu,
-                        mock_get_adjusted_balance,
                         mock_sync_balance_api_query,
                         mock_transaction_list_query,
                         valid_recipient):
     preferred_language = get_cached_preferred_language(activated_account.blockchain_address)
     available_balance = get_cached_available_balance(activated_account.blockchain_address)
     token_symbol = get_default_token_symbol()
-    with_available_balance = 'ussd.kenya.account_balances.available_balance'
-    with_fees = 'ussd.kenya.account_balances.with_fees'
+
+    tax = ''
+    bonus = ''
+    display_key = 'ussd.kenya.account_balances'
     ussd_menu = UssdMenu.find_by_name('account_balances')
     name = ussd_menu.get('name')
-    resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session)
-    assert resp == translation_for(with_available_balance,
+    resp = response(activated_account, display_key, name, init_database, generic_ussd_session)
+    assert resp == translation_for(display_key,
                                    preferred_language,
                                    available_balance=available_balance,
-                                   token_symbol=token_symbol)
-
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
-    key = cache_data_key(identifier, ':cic.adjusted_balance')
-    adjusted_balance = 45931650.64654012
-    cache_data(key, json.dumps(adjusted_balance))
-    resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session)
-    tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
-    tax = from_wei(int(tax_wei))
-    assert resp == translation_for(key=with_fees,
-                                   preferred_language=preferred_language,
-                                   available_balance=available_balance,
                                    tax=tax,
+                                   bonus=bonus,
                                    token_symbol=token_symbol)
 
     cached_statement = get_cached_statement(activated_account.blockchain_address)
@@ -135,15 +123,6 @@ def test_menu_processor(activated_account,
                                    account_balance=available_balance,
                                    account_token_name=token_symbol)
 
-    display_key = 'ussd.kenya.start'
-    ussd_menu = UssdMenu.find_by_name('start')
-    name = ussd_menu.get('name')
-    older_timestamp = (activated_account.created - datetime.timedelta(days=35))
-    activated_account.created = older_timestamp
-    init_database.flush()
-    response(activated_account, display_key, name, init_database, generic_ussd_session)
-    assert mock_get_adjusted_balance['timestamp'] == int((datetime.datetime.now() - datetime.timedelta(days=30)).timestamp())
-
     display_key = 'ussd.kenya.transaction_pin_authorization'
     ussd_menu = UssdMenu.find_by_name('transaction_pin_authorization')
     name = ussd_menu.get('name')

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py

@@ -49,7 +49,6 @@ def test_is_valid_transaction_amount(activated_account, amount, expected_result,
 ])
 def test_has_sufficient_balance(activated_account,
                                 cache_balances,
-                                cache_default_token_data,
                                 expected_result,
                                 generic_ussd_session,
                                 init_database,

apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py

@@ -121,7 +121,6 @@ def test_statement_callback(activated_account, mocker, transactions_list):
 def test_transaction_balances_callback(activated_account,
                                        balances,
                                        cache_balances,
-                                       cache_default_token_data,
                                        cache_person_metadata,
                                        cache_preferences,
                                        load_chain_spec,

apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py

@@ -13,8 +13,7 @@ from cic_ussd.translation import translation_for
 # tests imports
 
 
-def test_transaction(cache_default_token_data,
-                     celery_session_worker,
+def test_transaction(celery_session_worker,
                      load_support_phone,
                      mock_notifier_api,
                      notification_data,

apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py

@@ -30,11 +30,10 @@ def test_generate_statement(activated_account,
 
 
 def test_cache_statement(activated_account,
-                         cache_default_token_data,
                          cache_person_metadata,
-                         cache_preferences,
                          celery_session_worker,
                          init_database,
+                         preferences,
                          transaction_result):
     recipient_transaction, sender_transaction = transaction_actors(transaction_result)
     identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
@@ -42,7 +41,7 @@ def test_cache_statement(activated_account,
     cached_statement = get_cached_data(key)
     assert cached_statement is None
     s_parse_transaction = celery.signature(
-        'cic_ussd.tasks.processor.parse_transaction', [sender_transaction])
+        'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction])
     result = s_parse_transaction.apply_async().get()
     s_cache_statement = celery.signature(
         'cic_ussd.tasks.processor.cache_statement', [result, activated_account.blockchain_address]
@@ -62,15 +61,15 @@ def test_cache_statement(activated_account,
 
 def test_parse_transaction(activated_account,
                            cache_person_metadata,
-                           cache_preferences,
                            celery_session_worker,
                            init_database,
+                           preferences,
                            transaction_result):
     recipient_transaction, sender_transaction = transaction_actors(transaction_result)
     assert sender_transaction.get('metadata_id') is None
     assert sender_transaction.get('phone_number') is None
     s_parse_transaction = celery.signature(
-        'cic_ussd.tasks.processor.parse_transaction', [sender_transaction])
+        'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction])
     result = s_parse_transaction.apply_async().get()
     assert result.get('metadata_id') == activated_account.standard_metadata_id()
     assert result.get('phone_number') == activated_account.phone_number

apps/cic-ussd/tests/fixtures/patches/account.py

@@ -41,22 +41,6 @@ def mock_async_balance_api_query(mocker):
     return query_args
 
 
-@pytest.fixture(scope='function')
-def mock_get_adjusted_balance(mocker, task_uuid):
-    query_args = {}
-
-    def get_adjusted_balance(self, token_symbol, balance, timestamp):
-        sync_res = mocker.patch('celery.result.AsyncResult')
-        sync_res.id = task_uuid
-        sync_res.result = 45931650.64654012
-        query_args['balance'] = balance
-        query_args['timestamp'] = timestamp
-        query_args['token_symbol'] = token_symbol
-        return sync_res
-    mocker.patch('cic_eth_aux.erc20_demurrage_token.api.Api.get_adjusted_balance', get_adjusted_balance)
-    return query_args
-
-
 @pytest.fixture(scope='function')
 def mock_notifier_api(mocker):
     sms = {}

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -141,22 +141,12 @@ en:
         0. Back
       retry: |-
         %{retry_pin_entry}
-    account_balances:
-      available_balance: |-
-        CON Your balances are as follows:
-          balance: %{available_balance} %{token_symbol}
-        0. Back
-      with_fees: |-
-        CON Your balances are as follows:
-          balances: %{available_balance} %{token_symbol}
-          fees: %{tax} %{token_symbol}
-        0. Back
-      with_rewards: |-
-        CON Your balances are as follows:
-          balance: %{available_balance} %{token_symbol}
-          fees: %{tax} %{token_symbol}
-          rewards: %{bonus} %{token_symbol}
-        0. Back
+    account_balances: |-
+      CON Your balances are as follows:
+        balance: %{available_balance} %{token_symbol}
+        fees: %{tax} %{token_symbol}
+        rewards: %{bonus} %{token_symbol}
+      0. Back
     first_transaction_set: |-
       CON %{first_transaction_set}
       1. Next

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -140,22 +140,12 @@ sw:
         0. Nyuma
       retry: |-
         %{retry_pin_entry}
-    account_balances:
-      available_balance: |-
-        CON Salio zako ni zifuatazo:
-          salio: %{available_balance} %{token_symbol}
-        0. Nyuma
-      with_fees: |-
-        CON Salio zako ni zifuatazo:
-          salio: %{available_balance} %{token_symbol}
-          ushuru: %{tax} %{token_symbol}
-        0. Nyuma
-      with_rewards: |-
-        CON Salio zako ni zifuatazo:
-          salio: %{available_balance} %{token_symbol}
-          ushuru: %{tax} %{token_symbol}
-          tuzo: %{bonus} %{token_symbol}
-        0. Nyuma
+    account_balances: |-
+      CON Salio zako ni zifuatazo:
+        salio: %{available_balance} %{token_symbol}
+        ushuru: %{tax} %{token_symbol}
+        tuzo: %{bonus} %{token_symbol}
+      0. Nyuma
     first_transaction_set: |-
       CON %{first_transaction_set}
       1. Mbele

apps/contract-migration/config.sh

@@ -1,46 +0,0 @@
-#!/bin/bash
-
-set -a
-
-if [ -z $DEV_DATA_DIR ]; then
-	export DEV_DATA_DIR=`mktemp -d`
-else
-	mkdir -p $DEV_DATA_DIR
-fi
-
-if [ -z $DEV_CONFIG_RESET ]; then
-	if [ -f ${DEV_DATA_DIR}/env_reset ]; then
-		>&2 echo "importing existing configuration values from ${DEV_DATA_DIR}/env_reset"
-		. ${DEV_DATA_DIR}/env_reset
-	fi
-fi
-
-# Handle wallet
-export WALLET_KEY_FILE=${WALLET_KEY_FILE:-`realpath ./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c`}
-if [ ! -f $WALLET_KEY_FILE ]; then
-	>&2 echo "wallet path '$WALLET_KEY_FILE' does not point to a file"
-	exit 1
-fi
-export DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-checksum $(cat $WALLET_KEY_FILE | jq -r .address)`
-
-# Wallet dependent variable defaults
-export DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-export DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-export CIC_TRUST_ADDRESS=${CIC_TRUST_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
-export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
-
-
-# Legacy variable defaults
-
-
-# Migration variable processing
-confini-dump --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset
-echo "export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS
-export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
-export WALLET_KEY_FILE=$WALLET_KEY_FILE
-" >> ${DEV_DATA_DIR}/env_reset
-
-cat ${DEV_DATA_DIR}/env_reset
-
-set +a

apps/contract-migration/config/config.ini

@@ -1,13 +0,0 @@
-[dev]
-eth_account_contract_deployer =
-eth_account_reserve_minter =
-eth_account_accounts_index_writer =
-reserve_amount = 10000000000000000000000000000000000
-faucet_amount = 0
-gas_amount = 100000000000000000000000
-token_amount = 100000000000000000000000
-eth_gas_price =
-data_dir =
-pip_extra_index_url = 
-eth_provider_host = 
-eth_provider_port = 

apps/contract-migration/docker/Dockerfile_ci

@@ -0,0 +1,44 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e
+
+WORKDIR /root
+
+RUN touch /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
+RUN cat /etc/apt/sources.list.d/ethereum.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
+
+#RUN apt-get install solc
+
+RUN mkdir -vp /usr/local/etc/cic
+
+ENV CONFINI_DIR /usr/local/etc/cic/
+
+
+COPY config_template/ /usr/local/etc/cic/
+COPY requirements.txt . 
+COPY override_requirements.txt . 
+
+ARG pip_index_url=https://pypi.org/simple
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_ARGS=""
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG pip_trusted_host=pypi.org
+RUN pip install --index-url https://pypi.org/simple \
+    pip install --index-url https://pypi.org/simple \
+    --pre \
+    --force-reinstall \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r requirements.txt 
+
+RUN pip install --index-url https://pypi.org/simple \
+    --force-reinstall \
+    --pre \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r override_requirements.txt 
+
+COPY . .
+RUN chmod +x *.sh

apps/contract-migration/envlist

@@ -56,7 +56,6 @@ ETH_PROVIDER
 ETH_ABI_DIR
 SIGNER_SOCKET_PATH
 SIGNER_SECRET
-SIGNER_PROVIDER
 CELERY_BROKER_URL
 CELERY_RESULT_URL
 META_PROVIDER

apps/contract-migration/requirements.txt

@@ -1,5 +1,5 @@
 cic-eth[tools]==0.12.4a8
-chainlib-eth>=0.0.9a14,<0.1.0
+chainlib-eth>=0.0.9a9,<0.1.0
 eth-erc20>=0.1.2a3,<0.2.0
 erc20-demurrage-token>=0.0.5a2,<0.1.0
 eth-accounts-index>=0.1.2a2,<0.2.0
@@ -8,5 +8,3 @@ cic-eth-registry>=0.6.1a2,<0.7.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 erc20-faucet>=0.3.2a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0
-confini>=0.4.2rc3,<1.0.0
-crypto-dev-signer>=0.4.15a7,<=0.4.15

apps/contract-migration/reset.sh

@@ -2,122 +2,179 @@
 
 set -a
 
-. ${DEV_DATA_DIR}/env_reset
+default_token=giftable_erc20_token
+TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL}
+TOKEN_NAME=${TOKEN_NAME}
+TOKEN_TYPE=${TOKEN_TYPE:-$default_token}
+cat <<EOF
+external token settings:
+token_type: $TOKEN_TYPE
+token_symbol: $TOKEN_SYMBOL
+token_name: $TOKEN_NAME
+token_decimals: $TOKEN_DECIMALS
+token_demurrage: $TOKEN_DEMURRAGE_LEVEL
+token_redistribution_period: $TOKEN_REDISTRIBUTION_PERIOD
+token_supply_limit: $TOKEN_SUPPLY_LIMIT
+EOF
 
-WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60}
+CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC}
+RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER}
+
+DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
+DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
+DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
+DEV_FAUCET_AMOUNT=${DEV_FAUCET_AMOUNT:-0}
+DEV_ETH_KEYSTORE_FILE=${DEV_ETH_KEYSTORE_FILE:-`realpath ./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c`}
 
 set -e
 
+DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-checksum $(cat $DEV_ETH_KEYSTORE_FILE | jq -r .address)`
+
 if [ ! -z $DEV_ETH_GAS_PRICE ]; then
 	gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE"
 	fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE"
+	>&2 echo using static gas price $DEV_ETH_GAS_PRICE
 fi
 
+echo "environment:"
+printenv
+echo \n
+
+echo "using wallet address '$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER' from keystore file $DEV_ETH_KEYSTORE_FILE"
+
+# This is a grassroots team convention for building the Bancor contracts using the bancor protocol repository truffle setup
+# Running this in docker-internal dev container (built from Docker folder in this repo) will write a
+# source-able env file to CIC_DATA_DIR. Services dependent on these contracts can mount this file OR 
+# define these parameters at runtime
+# pushd /usr/src
+
+if [ -z $CIC_DATA_DIR ]; then
+	CIC_DATA_DIR=`mktemp -d`
+fi
+>&2 echo using data dir $CIC_DATA_DIR
+
+init_level_file=${CIC_DATA_DIR}/.init
+if [ ! -f ${CIC_DATA_DIR}/.init ]; then
+  echo "Creating .init file..."
+  mkdir -p $CIC_DATA_DIR
+  touch $CIC_DATA_DIR/.init
+#   touch $init_level_file
+fi
+echo -n 1 > $init_level_file
+
+# Abort on any error (including if wait-for-it fails).
+
 # Wait for the backend to be up, if we know where it is.
-if [ -z "${RPC_PROVIDER}" ]; then
-	echo "\$RPC_PROVIDER not set!"
-	exit 1
-fi
+if [[ -n "${ETH_PROVIDER}" ]]; then
 
-unset CONFINI_DIR
+	export CONFINI_DIR=$_CONFINI_DIR
+	unset CONFINI_DIR
 
-if [ ! -z "$DEV_USE_DOCKER_WAIT_SCRIPT" ]; then
-	IFS=: read -a p <<< "$RPC_PROVIDER"
-	read -i "/" rpc_provider_port <<< "${p[2]}"
-	rpc_provider_host=${p[1]:2}
-	echo "waiting for provider host $rpc_provider_host port $rpc_provider_port..."
-	./wait-for-it.sh "$rpc_provider_host:$rpc_provider_port" -t $WAIT_FOR_TIMEOUT
-fi
-
-if [ "$TOKEN_TYPE" == "giftable_erc20_token" ]; then
-	if [ -z "$TOKEN_SYMBOL" ]; then
-		>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
-		TOKEN_SYMBOL="GFT"
-		TOKEN_NAME="Giftable Token"
-	elif [ -z "$TOKEN_NAME" ]; then
-		>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
-		TOKEN_NAME=$TOKEN_SYMBOL
+	if [ ! -z "$DEV_USE_DOCKER_WAIT_SCRIPT" ]; then
+		echo "waiting for ${ETH_PROVIDER}..."
+		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 	fi
-	>&2 echo deploying default token $TOKEN_TYPE
-	echo giftable-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
-	DEV_RESERVE_ADDRESS=`giftable-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
-elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
-	if [ -z "$TOKEN_SYMBOL" ]; then
-		>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
-		TOKEN_SYMBOL="DET"
-		TOKEN_NAME="Demurrage Token"
-	elif [ -z "$TOKEN_NAME" ]; then
-		>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
-		TOKEN_NAME=$TOKEN_SYMBOL
-	fi
-	>&2 echo deploying token $TOKEN_TYPE
-	if [ -z $TOKEN_SINK_ADDRESS ]; then
-		if [ ! -z $TOKEN_REDISTRIBUTION_PERIOD ]; then
-			>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
+
+	if [ "$TOKEN_TYPE" == "$default_token" ]; then
+		if [ -z "$TOKEN_SYMBOL" ]; then
+			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
+			TOKEN_SYMBOL="GFT"
+			TOKEN_NAME="Giftable Token"
+		elif [ -z "$TOKEN_NAME" ]; then
+			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
+			TOKEN_NAME=$TOKEN_SYMBOL
 		fi
+		>&2 echo deploying default token $TOKEN_TYPE
+		echo giftable-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
+	elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
+		if [ -z "$TOKEN_SYMBOL" ]; then
+			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
+			TOKEN_SYMBOL="SARAFU"
+			TOKEN_NAME="Sarafu Token"
+		elif [ -z "$TOKEN_NAME" ]; then
+			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
+			TOKEN_NAME=$TOKEN_SYMBOL
+		fi
+		>&2 echo deploying token $TOKEN_TYPE
+		if [ -z $TOKEN_SINK_ADDRESS ]; then
+			if [ ! -z $TOKEN_REDISTRIBUTION_PERIOD ]; then
+				>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
+			fi
+		fi
+		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s` 
+	else
+		>&2 echo unknown token type $TOKEN_TYPE
+		exit 1
 	fi
-	DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s` 
+
+	echo "giftable-token-gift $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT"
+	giftable-token-gift $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
+
+	>&2 echo "deploy account index contract"
+	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -vv -s -u -w`
+	>&2 echo "add deployer address as account index writer"
+	eth-accounts-index-writer $fee_price_arg -s -u -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+
+	>&2 echo "deploy contract registry contract"
+	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $ETH_PROVIDER -vv -s -u -w`
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
+
+	# Deploy address declarator registry
+	>&2 echo "deploy address declarator contract"
+	declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
+	DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv $declarator_description`
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS
+
+	# Deploy transfer authorization contact
+	>&2 echo "deploy transfer auth contract"
+	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv --identifier TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
+
+	# Deploy token index contract
+	>&2 echo "deploy token index contract"
+	DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
+	>&2 echo "add reserve token to token index"
+	eth-token-index-add $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE  -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
+
+	# Sarafu faucet contract
+	>&2 echo "deploy token faucet contract"
+	DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s`
+
+	>&2 echo "set token faucet amount"
+	sarafu-faucet-set $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
+
+	>&2 echo "register faucet in registry"
+	eth-contract-registry-set -s -u $fee_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS
+
+	>&2 echo "set faucet as token minter"
+	giftable-token-minter -s -u $fee_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -e $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS
+
+	export CONFINI_DIR=$_CONFINI_DIR
 else
-	>&2 echo unknown token type $TOKEN_TYPE
+	echo "\$ETH_PROVIDER not set!"
 	exit 1
 fi
 
-echo "giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT"
-giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
+mkdir -p $CIC_DATA_DIR
+>&2 echo using data dir $CIC_DATA_DIR for environment variable dump
 
->&2 echo "deploy account index contract"
-DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w`
->&2 echo "add deployer address as account index writer"
-eth-accounts-index-writer $fee_price_arg -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+# this is consumed in downstream services to set environment variables
+cat << EOF > $CIC_DATA_DIR/.env
+export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
+export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
+EOF
 
->&2 echo "deploy contract registry contract"
-CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $RPC_PROVIDER -vv -s -u -w`
-eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $RPC_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
-eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
-
-# Deploy address declarator registry
->&2 echo "deploy address declarator contract"
-declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
-DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv $declarator_description`
-eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS
-
-# Deploy transfer authorization contact
->&2 echo "deploy transfer auth contract"
-DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
-eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $RPC_PROVIDER -vv --identifier TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
-
-# Deploy token index contract
->&2 echo "deploy token index contract"
-DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
-eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
->&2 echo "add reserve token to token index"
-eth-token-index-add $fee_price_arg -s -u -w -y $WALLET_KEY_FILE  -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
-
-# Sarafu faucet contract
->&2 echo "deploy token faucet contract"
-DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s`
-
->&2 echo "set token faucet amount"
-sarafu-faucet-set $fee_price_arg -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
-
->&2 echo "register faucet in registry"
-eth-contract-registry-set -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS
-
->&2 echo "set faucet as token minter"
-giftable-token-minter -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv $DEV_FAUCET_ADDRESS
-
-
-#echo "export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL" >> ${DEV_DATA_DIR}/env_reset
-export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
-
-echo "Writing env_reset file ..."
-
-echo "export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
-export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
-export TOKEN_NAME=$TOKEN_NAME
-" >> "${DEV_DATA_DIR}"/env_reset
+cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
+cat ./envlist
+# popd
 
 set +a
 set +e
 
+echo -n 2 > $init_level_file
+
 exec "$@"

apps/contract-migration/run_job.sh

@@ -1,13 +1,5 @@
 #! /bin/bash
 
->&2 echo -e "\033[;96mRUNNING\033[;39m configurations"
-./config.sh
-if [ $? -ne "0" ]; then
-	>&2 echo -e "\033[;31mFAILED\033[;39m configurations"
-	exit 1;
-fi
->&2 echo -e "\033[;32mSUCCEEDED\033[;39m configurations"
-
 if [[ $((RUN_MASK & 1)) -eq 1 ]]
 then
 	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 1 - contract deployment"

apps/contract-migration/seed_cic_eth.sh

@@ -1,40 +1,68 @@
 #!/bin/bash
 
 # defaults
-source ${DEV_DATA_DIR}/env_reset
-cat ${DEV_DATA_DIR}/env_reset
+#initlevel=`cat ${CIC_DATA_DIR}/.init`
+#echo $inilevel
+#if [ $initlevel -lt 2 ]; then
+#	>&2 echo "initlevel too low $initlevel"
+#	exit 1 
+#fi
+source ${CIC_DATA_DIR}/.env
+source ${CIC_DATA_DIR}/.env_all
+DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
+DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"}
+CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic} 
+ETH_PASSPHRASE=''
+CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
+
+CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC}
+RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER}
 
 # Debug flag
+DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+keystore_file=./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
 debug='-vv'
+gas_amount=100000000000000000000000
+token_amount=${gas_amount}
+env_out_file=${CIC_DATA_DIR}/.env_seed
+init_level_file=${CIC_DATA_DIR}/.init
 empty_config_dir=$CONFINI_DIR/empty
+truncate $env_out_file -s 0
 
 set -e
 set -a
 
+#pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL  eth-address-index==0.1.1a7
+
+export CONFINI_DIR=$_CONFINI_DIR
 unset CONFINI_DIR
 
 # get required addresses from registries
-token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw TokenRegistry`
-account_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw AccountRegistry`
-reserve_address=`eth-token-index-list -i $CHAIN_SPEC -u -p $RPC_PROVIDER -e $token_index_address -vv --raw $CIC_DEFAULT_TOKEN_SYMBOL`
-
->&2 echo "Token registry: $token_index_address"
->&2 echo "Account registry: $account_index_address"
->&2 echo "Reserve address: $reserve_address ($TOKEN_SYMBOL)"
+DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $ETH_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw TokenRegistry`
+DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $ETH_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw AccountRegistry`
+DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -u -p $ETH_PROVIDER -e $DEV_TOKEN_INDEX_ADDRESS -vv --raw $CIC_DEFAULT_TOKEN_SYMBOL`
+cat <<EOF
+Token registry: $DEV_TOKEN_INDEX_ADDRESS
+Account reigstry: $DEV_ACCOUNT_INDEX_ADDRESS
+Reserve address: $DEV_RESERVE_ADDRESS ($CIC_DEFAULT_TOKEN_SYMBOL)
+EOF
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
-#DEV_ETH_ACCOUNT_GAS_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create --redis-timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
-DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --redis-timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_GAS_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create --redis-timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
 cic-eth-tag -i $CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
 
 
 >&2 echo "create account for sarafu gifter"
 DEV_ETH_ACCOUNT_SARAFU_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
 cic-eth-tag -i $CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER
 
 >&2 echo "create account for approval escrow owner"
 DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file
 cic-eth-tag -i $CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER 
 
 #>&2 echo "create account for faucet owner"
@@ -44,45 +72,50 @@ cic-eth-tag -i $CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFE
 
 >&2 echo "create account for accounts index writer"
 DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file
 cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"
-eth-accounts-index-writer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $account_index_address -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
+eth-accounts-index-writer -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 
 # Transfer gas to custodial gas provider adddress
 _CONFINI_DIR=$CONFINI_DIR
 unset CONFINI_DIR
 >&2 echo gift gas to gas gifter
->&2 eth-gas -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $DEV_GAS_AMOUNT
+>&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
 
 >&2 echo gift gas to sarafu token owner
->&2 eth-gas -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $DEV_GAS_AMOUNT
+>&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
 
 >&2 echo gift gas to account index owner
->&2 eth-gas -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $DEV_GAS_AMOUNT
+>&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
 
 
 # Send token to token creator
 >&2 echo "gift tokens to sarafu owner"
-echo "giftable-token-gift -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $reserve_address -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $DEV_TOKEN_AMOUNT"
->&2 giftable-token-gift -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $reserve_address -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $DEV_TOKEN_AMOUNT
+echo "giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount"
+>&2 giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount
 
 # Send token to token gifter
 >&2 echo "gift tokens to keystore address"
->&2 giftable-token-gift -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $reserve_address -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $DEV_TOKEN_AMOUNT
+>&2 giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount
 
 >&2 echo "set sarafu token to reserve token (temporarily while bancor contracts are not connected)"
+echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >> $env_out_file
 export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 # Transfer tokens to gifter address
 >&2 echo "transfer tokens to token gifter address"
->&2 erc20-transfer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER --fee-limit 100000 -e $reserve_address -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${DEV_TOKEN_AMOUNT:0:-1}
+>&2 erc20-transfer -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --fee-limit 100000 -e $DEV_RESERVE_ADDRESS -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
 
+#echo -n 0 > $init_level_file
+
+#CONFINI_DIR=$_CONFINI_DIR
 # Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
 cic-eth-ctl -i $CHAIN_SPEC unlock INIT
 cic-eth-ctl -i $CHAIN_SPEC unlock SEND
 cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE
 
-#confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config
+export CONFINI_DIR=$_CONFINI_DIR
 
 set +a
 set +e

apps/data-seeding/cic_eth/paths.py

@@ -7,7 +7,6 @@ import json
 # external imports
 import celery
 import confini
-import celery.utils.graph
 
 # local imports
 from cic_eth.api import Api
@@ -36,7 +35,7 @@ class Fmtr(celery.utils.graph.GraphFormatter):
 
 def main():
     api = Api(
-        config.get('CHAIN_SPEC'),
+        config.get('CIC_CHAIN_SPEC'),
         queue='cic-eth',
         #callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
         #callback_task='cic_eth.callbacks.redis.redis',

apps/data-seeding/cic_ussd/import_balance.py

@@ -16,7 +16,7 @@ from crypto_dev_signer.keystore.dict import DictKeystore
 from import_util import BalanceProcessor, get_celery_worker_status
 from import_task import ImportTask, MetadataTask
 
-default_config_dir = '/usr/local/etc/data-seeding/'
+default_config_dir = './config'
 logg = logging.getLogger()
 
 arg_parser = argparse.ArgumentParser(description='Daemon worker that handles data seeding tasks.')

apps/data-seeding/config/cic.ini

@@ -1,10 +1,11 @@
 [cic]
-registry_address =
+registry_address = 0x32E860c2A0645d1B7B005273696905F5D6DC5D05
 token_index_address =
 accounts_index_address = 
 declarator_address = 
 approval_escrow_address =
-chain_spec =
+chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
-trust_address =
+trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
 user_ussd_svc_service_port =
+

apps/data-seeding/create_import_users.py

@@ -28,11 +28,12 @@ logg = logging.getLogger()
 
 fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
 
-default_config_dir = './config'
-
+script_dir = os.path.realpath(os.path.dirname(__file__))
+# config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
 
 argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=default_config_dir, help='Config dir')
+argparser.add_argument('-c', type=str, default=config_dir, help='Config dir')
 argparser.add_argument('--tag', type=str, action='append',
                        help='Tags to add to record')
 argparser.add_argument('--gift-threshold', type=int,
@@ -52,7 +53,7 @@ elif args.vv:
 
 config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
-logg.debug('loaded config\n{}'.format(config))
+logg.info('loaded config\n{}'.format(config))
 
 
 dt_now = datetime.datetime.utcnow()

apps/data-seeding/docker/Dockerfile

@@ -13,8 +13,7 @@ COPY package.json \
 RUN npm ci --production 
 #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
 
-COPY requirements.txt .
-COPY config/ /usr/local/etc/data-seeding
+COPY requirements.txt . 
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"

apps/data-seeding/docker/Dockerfile_ci

@@ -0,0 +1,24 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
+
+WORKDIR /root
+
+RUN mkdir -vp /usr/local/etc/cic
+
+COPY package.json \
+     package-lock.json \
+     .
+
+RUN npm install
+
+COPY requirements.txt . 
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL -r requirements.txt
+
+COPY . . 
+
+ENTRYPOINT [ ]

apps/data-seeding/import_ussd.sh

@@ -1,94 +1,60 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-if [[ -d "$OUT_DIR" ]]
-then
-  echo "found existing IMPORT DIR cleaning up..."
-  rm -rf "$OUT_DIR"
-  mkdir -p "$OUT_DIR"
-else
-  echo "IMPORT DIR does not exist creating it."
-  mkdir -p "$OUT_DIR"
-fi
-
-# using timeout because the timeout flag for celery inspect does not work
-timeout 5 celery inspect ping -b "$CELERY_BROKER_URL"
-if [[ $? -eq 124 ]]
-then
-  >&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?"
-  exit 1
-fi
+set -e
 
 echo "Creating seed data..."
-python create_import_users.py -vv  -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
+python create_import_users.py -vv --dir "$IMPORT_DIR" "$ACCOUNT_COUNT"
 wait $!
-
-echo "Check for running celery workers ..."
-if [ -f ./cic-ussd-import.pid ];
-then
-  echo "Found a running worker. Killing ..."
-  kill -9 $(<cic-ussd-import.pid)
-fi
-
 echo "Purge tasks from celery worker"
 celery -A cic_ussd.import_task purge -Q "$CELERY_QUEUE" --broker redis://"$REDIS_HOST":"$REDIS_PORT" -f
-
 echo "Start celery work and import balance job"
 if [ "$INCLUDE_BALANCES" != "y" ]
 then
   echo "Running worker without opening balance transactions"
-  TARGET_TX_COUNT=$NUMBER_OF_USERS
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
+  TARGET_TX_COUNT=$ACCOUNT_COUNT
+  python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$IMPORT_DIR" &
 else
   echo "Running worker with opening balance transactions"
-  TARGET_TX_COUNT=$((NUMBER_OF_USERS*2))
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" &
+  TARGET_TX_COUNT=$((ACCOUNT_COUNT*2))
+  python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$IMPORT_DIR" &
 fi
 
-echo "Target count set to ${TARGET_TX_COUNT}"
 until [ -f ./cic-import-ussd.pid ]
 do
   echo "Polling for celery worker pid file..."
   sleep 1
 done
 IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid)
-
 echo "Start import users job"
 if [ "$USSD_SSL" == "y" ]
 then
   echo "Targeting secure ussd-user server"
-  python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$IMPORT_DIR"
 else
-  python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$IMPORT_DIR"
 fi
-
 echo "Waiting for import balance job to complete ..."
 tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null
 set -e
-
 echo "Importing pins"
-python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_pins.py -c "$CONFIG" -vv "$IMPORT_DIR"
 set +e
 wait $!
 set -e
-
 echo "Importing ussd data"
-python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$IMPORT_DIR"
 set +e
 wait $!
-
 echo "Importing person metadata"
-node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS"
-
+node cic_meta/import_meta.js "$IMPORT_DIR" "$ACCOUNT_COUNT"
 echo "Import preferences metadata"
-node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS"
-
+node cic_meta/import_meta_preferences.js "$IMPORT_DIR" "$ACCOUNT_COUNT"
 CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME
 NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
 while [[ "$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ]]
 do
   NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
   sleep 5
-  echo "Notification count is: ${NOTIFICATION_COUNT} of ${TARGET_TX_COUNT}. Checking after 5 ..."
+  echo "Notification count is: ${NOTIFICATION_COUNT}. Checking after 5 ..."
 done
-echo "Running verify script."
-python verify.py -c "$CONFIG" -v -p "$ETH_PROVIDER"  -r "$CIC_REGISTRY_ADDRESS" --exclude "$EXCLUSIONS" --meta-provider "$META_URL" --token-symbol "$TOKEN_SYMBOL" --ussd-provider "$USSD_PROVIDER" "$OUT_DIR"
+python verify.py -c "$CONFIG" -v -p "$ETH_PROVIDER"  -r "$CIC_REGISTRY_ADDRESS" --exclude "$EXCLUSIONS" --token-symbol "$TOKEN_SYMBOL" "$IMPORT_DIR"

apps/data-seeding/requirements.txt

@@ -1,14 +1,13 @@
-sarafu-faucet~=0.0.7a2
+sarafu-faucet~=0.0.7a1
 cic-eth[tools]~=0.12.4a8
-cic-types~=0.1.0a15
-crypto-dev-signer~=0.4.15a7
+cic-types~=0.1.0a14
+crypto-dev-signer>=0.4.15a1,<=0.4.15
 faker==4.17.1
 chainsyncer~=0.0.6a3
-chainlib-eth~=0.0.9a14
+chainlib-eth~=0.0.9a7
 eth-address-index~=0.2.3a4
 eth-contract-registry~=0.6.3a3
 eth-accounts-index~=0.1.2a3
-eth-erc20~=0.1.2a3
+eth-erc20~=0.1.2a2
 erc20-faucet~=0.3.2a2
 psycopg2==2.8.6
-liveness~=0.0.1a7

apps/data-seeding/verify.py

@@ -164,8 +164,20 @@ for t in custodial_tests:
         logg.info('activating custodial module'.format(t))
         break
 
+cols = os.get_terminal_size().columns
 
-outfunc = logg.debug
+
+def to_terminalwidth(s):
+    ss = s.ljust(int(cols)-1)
+    ss += "\r"
+    return ss
+
+def default_outfunc(s):
+    ss = to_terminalwidth(s)
+    sys.stdout.write(ss)
+outfunc = default_outfunc
+if logg.isEnabledFor(logging.DEBUG):
+    outfunc = logg.debug
 
 
 def send_ussd_request(address, data_dir):

ci_templates/.cic-template.yml

@@ -1,12 +1,12 @@
+image:
+  name: gcr.io/kaniko-project/executor:debug
+  entrypoint: [""]
 
 variables:
   KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
   MR_IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA
 
 .py_build_merge_request:
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: [""]
   stage: build
   script:
     - mkdir -p /kaniko/.docker
@@ -16,9 +16,6 @@ variables:
       --cache-repo $CI_REGISTRY_IMAGE --destination $MR_IMAGE_TAG 
 
 .py_build_target_dev:
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: [""]
   stage: build
   variables:
     IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA
@@ -31,9 +28,6 @@ variables:
       --destination $MR_IMAGE_TAG
 
 .py_build_push:
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: [""]
   stage: build
   variables:
       IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA

docker-compose.yml

@@ -60,6 +60,8 @@ services:
 
   contract-migration:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest}
+    profiles:
+      - migrations 
     build:
       context: apps/contract-migration
       dockerfile: docker/Dockerfile
@@ -70,14 +72,16 @@ services:
         EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
-      RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
-      ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
-      RPC_HTTP_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
+      ETH_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545}
+      RPC_HTTP_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545}
+      # And these two are for wait-for-it (could parse this)
       DEV_USE_DOCKER_WAIT_SCRIPT: 1
+      ETH_PROVIDER_HOST: eth 
+      ETH_PROVIDER_PORT: 8545
       CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
-      DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}
-      DEV_CONFIG_RESET: $DEV_CONFIG_RESET
+      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
@@ -91,19 +95,18 @@ services:
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
       RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
-      DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-50000000}
-      DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE
-      TOKEN_NAME: ${TOKEN_NAME:-Giftable Token}
-      TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
-      TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token}
+      DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
+      #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
+      DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
+      CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
+      TOKEN_NAME: $TOKEN_NAME
       TOKEN_DECIMALS: $TOKEN_DECIMALS
-      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD
-      TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
+      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
       TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
-      TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL
+      TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
       TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
-      SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
-    restart: on-failure 
+      TOKEN_TYPE: $TOKEN_TYPE
+        #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
     command: ["./run_job.sh"]
         #command: ["./reset.sh"]
     depends_on:
@@ -114,51 +117,10 @@ services:
     volumes:
       - contract-config:/tmp/cic/config
 
-
-  data-seeding:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:${TAG:-latest}
-    build:
-      context: apps/data-seeding
-      dockerfile: docker/Dockerfile
-      args:
-        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
-        pip_extra_args: $PIP_EXTRA_ARGS
-    environment:
-      CIC_REGISTRY_ADDRESS: ${CIC_REGISTRY_ADDRESS:-0xea6225212005e86a4490018ded4bf37f3e772161}
-      OUT_DIR: out
-      NUMBER_OF_USERS: 10
-      CONFIG: /usr/local/etc/data-seeding
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
-      ETH_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545}
-      TOKEN_SYMBOL: GFT 
-      KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
-      USSD_HOST: cic-user-ussd-server 
-      USSD_PORT: 9000
-      INCLUDE_BALANCES: y
-      USSD_SSL: n
-      DATABASE_NAME: ${DATABASE_NAME:-cic_ussd}
-      DATABASE_USER: ${DATABASE_USER:-grassroots}
-      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
-      DATABASE_HOST: ${DATABASE_HOST:-postgres}
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
-      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
-      NOTIFY_DATABASE_NAME: cic_notify
-      REDIS_HOST: redis 
-      REDIS_PORT: 6379
-      REDIS_DB: 0
-      META_HOST: meta
-      META_PORT: 8000
-      META_URL: http://meta:8000
-      USSD_PROVIDER: http://cic-user-ussd-server:9000
-      CELERY_QUEUE: cic-import-ussd
-      EXCLUSIONS: ussd
-    command: bash import_ussd.sh
-    volumes:
-      - contract-config:/tmp/cic/config/:ro
-
   cic-cache-tracker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
+    profiles:
+      - cache 
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -166,8 +128,8 @@ services:
         EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
     environment:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
-      ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
-      RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
+      ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545}
+      RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545}
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
@@ -181,7 +143,9 @@ services:
       CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
       CELERY_BROKER_URL: redis://redis:6379
       CELERY_RESULT_URL: redis://redis:6379
-    restart: on-failure 
+    deploy: 
+      restart_policy:
+        condition: on-failure
     depends_on:
       - redis
       - postgres
@@ -190,13 +154,15 @@ services:
       - /bin/bash
       - -c
       - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         ./start_tracker.sh -c /usr/local/etc/cic-cache -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-tasker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
+    profiles:
+      - cache 
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -218,7 +184,9 @@ services:
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CELERY_BROKER_URL: redis://redis:6379
       CELERY_RESULT_URL: redis://redis:6379
-    restart: unless-stopped 
+    deploy: 
+      restart_policy:
+        condition: on-failure
     depends_on:
       - redis
       - postgres
@@ -227,13 +195,15 @@ services:
       - /bin/bash
       - -c
       - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         /usr/local/bin/cic-cache-taskerd -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-server:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
+    profiles:
+      - cache 
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -248,18 +218,18 @@ services:
       DATABASE_DEBUG: 1
         #PGPASSWORD: $DATABASE_PASSWORD
       SERVER_PORT: 8000
-    restart: on-failure 
     ports:
       - ${HTTP_PORT_CIC_CACHE:-63313}:8000
     depends_on:
       - postgres
-      - cic-cache-tasker
-      - cic-cache-tracker
+    deploy:
+      restart_policy:
+        condition: on-failure
     command:
       - /bin/bash
       - -c
       - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         "/usr/local/bin/uwsgi" \
         --wsgi-file /root/cic_cache/runnable/daemons/server.py \
         --http :8000 \
@@ -278,7 +248,7 @@ services:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
       ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER
       ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
-      RPC_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
+      RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
@@ -296,81 +266,28 @@ services:
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
       CELERY_DEBUG: ${CELERY_DEBUG:-1}
-      #SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
-      SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-eth-signer:8000}
+      SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc}
+      SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc}
       SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
       ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
       TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
       CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
-    restart: unless-stopped 
     depends_on:
       - eth
       - postgres
       - redis
-      - cic-eth-signer
+    deploy:
+      restart_policy:
+        condition: on-failure
     volumes:
-      - signer-data:/run/crypto-dev-signer
+      - signer-data:/tmp/cic/signer
       - contract-config:/tmp/cic/config/:ro
-        #command: ["/usr/local/bin/cic-eth-taskerd"]
-        #command: ["sleep", "3600"]
     command:
       - /bin/bash
       - -c
       - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         ./start_tasker.sh --aux-all -q cic-eth  -vv
-    
-  cic-eth-signer:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
-    build: 
-        context: apps/cic-eth
-        dockerfile: docker/Dockerfile
-        target: dev
-        args:
-          EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-      ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER
-      ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
-      RPC_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
-      DATABASE_USER: ${DATABASE_USER:-grassroots}
-      DATABASE_HOST: ${DATABASE_HOST:-postgres}
-      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
-      DATABASE_NAME: ${DATABASE_NAME_CIC_ETH:-cic_eth}
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
-      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
-      DATABASE_POOL_SIZE: 0
-      REDIS_PORT: 6379
-      REDIS_HOST: redis
-      PGPASSWORD: ${DATABASE_PASSWORD:-tralala}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
-      CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
-      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
-      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
-      CELERY_DEBUG: ${CELERY_DEBUG:-1}
-      SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000}
-      SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000}
-      SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
-      ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
-      TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
-      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
-    restart: on-failure 
-    depends_on:
-      - eth
-      - postgres
-      - redis
-    volumes:
-      - signer-data:/run/crypto-dev-signer
-      - contract-config:/tmp/cic/config/:ro
-    command: ["python", "/usr/local/bin/crypto-dev-daemon", "-c", "/usr/local/etc/crypto-dev-signer", "-vv"]
-        #command:
-        #  - /bin/bash
-        #  - -c
-        #  - |
-        #    if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        #    ./start_tasker.sh --aux-all -q cic-eth  -vv
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker:
@@ -382,8 +299,8 @@ services:
         args:
           EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
     environment:
-      RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
-      ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
+      RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545}
+      ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545}
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
@@ -395,21 +312,24 @@ services:
       CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
       CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
+        #BANCOR_DIR: $BANCOR_DIR
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
-      TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
-    restart: on-failure 
+      TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
     depends_on:
       - eth
       - postgres
       - redis
+    deploy:
+      restart_policy:
+        condition: on-failure
     volumes:
       - contract-config:/tmp/cic/config/:ro
     command:
       - /bin/bash
       - -c
       - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         #./start_tracker.sh -vv -c /usr/local/etc/cic-eth
         ./start_tracker.sh -vv
     # command: "/root/start_manager.sh head -vv"
@@ -425,7 +345,7 @@ services:
           EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
     environment:
       ETH_PROVIDER: http://eth:8545
-      RPC_PROVIDER: http://eth:8545
+      RPC_HTTP_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
@@ -442,21 +362,22 @@ services:
       TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
       DATABASE_DEBUG: ${DATABASE_DEBUG:-false}
         #DATABASE_DEBUG: 1
-    restart: on-failure 
     depends_on:
       - eth
       - postgres
       - redis
+    deploy:
+      restart_policy:
+        condition: on-failure
     volumes:
         - contract-config:/tmp/cic/config/:ro
     command:
       - /bin/bash
       - -c
       - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         ./start_dispatcher.sh -q cic-eth -vv
     # command: "/root/start_dispatcher.sh -q cic-eth -vv"
-    
 
 
   cic-eth-retrier:
@@ -469,7 +390,7 @@ services:
           EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
     environment:
       ETH_PROVIDER: http://eth:8545
-      RPC_PROVIDER: http://eth:8545
+      RPC_HTTP_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
@@ -488,18 +409,20 @@ services:
       CIC_TX_RETRY_DELAY: 60
       BATCH_SIZE: ${RETRIER_BATCH_SIZE:-50}
         #DATABASE_DEBUG: 1
-    restart: on-failure 
     depends_on:
       - eth
       - postgres
       - redis
+    deploy:
+      restart_policy:
+        condition: on-failure
     volumes:
         - contract-config:/tmp/cic/config/:ro
     command:
       - /bin/bash
       - -c
       - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         ./start_retry.sh -vv 
     # command: "/root/start_retry.sh -q cic-eth -vv"
 
@@ -525,15 +448,19 @@ services:
       AFRICASTALKING_API_USERNAME: $AFRICASTALKING_API_USERNAME
       AFRICASTALKING_API_KEY: $AFRICASTALKING_API_KEY
       AFRICASTALKING_API_SENDER_ID: $AFRICASTALKING_API_SENDER_ID
-    restart: unless-stopped 
     depends_on:
       - postgres
       - redis
+    deploy:
+      restart_policy:
+        condition: on-failure
     command: "/root/start_tasker.sh -q cic-notify -vv"
 
 
   cic-meta-server:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest}
+    profiles:
+      - custodial-meta
     hostname: meta
     build:
       context: apps/cic-meta
@@ -555,17 +482,21 @@ services:
       PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc
       PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc
       SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql
-    restart: on-failure 
     ports:
       - ${HTTP_PORT_CIC_META:-63380}:8000
     depends_on:
       - postgres
+    deploy:
+      restart_policy:
+        condition: on-failure
     volumes:
       - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp
     # command: "/root/start_server.sh -vv"
 
   cic-user-ussd-server:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
+    profiles:
+      - custodial-ussd
     build:
       context: apps/cic-ussd
       dockerfile: docker/Dockerfile
@@ -583,7 +514,6 @@ services:
       PGP_PASSPHRASE: merman
       SERVER_PORT: 9000
       CIC_META_URL: ${CIC_META_URL:-http://meta:8000}
-    restart: on-failure 
     ports:
       - ${HTTP_PORT_CIC_USER_USSD_SERVER:-63315}:9000
     depends_on:
@@ -591,10 +521,15 @@ services:
       - redis
     volumes:
       - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/
+    deploy:
+      restart_policy:
+        condition: on-failure
     command: "/root/start_cic_user_ussd_server.sh -vv"
 
   cic-user-server:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
+    profiles:
+      - custodial-ussd
     build:
       context: apps/cic-ussd
       dockerfile: docker/Dockerfile
@@ -608,15 +543,19 @@ services:
       DATABASE_ENGINE: postgresql
       DATABASE_DRIVER: psycopg2
       DATABASE_POOL_SIZE: 0
-    restart: on-failure 
     ports:
       - ${HTTP_PORT_CIC_USER_SERVER:-63415}:9500
     depends_on:
       - postgres
+    deploy:
+      restart_policy:
+        condition: on-failure
     command: "/root/start_cic_user_server.sh -vv"
 
   cic-user-tasker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
+    profiles:
+      - custodial-ussd
     build:
       context: apps/cic-ussd/
       dockerfile: docker/Dockerfile
@@ -634,10 +573,12 @@ services:
       CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis}
       PGP_PASSPHRASE: merman
       CIC_META_URL: ${CIC_META_URL:-http://meta:8000}
-    restart: unless-stopped 
     depends_on:
       - postgres
       - redis
     volumes:
       - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/
+    deploy:
+      restart_policy:
+        condition: on-failure
     command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv"

scripts/dump.sh

@@ -1 +0,0 @@
-docker run -t -v  --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset