Merge remote-tracking branch 'origin/master' into lash/migration-last-gasp

apps/cic-cache/.config/database.ini

@@ -1,5 +1,5 @@
 [database]
-NAME=cic-eth
+NAME=cic_cache
 USER=postgres
 PASSWORD=
 HOST=localhost

apps/cic-cache/cic_cache/db/migrations/default/alembic.ini

@@ -36,7 +36,7 @@ script_location = .
 # output_encoding = utf-8
 
 #sqlalchemy.url = driver://user:pass@localhost/dbname
-sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic-cache
+sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic_cache
 
 
 [post_write_hooks]

apps/cic-cache/db/initdb_files/create_all_db.sql

@@ -1,5 +0,0 @@
-CREATE DATABASE "cic-cache";
-CREATE DATABASE "cic-eth";
-CREATE DATABASE "cic-notify";
-CREATE DATABASE "cic-meta";
-CREATE DATABASE "cic-signer";

apps/cic-cache/setup.cfg

@@ -25,6 +25,7 @@ licence_files =
 python_requires = >= 3.6
 packages = 
 	cic_cache
+	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
 	cic_cache.runnable
@@ -33,5 +34,6 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-cache-tracker = cic_cache.runnable.tracker:main
+	cic-cache-trackerd = cic_cache.runnable.tracker:main
-	cic-cache-server = cic_cache.runnable.server:main
+	cic-cache-serverd = cic_cache.runnable.server:main
+	cic-cache-taskerd = cic_cache.runnable.tasker:main

apps/cic-notify/cic_notify/version.py

@@ -10,7 +10,7 @@ from cic_notify.error import PleaseCommitFirstError
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.2')
+version = (0, 4, 0, 'alpha.3')
 
 version_object = semver.VersionInfo(
         major=version[0],
@@ -24,9 +24,6 @@ version_string = str(version_object)
 
 def git_hash():
     import subprocess
-    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
-    if len(git_diff.stdout) > 0:
-        raise PleaseCommitFirstError()
 
     git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
     git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
@@ -35,7 +32,7 @@ def git_hash():
 
 try:
     version_git = git_hash()
-    version_string += '.build.{}'.format(version_git)
+    version_string += '+build.{}'.format(version_git)
 except FileNotFoundError:
     time_string_pair = str(time.time()).split('.')
     version_string += '+build.{}{:<09d}'.format(

apps/cic-notify/requirements.txt

@@ -1,5 +1,5 @@
-celery~=4.4.7
-confini~=0.3.6a1
 alembic~=1.4.2
+celery~=4.4.7
+confini~=0.3.6rc3
 redis~=3.5.3
 semver==2.13.0

apps/cic-ussd/cic_ussd/operations.py

@@ -14,7 +14,7 @@ from cic_ussd.db.models.user import User
 from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.menu.ussd_menu import UssdMenu
-from cic_ussd.processor import custom_display_text, process_request
+from cic_ussd.processor import custom_display_text, process_request, retrieve_most_recent_ussd_session
 from cic_ussd.redis import InMemoryStore
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.validator import check_known_user, validate_response_type
@@ -60,7 +60,8 @@ def create_ussd_session(
         phone: str,
         service_code: str,
         user_input: str,
-        current_menu: str) -> InMemoryUssdSession:
+        current_menu: str,
+        session_data: Optional[dict] = None) -> InMemoryUssdSession:
     """
     Creates a new ussd session
     :param external_session_id: Session id value provided by AT
@@ -73,6 +74,8 @@ def create_ussd_session(
     :type user_input: str
     :param current_menu: Menu name that is currently being displayed on the ussd session
     :type current_menu: str
+    :param session_data: Any additional data that was persisted during the user's interaction with the system.
+    :type session_data: dict.
     :return: ussd session object
     :rtype: Session
     """
@@ -81,7 +84,8 @@ def create_ussd_session(
         msisdn=phone,
         user_input=user_input,
         state=current_menu,
-        service_code=service_code
+        service_code=service_code,
+        session_data=session_data
     )
     return session
 
@@ -126,7 +130,9 @@ def create_or_update_session(
             phone=phone,
             service_code=service_code,
             user_input=user_input,
-            current_menu=current_menu)
+            current_menu=current_menu,
+            session_data=session_data
+        )
     return ussd_session
 
 
@@ -338,14 +344,26 @@ def process_menu_interaction_requests(chain_str: str,
                 user_input=user_input
             )
 
-        # create or update the ussd session as appropriate
+        last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
-        ussd_session = create_or_update_session(
+
-            external_session_id=external_session_id,
+        if last_ussd_session:
-            phone=phone_number,
+            # create or update the ussd session as appropriate
-            service_code=service_code,
+            ussd_session = create_or_update_session(
-            user_input=user_input,
+                external_session_id=external_session_id,
-            current_menu=current_menu.get('name')
+                phone=phone_number,
-        )
+                service_code=service_code,
+                user_input=user_input,
+                current_menu=current_menu.get('name'),
+                session_data=last_ussd_session.session_data
+            )
+        else:
+            ussd_session = create_or_update_session(
+                external_session_id=external_session_id,
+                phone=phone_number,
+                service_code=service_code,
+                user_input=user_input,
+                current_menu=current_menu.get('name')
+            )
 
         # define appropriate response
         response = custom_display_text(

apps/cic-ussd/cic_ussd/processor.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 # third party imports
 import celery
-from cic_types.models.person import Person
+from sqlalchemy import desc
 from tinydb.table import Document
 
 # local imports
@@ -315,6 +315,16 @@ def process_start_menu(display_key: str, user: User):
     )
 
 
+def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession:
+    # get last ussd session based on user phone number
+    last_ussd_session = UssdSession.session\
+        .query(UssdSession)\
+        .filter_by(msisdn=phone_number)\
+        .order_by(desc(UssdSession.created))\
+        .first()
+    return last_ussd_session
+
+
 def process_request(user_input: str, user: User, ussd_session: Optional[dict] = None) -> Document:
     """This function assesses a request based on the user from the request comes, the session_id and the user's
     input. It determines whether the request translates to a return to an existing session by checking whether the
@@ -337,7 +347,23 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] =
             return UssdMenu.find_by_name(name=successive_state)
     else:
         if user.has_valid_pin():
-            return UssdMenu.find_by_name(name='start')
+            last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
+
+            key = create_cached_data_key(
+                identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
+                salt='cic.person'
+            )
+            user_metadata = get_cached_data(key=key)
+
+            if last_ussd_session:
+                # get last state
+                last_state = last_ussd_session.state
+                logg.debug(f'LAST USSD SESSION STATE: {last_state}')
+                # if last state is account_creation_prompt and metadata exists, show start menu
+                if last_state == 'account_creation_prompt' and user_metadata is not None:
+                    return UssdMenu.find_by_name(name='start')
+                else:
+                    return UssdMenu.find_by_name(name=last_state)
         else:
             if user.failed_pin_attempts >= 3 and user.get_account_status() == AccountStatus.LOCKED.name:
                 return UssdMenu.find_by_name(name='exit_pin_blocked')

apps/cic-ussd/docker/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.8.5-alpine
+# FROM python:3.8.5-alpine
+FROM python:3.8.6-slim-buster
 
 # set working directory
 WORKDIR /usr/src
@@ -6,10 +7,8 @@ WORKDIR /usr/src
 # add args for installing from self-hosted packages
 ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
 
-# add alpine sys packages
+RUN apt-get update && \
-RUN apk update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-        apk add git linux-headers postgresql-dev gnupg bash
-RUN apk add --update musl-dev gcc libffi-dev
 
 # create secrets directory
 RUN mkdir -vp pgp/keys

apps/cic-ussd/requirements.txt

@@ -1,21 +1,49 @@
-cic_base[full_graph]~=0.1.1a24
+#cic_base[full_graph]~=0.1.1a24
+africastalking==1.2.3
 alembic==1.4.2
+amqp==2.6.1
+attrs==20.2.0
 bcrypt==3.2.0
+betterpath==0.2.2
+billiard==3.6.3.0
 celery==4.4.7
-chainlib~=0.0.1a20
+cffi==1.14.3
-cic-eth~=0.10.0a40
+cic-eth~=0.10.0a46
-cic-notify~=0.4.0a2
+cic-notify~=0.4.0a3
-cic-types==0.1.0a8
+cic-types~=0.1.0a8
+click==7.1.2
 confini~=0.3.6rc3
 cryptography==3.2.1
 faker==4.17.1
+iniconfig==1.1.1
+kombu==4.6.11
+Mako==1.1.3
+MarkupSafe==1.1.1
+mirakuru==2.3.0
+more-itertools==8.5.0
+packaging==20.4
 phonenumbers==8.12.12
+pluggy==0.13.1
+port-for==0.4
+psutil==5.7.3
 psycopg2==2.8.6
+py==1.9.0
+pycparser==2.20
+pyparsing==2.4.7
+python-dateutil==2.8.1
+python-editor==1.0.4
 python-gnupg==0.4.6
 python-i18n==0.3.9
+pytz==2020.1
+PyYAML==5.3.1
 redis==3.5.3
 semver==2.13.0
+six==1.15.0
 SQLAlchemy==1.3.20
 tinydb==4.2.0
+toml==0.10.1
 transitions==0.8.4
 uWSGI==2.0.19.1
+vcversioner==2.16.0.0
+vine==1.3.0
+zope.interface==5.1.2

apps/contract-migration/reset.sh

@@ -19,6 +19,12 @@ echo \n
 # pushd /usr/src
 
 init_level_file=${CIC_DATA_DIR}/.init
+if [ ! -f ${CIC_DATA_DIR}/.init ]; then
+  echo "Creating .init file..."
+  mkdir -p $CIC_DATA_DIR
+  touch /tmp/cic/config/.init
+#   touch $init_level_file
+fi
 echo -n 1 > $init_level_file
 
 # Abort on any error (including if wait-for-it fails).
@@ -74,10 +80,10 @@ mkdir -p $CIC_DATA_DIR
 cat << EOF > $CIC_DATA_DIR/.env
 export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
 export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
-
+export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
 EOF
 
-cat $CIC_DATA_DIR/envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
+cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
 # popd
 
 set +a

ci_templates/.cic-template.yml

@@ -1,54 +1,35 @@
-image: docker:19.03.13
+image:
+  name: gcr.io/kaniko-project/executor:debug
+  entrypoint: [""]
 
 variables:
-    # docker host
+  KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
-    DOCKER_HOST: tcp://docker:2376
+  CONTEXT: $CI_PROJECT_DIR/apps/
-    # container, thanks to volume mount from config.toml
-    DOCKER_TLS_CERTDIR: "/certs"
-    # These are usually specified by the entrypoint, however the
-    # Kubernetes executor doesn't run entrypoints
-    # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125
-    DOCKER_TLS_VERIFY: 1
-    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-    # We are building these from the apps dir to easily share the requirements file there.
-    # It would be nicer to build from the app dir context. TODO figure out a nice way to do this in local DOCKER_TLS_VERIFY
-    CONTEXT: apps/
-
-services:
-  - docker:19.03.13-dind
-
-before_script:
-  - docker info
 
 .py_build_merge_request:
   stage: build
-  before_script:
-    - cd $CONTEXT
   variables:
-    CI_DEBUG_TRACE: "true"
+    - CI_DEBUG_TRACE: "true"
-    IMAGE_TAG: $APP_NAME:$CI_COMMIT_SHORT_SHA
   script:
-    - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
 
 .py_build_push:
-    stage: build
+  stage: build
-    before_script:
+  variables:
-      - cd $CONTEXT
+      IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
-      - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" $CI_REGISTRY --password-stdin
+      LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
-    variables:
+  script:
-        CI_DEBUG_TRACE: "true"
+    - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
-        IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
+    - mkdir -p /kaniko/.docker
-        LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    script:
+    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
-      - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
-      - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
+  rules:
-      - docker push $IMAGE_TAG
-      - docker tag $IMAGE_TAG $LATEST_TAG
-      - docker push $LATEST_TAG
-    rules:
       - if: $CI_COMMIT_BRANCH == "master"
         when: always
 

docker-compose.yml

@@ -36,7 +36,7 @@ services:
     restart: unless-stopped
     ports:
       - ${DEV_ETH_PORT_HTTP:-63545}:8545
-      - ${DEV_ETH_PORT_WS-63546}:8546
+      - ${DEV_ETH_PORT_WS:-63546}:8546
       - 30303
     volumes:
       - ./apps/bloxbergValidatorSetup/keys:/root/keys # stores the signing key locally
@@ -54,6 +54,7 @@ services:
     volumes:
       - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql 
       - ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh
+      - ./apps/cic-cache/db/psycopg2/db.sql:/docker-entrypoint-initdb.d/3-init-cic-meta.sql
       - postgres-db:/var/lib/postgresql/data
 
   redis:
@@ -79,24 +80,7 @@ services:
     build:
       context: apps/
       dockerfile: contract-migration/docker/Dockerfile
-    environment:
+    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
-      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
-      ETH_PROVIDER: http://eth:8545
-      # And these two are for wait-for-it (could parse this)
-      ETH_PROVIDER_HOST: eth 
-      ETH_PROVIDER_PORT: 8545
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
-      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
-    command: ["./reset.sh"]
-    depends_on:
-      - eth
-    volumes:
-      - contract-config:/tmp/cic/config
-
-  seed-cic-eth:
-    build:
-      context: apps/
-      dockerfile: contract-migration/docker/Dockerfile
     environment:
       # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
       ETH_PROVIDER: http://eth:8545
@@ -117,10 +101,8 @@ services:
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
-    command: ["./seed_cic_eth.sh"]
+      RUN_LEVEL: 2 #0: noop 1: contract migrations 2: seed data
-      #  deploy:
+    command: ["./run_job.sh"]
-      #restart_policy:
-      #  condition: on-failure
     depends_on:
       - eth
       - postgres
@@ -129,7 +111,6 @@ services:
     volumes:
       - contract-config:/tmp/cic/config
 
-
   cic-cache-tracker:
     build:
       context: apps
@@ -162,7 +143,43 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        /usr/local/bin/cic-cache-tracker -vv
+        /usr/local/bin/cic-cache-trackerd -vv
+    volumes:
+      - contract-config:/tmp/cic/config/:ro
+
+  cic-cache-tasker:
+    build:
+      context: apps
+      dockerfile: cic-cache/docker/Dockerfile
+    environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
+      ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
+      DATABASE_USER: ${DATABASE_USER:-grassroots}
+      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
+      DATABASE_HOST: ${DATABASE_HOST:-postgres}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
+      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
+      DATABASE_DEBUG: 1
+      ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
+      CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
+      CELERY_BROKER_URL: redis://redis:6379
+      CELERY_RESULT_URL: redis://redis:6379
+    deploy: 
+      restart_policy:
+        condition: on-failure
+    depends_on:
+      - redis
+      - postgres
+      - eth
+    command:
+      - /bin/bash
+      - -c
+      - |
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
+        /usr/local/bin/cic-cache-taskerd -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -192,7 +209,7 @@ services:
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         "/usr/local/bin/uwsgi" \
-        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \
+        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \
         --http :8000 \
         --pyargv -vv