Merge remote-tracking branch 'origin/master' into lash/cli-rehabilitations

apps/cic-cache/setup.cfg

@@ -25,7 +25,6 @@ licence_files =
 python_requires = >= 3.6
 packages = 
 	cic_cache
-	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
 	cic_cache.runnable
@@ -34,6 +33,5 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-cache-trackerd = cic_cache.runnable.tracker:main
-	cic-cache-serverd = cic_cache.runnable.server:main
-	cic-cache-taskerd = cic_cache.runnable.tasker:main
+	cic-cache-tracker = cic_cache.runnable.tracker:main
+	cic-cache-server = cic_cache.runnable.server:main

apps/cic-ussd/requirements.txt

@@ -1,47 +1,21 @@
-cic-types==0.1.0a8
+cic_base[full-graph]~=0.1.1a17
 alembic==1.4.2
-amqp==2.6.1
-attrs==20.2.0
 bcrypt==3.2.0
-betterpath==0.2.2
-billiard==3.6.3.0
 celery==4.4.7
-cffi==1.14.3
-cic-eth~=0.10.0a22
-cic-notify==0.3.1
-click==7.1.2
-confini~=0.3.6a1
+chainlib~=0.0.1a20
+cic-eth~=0.10.0a40
+cic-notify~=0.4.0a2
+cic-types==0.1.0a8
+confini~=0.3.6rc3
 cryptography==3.2.1
 faker==4.17.1
-iniconfig==1.1.1
-kombu==4.6.11
-Mako==1.1.3
-MarkupSafe==1.1.1
-mirakuru==2.3.0
-more-itertools==8.5.0
-packaging==20.4
 phonenumbers==8.12.12
-pluggy==0.13.1
-port-for==0.4
-psutil==5.7.3
 psycopg2==2.8.6
-py==1.9.0
-pycparser==2.20
-pyparsing==2.4.7
-python-dateutil==2.8.1
-python-editor==1.0.4
 python-gnupg==0.4.6
 python-i18n==0.3.9
-pytz==2020.1
-PyYAML==5.3.1
 redis==3.5.3
 semver==2.13.0
-six==1.15.0
 SQLAlchemy==1.3.20
 tinydb==4.2.0
-toml==0.10.1
 transitions==0.8.4
 uWSGI==2.0.19.1
-vcversioner==2.16.0.0
-vine==1.3.0
-zope.interface==5.1.2

apps/contract-migration/docker/Dockerfile

@@ -127,6 +127,5 @@ COPY contract-migration/from_env.sh from_env.sh
 COPY contract-migration/seed_cic_eth.sh seed_cic_eth.sh
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
-COPY contract-migration/envlist .
 
-# ENTRYPOINT [ "/bin/bash" ]
+ENTRYPOINT [ "/bin/bash" ]

apps/contract-migration/envlist

@@ -1,61 +0,0 @@
-SYNCER_LOOP_INTERVAL
-SSL_ENABLE_CLIENT
-SSL_CERT_FILE
-SSL_KEY_FILE
-SSL_PASSWORD
-SSL_CA_FILE
-BANCOR_DIR
-REDIS_HOST
-REDIS_PORT
-REDIS_DB
-PGP_EXPORTS_DIR
-PGP_PRIVATEKEY_FILE
-PGP_PASSPHRASE
-DATABASE_USER
-DATABASE_PASSWORD
-DATABASE_NAME
-DATABASE_HOST
-DATABASE_PORT
-DATABASE_ENGINE
-DATABASE_DRIVER
-DATABASE_DEBUG
-TASKS_AFRICASTALKING
-TASKS_SMS_DB
-TASKS_LOG
-TASKS_TRACE_QUEUE_STATUS
-TASKS_TRANSFER_CALLBACKS
-DEV_MNEMONIC
-DEV_ETH_RESERVE_ADDRESS
-DEV_ETH_ACCOUNTS_INDEX_ADDRESS
-DEV_ETH_RESERVE_AMOUNT
-DEV_ETH_ACCOUNT_BANCOR_DEPLOYER
-DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
-DEV_ETH_ACCOUNT_GAS_PROVIDER
-DEV_ETH_ACCOUNT_RESERVE_OWNER
-DEV_ETH_ACCOUNT_RESERVE_MINTER
-DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER
-DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
-DEV_ETH_ACCOUNT_SARAFU_OWNER
-DEV_ETH_ACCOUNT_SARAFU_GIFTER
-DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
-DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
-DEV_ETH_SARAFU_TOKEN_NAME
-DEV_ETH_SARAFU_TOKEN_SYMBOL
-DEV_ETH_SARAFU_TOKEN_DECIMALS
-DEV_ETH_SARAFU_TOKEN_ADDRESS
-DEV_PGP_PUBLICKEYS_ACTIVE_FILE
-DEV_PGP_PUBLICKEYS_TRUSTED_FILE
-DEV_PGP_PUBLICKEYS_ENCRYPT_FILE
-CIC_REGISTRY_ADDRESS
-CIC_APPROVAL_ESCROW_ADDRESS
-CIC_TOKEN_INDEX_ADDRESS
-CIC_ACCOUNTS_INDEX_ADDRESS
-CIC_DECLARATOR_ADDRESS
-CIC_CHAIN_SPEC
-ETH_PROVIDER
-ETH_ABI_DIR
-SIGNER_SOCKET_PATH
-SIGNER_SECRET
-CELERY_BROKER_URL
-CELERY_RESULT_URL
-META_PROVIDER

apps/contract-migration/reset.sh

@@ -64,7 +64,7 @@ export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
 EOF
 
-cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
+cat $CIC_DATA_DIR/envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
 # popd
 
 set +a

ci_templates/.cic-template.yml

@@ -1,35 +1,54 @@
-image:
-  name: gcr.io/kaniko-project/executor:debug
-  entrypoint: [""]
+image: docker:19.03.13
 
 variables:
-  KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
-  CONTEXT: $CI_PROJECT_DIR/apps/
+    # docker host
+    DOCKER_HOST: tcp://docker:2376
+    # container, thanks to volume mount from config.toml
+    DOCKER_TLS_CERTDIR: "/certs"
+    # These are usually specified by the entrypoint, however the
+    # Kubernetes executor doesn't run entrypoints
+    # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125
+    DOCKER_TLS_VERIFY: 1
+    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+    # We are building these from the apps dir to easily share the requirements file there.
+    # It would be nicer to build from the app dir context. TODO figure out a nice way to do this in local DOCKER_TLS_VERIFY
+    CONTEXT: apps/
+
+services:
+  - docker:19.03.13-dind
+
+before_script:
+  - docker info
 
 .py_build_merge_request:
   stage: build
+  before_script:
+    - cd $CONTEXT
   variables:
-    - CI_DEBUG_TRACE: "true"
+    CI_DEBUG_TRACE: "true"
+    IMAGE_TAG: $APP_NAME:$CI_COMMIT_SHORT_SHA
   script:
-    - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push
+    - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
 
 .py_build_push:
-  stage: build
-  variables:
-      IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
-      LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
-  script:
-    - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
-    - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
-  rules:
+    stage: build
+    before_script:
+      - cd $CONTEXT
+      - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" $CI_REGISTRY --password-stdin
+    variables:
+        CI_DEBUG_TRACE: "true"
+        IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
+        LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
+    script:
+      - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
+      - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
+      - docker push $IMAGE_TAG
+      - docker tag $IMAGE_TAG $LATEST_TAG
+      - docker push $LATEST_TAG
+    rules:
       - if: $CI_COMMIT_BRANCH == "master"
         when: always
 

docker-compose.yml

@@ -162,43 +162,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        /usr/local/bin/cic-cache-trackerd -vv
-    volumes:
-      - contract-config:/tmp/cic/config/:ro
-
-  cic-cache-tasker:
-    build:
-      context: apps
-      dockerfile: cic-cache/docker/Dockerfile
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
-      ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
-      DATABASE_USER: ${DATABASE_USER:-grassroots}
-      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
-      DATABASE_HOST: ${DATABASE_HOST:-postgres}
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
-      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
-      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_DEBUG: 1
-      ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
-      CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
-      CELERY_BROKER_URL: redis://redis:6379
-      CELERY_RESULT_URL: redis://redis:6379
-    deploy: 
-      restart_policy:
-        condition: on-failure
-    depends_on:
-      - redis
-      - postgres
-      - eth
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        /usr/local/bin/cic-cache-taskerd -vv
+        /usr/local/bin/cic-cache-tracker -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -228,7 +192,7 @@ services:
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         "/usr/local/bin/uwsgi" \
-        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \
+        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \
         --http :8000 \
         --pyargv -vv