Merge branch 'bvander/docker-vm-builds' into 'master'

docker vm builds

See merge request grassrootseconomics/cic-internal-integration!259

.gitlab-ci.yml

@@ -1,14 +1,36 @@
 include:
-  - local: 'ci_templates/.cic-template.yml'
+  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
-  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  # these includes are app specific unit tests
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
+    #- local: 'apps/contract-migration/.gitlab-ci.yml'
+    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build
   - test
-  - release
+  - deploy 
+
+image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest  
+
+variables:
+  DOCKER_BUILDKIT: "1"
+  COMPOSE_DOCKER_CLI_BUILD: "1" 
+  CI_DEBUG_TRACE: "true"
+
+before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+# runs on protected branches and pushes to repo
+build-push:
+  stage: build
+  tags:
+    - integration
+  script:
+    - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always

apps/cic-base-os/Dockerfile

@@ -1,34 +0,0 @@
-# The solc image messes up the alpine environment, so we have to go all over again
-FROM python:3.8.6-slim-buster
-
-LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
-LABEL spdx-license-identifier="GPL-3.0-or-later"
-LABEL description="Base layer for buiding development images for the cic component suite"
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
-
-
-RUN echo installing nodejs tooling
-
-COPY ./dev/nvm.sh /root/
-
-# Install nvm with node and npm
-# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
-ENV NVM_DIR /root/.nvm
-ENV NODE_VERSION 15.3.0
-ENV BANCOR_NODE_VERSION 10.16.0
-
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
-	&& . $NVM_DIR/nvm.sh \
-	&& nvm install $NODE_VERSION \
-	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION \
-# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
-# owner of these files is "1001" by default - wtf
-	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
-ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
-

apps/cic-base-os/README.md

@@ -1 +0,0 @@
-## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-cache/.gitlab-ci.yml

@@ -1,52 +1,17 @@
-.cic_cache_variables:
+build-test-cic-cache:
+  stage: test 
+  tags:
+    - integration
   variables:
     APP_NAME: cic-cache
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-cache:
-    extends:
-        - .py_build_merge_request
-        - .cic_cache_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-
-test-mr-cic-cache:
-    stage: test
-    extends:
-        - .cic_cache_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
   script:
-        - cd apps/$APP_NAME/
+    - cd apps/cic-cache
-        - >
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+  allow_failure: true
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-cache"]
   rules:
   - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     changes:
         - apps/$APP_NAME/**/*
     when: always
-
-build-push-cic-cache:
-    extends:
-        - .py_build_push
-        - .cic_cache_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-        
-

apps/cic-cache/README.md

@@ -0,0 +1 @@
+# CIC-CACHE

apps/cic-cache/docker/Dockerfile_ci

@@ -1,38 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+

apps/cic-eth/.gitlab-ci.yml

@@ -1,52 +1,16 @@
-.cic_eth_variables:
+build-test-cic-eth:
+  stage: test
+  tags:
+    - integration
   variables:
     APP_NAME: cic-eth
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-eth:
-    extends:
-        - .cic_eth_variables
-        - .py_build_target_dev
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
-
-test-mr-cic-eth:
-    stage: test
-    extends:
-        - .cic_eth_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
   script:
-        - cd apps/$APP_NAME/
+    - cd apps/cic-eth
-        - >
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+      #rules:
-          -r admin_requirements.txt
+      #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
-          -r services_requirements.txt
+      #  changes:
-          -r test_requirements.txt 
+      #      - apps/$APP_NAME/**/*
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+      #  when: always
-    needs: ["build-mr-cic-eth"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
-
-build-push-cic-eth:
-    extends:
-        - .py_build_push
-        - .cic_eth_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-eth/**/*
-      when: always

apps/cic-eth/docker/Dockerfile_ci

@@ -1,71 +0,0 @@
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-WORKDIR /usr/src/cic-eth
-
-# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
-# do load these all into setup.py later
-# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
-#COPY cic-eth/requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
-#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-#    pip install --index-url https://pypi.org/simple \
-#    --force-reinstall \
-#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-#    -r requirements.txt 
-COPY *requirements.txt . 
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
-    -r requirements.txt \
-    -r services_requirements.txt \
-    -r admin_requirements.txt
-    
-COPY . .
-RUN python setup.py install
-
-COPY docker/entrypoints/* ./ 
-RUN chmod 755 *.sh
-
-# # ini files in config directory defines the configurable parameters for the application
-# # they can all be overridden by environment variables
-# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-ENTRYPOINT []
-
-# ------------------ PRODUCTION CONTAINER ----------------------
-#FROM python:3.8.6-slim-buster as prod 
-#
-#RUN apt-get update && \
-#	apt install -y gnupg libpq-dev procps
-#
-#WORKDIR /root
-#
-#COPY --from=dev /usr/local/bin/ /usr/local/bin/
-#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
-#                          /usr/local/lib/python3.8/site-packages/
-#
-#COPY docker/entrypoints/* ./ 
-#RUN chmod 755 *.sh
-#
-## # ini files in config directory defines the configurable parameters for the application
-## # they can all be overridden by environment variables
-## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-eth/
-#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-#COPY scripts/ scripts/ 
-#
-## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-##COPY util/liveness/health.sh /usr/local/bin/health.sh
-#
-#ENTRYPOINT []
-#

apps/cic-eth/docker/run_tests.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r admin_requirements.txt \
+-r services_requirements.txt \
+-r test_requirements.txt  
+
+export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+

apps/cic-eth/tests/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+-r admin_requirements.txt
+-r services_requirements.txt
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests

apps/cic-meta/.gitlab-ci.yml

@@ -1,43 +1,16 @@
-
+build-test-cic-meta:
-.cic_meta_variables:
+  stage: test
+  tags:
+   - integration
   variables:
     APP_NAME: cic-meta
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-meta:
-    extends:
-        - .py_build_merge_request
-        - .cic_meta_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-meta/**/*
-      when: always
-
-test-mr-cic-meta:
-    extends:
-        - .cic_meta_variables
-    stage: test
-    image: $MR_IMAGE_TAG 
   script:
-        - cd /root
+    - cd apps/cic-meta
-        - npm install --dev 
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-        - npm run test
+    - docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh
-        - npm run test:coverage
+     #rules:
-    needs: ["build-mr-cic-meta"]
+     #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    rules:
+     #  changes:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+     #      - apps/$APP_NAME/**/*
-      changes:
+     #  when: always
-          - apps/cic-meta/**/*
-      when: always
-
-build-push-cic-meta:
-    extends:
-        - .py_build_push
-        - .cic_meta_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-meta/**/*
-      when: always

apps/cic-meta/docker/Dockerfile

@@ -15,11 +15,10 @@ RUN --mount=type=cache,mode=0755,target=/root/.npm \
 COPY webpack.config.js .
 COPY tsconfig.json . 
 ## required to build the cic-client-meta module
-COPY src/ src/
+COPY . .
-COPY scripts/ scripts/
-COPY tests/ tests/
 COPY tests/*.asc /root/pgp/
 
+
 ## copy runtime configs
 COPY .config/ /usr/local/etc/cic-meta/
 #

apps/cic-meta/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-#FROM node:15.3.0-alpine3.10
-FROM node:lts-alpine3.14
-
-WORKDIR /root
-
-RUN apk add --no-cache postgresql bash
-
-# copy the dependencies
-COPY package.json package-lock.json .
-RUN npm set cache /root/.npm && \
-    npm ci
-
-COPY webpack.config.js .
-COPY tsconfig.json . 
-## required to build the cic-client-meta module
-COPY src/ src/
-COPY scripts/ scripts/
-COPY tests/ tests/
-COPY tests/*.asc /root/pgp/
-
-## copy runtime configs
-COPY .config/ /usr/local/etc/cic-meta/
-#
-## db migrations
-COPY docker/db.sh ./db.sh
-RUN chmod 755 ./db.sh
-#
-RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY docker/start_server.sh ./start_server.sh
-RUN chmod 755 ./start_server.sh
-ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/run_tests.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+set -e
+
+npm install --dev 
+npm run test
+npm run test:coverage

apps/cic-notify/.gitlab-ci.yml

@@ -1,52 +1,17 @@
-.cic_notify_variables:
+build-test-cic-notify:
+  stage: test
+  tags:
+    - integration
   variables:
     APP_NAME: cic-notify
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-notify:
-    extends:
-        - .py_build_merge_request
-        - .cic_notify_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-notify/**/*
-      when: always
-
-test-mr-cic-notify:
-    stage: test
-    extends:
-        - .cic_notify_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
   script:
-        - cd apps/$APP_NAME/
+    - cd apps/cic-notify
-        - >
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+  allow_failure: true
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-notify"]
   rules:
   - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     changes:
         - apps/$APP_NAME/**/*
     when: always
-
-build-push-cic-notify:
-    extends:
-        - .py_build_push
-        - .cic_notify_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-notify/**/*
-      when: always
-        
-

apps/cic-notify/docker/Dockerfile_ci

@@ -1,27 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-COPY requirements.txt .
-
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . .
-
-RUN python setup.py install
-
-COPY docker/*.sh .
-RUN chmod +x *.sh 
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY .config/ /usr/local/etc/cic-notify/
-COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
-
-ENTRYPOINT []

apps/cic-notify/docker/run_tests.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests

apps/cic-ussd/.gitlab-ci.yml

@@ -1,52 +1,16 @@
-.cic_ussd_variables:
+build-test-cic-ussd:
+  stage: test
+  tags:
+  - integration
   variables:
     APP_NAME: cic-ussd
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-ussd:
-    extends:
-        - .py_build_merge_request
-        - .cic_ussd_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-ussd/**/*
-      when: always
-
-test-mr-cic-ussd:
-    stage: test
-    extends:
-        - .cic_ussd_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
   script:
-        - cd apps/$APP_NAME/
+    - cd apps/cic-ussd
-        - >
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd
-    needs: ["build-mr-cic-ussd"]
   rules:
   - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     changes:
         - apps/$APP_NAME/**/*
     when: always
-
-build-push-cic-ussd:
-    extends:
-        - .py_build_push
-        - .cic_ussd_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-ussd/**/*
-      when: always
-
-

apps/cic-ussd/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-RUN apt-get install -y redis-server
-
-# create secrets directory
-RUN mkdir -vp pgp/keys
-
-# create application directory
-RUN mkdir -vp cic-ussd
-RUN mkdir -vp data
-
-COPY requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . .
-RUN python setup.py install
-
-COPY cic_ussd/db/ussd_menu.json data/
-
-COPY docker/*.sh .
-RUN chmod +x /root/*.sh
-
-# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY config/ /usr/local/etc/cic-ussd/
-COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
-
-ENTRYPOINT []

apps/cic-ussd/docker/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd

apps/contract-migration/.gitlab-ci.yml

@@ -1,25 +1,25 @@
-.contract_migration_variables:
+#.contract_migration_variables:
-    variables:
+#    variables:
-        APP_NAME: contract-migration
+#        APP_NAME: contract-migration
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+#        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
+#        CONTEXT: apps/$APP_NAME
-
+#
-build-mr-contract-migration:
+#build-mr-contract-migration:
-    extends:
+#    extends:
-        - .py_build_merge_request
+#        - .py_build_merge_request
-        - .contract_migration_variables
+#        - .contract_migration_variables
-    rules:
+#    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+#    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
+#      changes:
-          - apps/contract-migration/**/*
+#          - apps/contract-migration/**/*
-      when: always
+#      when: always
-
+#
-build-push-contract-migration:
+#build-push-contract-migration:
-    extends:
+#    extends:
-        - .py_build_push
+#        - .py_build_push
-        - .contract_migration_variables
+#        - .contract_migration_variables
-    rules:
+#    rules:
-    - if: $CI_COMMIT_BRANCH == "master"
+#    - if: $CI_COMMIT_BRANCH == "master"
-      changes:
+#      changes:
-          - apps/contract-migration/**/*
+#          - apps/contract-migration/**/*
-      when: always
+#      when: always

docker-compose.yml

@@ -13,20 +13,8 @@ networks:
     name: cic-network
 
 services:
-  # eth:
-  #   image: trufflesuite/ganache-cli
-  #   ports:
-  #     - ${HTTP_PORT_ETH:-8545}
-  #     - ${WS_PORT_ETH:-8546}
-  #   # Note! -e switch doesnt work, whatever you  put there, it will be 100
-  #   command: "-i 8996 -e 1000 -l 90000000 \
-  #           -m '${DEV_MNEMONIC:-\"history stumble mystery avoid embark arrive mom foil pledge keep grain dice\"}' \
-  #           -v --db /tmp/cic/ganache/ganache.db \
-  #           --noVMErrorsOnRPCResponse --allowUnlimitedContractSize"
-  #   volumes:
-  #     - ganache-db:/tmp/cic/ganache
-
   eth:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/bloxberg-node:${TAG:-latest}
     build:
       context: apps/bloxbergValidatorSetup
     restart: unless-stopped
@@ -71,6 +59,7 @@ services:
       - bee-data:/tmp/cic/bee
 
   contract-migration:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest}
     profiles:
       - migrations 
     build:
@@ -129,6 +118,7 @@ services:
       - contract-config:/tmp/cic/config
 
   cic-cache-tracker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
     profiles:
       - cache 
     build:
@@ -170,6 +160,7 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
     profiles:
       - cache 
     build:
@@ -210,6 +201,7 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
     profiles:
       - cache 
     build:
@@ -245,6 +237,7 @@ services:
 
 
   cic-eth-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build: 
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -298,6 +291,7 @@ services:
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -342,6 +336,7 @@ services:
 
 
   cic-eth-dispatcher:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -386,6 +381,7 @@ services:
 
 
   cic-eth-retrier:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
     build:
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -433,6 +429,7 @@ services:
 
 
   cic-notify-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:${TAG:-latest}
     build:
       context: apps/cic-notify
       dockerfile: docker/Dockerfile
@@ -461,6 +458,7 @@ services:
 
 
   cic-meta-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest}
     profiles:
       - custodial-meta
     hostname: meta
@@ -496,6 +494,7 @@ services:
     # command: "/root/start_server.sh -vv"
 
   cic-user-ussd-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
     profiles:
       - custodial-ussd
     build:
@@ -528,6 +527,7 @@ services:
     command: "/root/start_cic_user_ussd_server.sh -vv"
 
   cic-user-server:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
     profiles:
       - custodial-ussd
     build:
@@ -553,6 +553,7 @@ services:
     command: "/root/start_cic_user_server.sh -vv"
 
   cic-user-tasker:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
     profiles:
       - custodial-ussd
     build:

scripts/build-push.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env sh
+
+# Exit in case of error
+set -e
+
+TAG=${TAG?Variable not set} \
+sh ./scripts/build.sh
+
+docker-compose -f docker-compose.yml push

scripts/build.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env sh
+
+# Exit in case of error
+set -e
+
+TAG=${TAG?Variable not set} \
+docker-compose \
+-f docker-compose.yml \
+build

scripts/test-local.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+# Exit in case of error
+set -e
+
+docker-compose down -v --remove-orphans # Remove possibly previous broken stacks left hanging after an error
+
+if [ $(uname -s) = "Linux" ]; then
+    echo "Remove __pycache__ files"
+    sudo find . -type d -name __pycache__ -exec rm -r {} \+
+fi
+
+docker-compose build
+docker-compose up -d
+docker-compose exec -T backend bash /app/tests-start.sh "$@"