Update .gitlab-ci.yml, ci_templates/.cic-template.yml, apps/cic-eth/docker/Dockerfile, apps/cic-eth/.gitlab-ci.yml files

.gitlab-ci.yml

@@ -1,12 +1,12 @@
 include:
   - local: 'ci_templates/.cic-template.yml'
-  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  # - local: 'apps/contract-migration/.gitlab-ci.yml'
   - local: 'apps/cic-eth/.gitlab-ci.yml'
-  - local: 'apps/cic-ussd/.gitlab-ci.yml'
+  # - local: 'apps/cic-ussd/.gitlab-ci.yml'
-  - local: 'apps/cic-notify/.gitlab-ci.yml'
+  # - local: 'apps/cic-notify/.gitlab-ci.yml'
-  - local: 'apps/cic-meta/.gitlab-ci.yml'
+  # - local: 'apps/cic-meta/.gitlab-ci.yml'
-  - local: 'apps/cic-cache/.gitlab-ci.yml'
+  # - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
+  # - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build

apps/cic-eth/.gitlab-ci.yml

@@ -14,16 +14,17 @@ build-mr-cic-eth:
     extends:
         - .cic_eth_variables
         - .cic_eth_changes_target
-        - .py_build_target_test
+        - .py_build_merge_request
 
 test-mr-cic-eth:
+    stage: test
     extends:
         - .cic_eth_variables
         - .cic_eth_changes_target
-    stage: test
+    image: $IMAGE_TAG
-    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
     script:
         - cd apps/$APP_NAME/
+        - pip install -r test_requirements.txt
         - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
     needs: ["build-mr-cic-eth"]
 

apps/cic-eth/docker/Dockerfile

@@ -1,45 +1,29 @@
-FROM python:3.8.6-slim-buster as compile
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
 WORKDIR /usr/src/cic-eth
 
-RUN apt-get update && \
+# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+# do load these all into setup.py later
+# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+#COPY cic-eth/requirements.txt .
 
-#RUN python -m venv venv && . venv/bin/activate
+#ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+#ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#    pip install --index-url https://pypi.org/simple \
+#    --force-reinstall \
+#    --trusted-host $pip_trusted_host \
+#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+#    -r requirements.txt 
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-RUN /usr/local/bin/python -m pip install --upgrade pip
-RUN pip install semver
-
-# TODO use a packaging style that lets us copy requirments only ie. pip-tools 
 COPY cic-eth/ .
-RUN	pip install $pip_extra_index_url_flag .
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple \
+    --force-reinstall \
+    --trusted-host $pip_trusted_host \
+	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL .
 
-# --- TEST IMAGE ---
+ENTRYPOINT: []
-FROM python:3.8.6-slim-buster as test 
-
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-WORKDIR /usr/src/cic-eth
-
-RUN /usr/local/bin/python -m pip install --upgrade pip
-
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-# TODO we could use venv inside container to isolate the system and app deps further
-# COPY --from=compile /usr/src/cic-eth/ .
-# RUN . venv/bin/activate
-
-COPY cic-eth/test_requirements.txt .
-RUN	pip install $pip_extra_index_url_flag -r test_requirements.txt 
-
-COPY cic-eth . 
-
-ENV PYTHONPATH .
-
-ENTRYPOINT ["pytest"]
 
 # --- RUNTIME ---
 FROM python:3.8.6-slim-buster as runtime 
@@ -49,8 +33,8 @@ RUN apt-get update && \
 
 WORKDIR /usr/src/cic-eth
 
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=dev /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
                           /usr/local/lib/python3.8/site-packages/
 
 COPY cic-eth/docker/* ./ 

ci_templates/.cic-template.yml

@@ -10,25 +10,26 @@ variables:
   stage: build
   variables:
     CI_DEBUG_TRACE: "true"
+    IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA
   script:
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --destination IMAGE_TAG
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
 
-.py_build_target_test:
+# .py_build_target_test:
-  stage: build
+#   stage: build
-  variables:
+#   variables:
-    CI_DEBUG_TRACE: "true"
+#     CI_DEBUG_TRACE: "true"
-  script:
+#   script:
-    - mkdir -p /kaniko/.docker
+#     - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+#     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --tarPath $APP_NAME-test-image.tar --destination $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
+#     - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --tarPath $APP_NAME-test-image.tar --destination $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
-  rules:
+#   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+#     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: always
+#       when: always
 
 .py_build_push:
   stage: build