Merge branch 'cic-eth-unittest' into 'master'

Unit tests for cic-eth

See merge request grassrootseconomics/cic-internal-integration!164

apps/cic-eth/.gitlab-ci.yml

@@ -5,18 +5,31 @@
 
 .cic_eth_changes_target:
     rules:
-        - changes:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-            - $CONTEXT/$APP_NAME/*
+      #changes:
+          #- $CONTEXT/$APP_NAME/**/*
+      when: always
 
 build-mr-cic-eth:
     extends:
-        - .cic_eth_changes_target
-        - .py_build_merge_request
         - .cic_eth_variables
+        - .cic_eth_changes_target
+        - .py_build_target_test
+
+test-mr-cic-eth:
+    extends:
+        - .cic_eth_variables
+        - .cic_eth_changes_target
+    stage: test
+    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
+    script:
+        - cd apps/$APP_NAME/
+        - pytest tests/unit/ 
+        - pytest tests/task/
+        - pytest tests/filters/ 
+    needs: ["build-mr-cic-eth"]
 
 build-push-cic-eth:
     extends:
         - .py_build_push
         - .cic_eth_variables
-        
-

apps/cic-eth/docker/Dockerfile

@@ -1,48 +1,62 @@
-# FROM grassrootseconomics:cic
+FROM python:3.8.6-slim-buster as compile
-
-#FROM python:3.8.6-alpine
-FROM python:3.8.6-slim-buster
-
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
 
 WORKDIR /usr/src/cic-eth
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
-
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
-# Copy shared requirements from top of mono-repo
+#RUN python -m venv venv && . venv/bin/activate
-RUN echo "copying root req file:  ${root_requirement_file}"
-#COPY $root_requirement_file .
-#RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
-RUN /usr/local/bin/python -m pip install --upgrade pip
-#RUN git clone https://gitlab.com/grassrootseconomics/cic-base.git && \
-#	cd cic-base && \
-#	git checkout 7ae1f02efc206b13a65873567b0f6d1c3b7f9bc0 && \
-#	python merge_requirements.py | tee merged_requirements.txt 
-#RUN cd cic-base && \
-#	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN  pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-COPY cic-eth/scripts/ scripts/
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-COPY cic-eth/setup.cfg cic-eth/setup.py ./
+RUN /usr/local/bin/python -m pip install --upgrade pip
-COPY cic-eth/cic_eth/ cic_eth/
+RUN pip install semver
-# Copy app specific requirements
+
-COPY cic-eth/requirements.txt .
+# TODO use a packaging style that lets us copy requirments only ie. pip-tools 
-COPY cic-eth/test_requirements.txt .
+COPY cic-eth/ .
 RUN	pip install $pip_extra_index_url_flag .
 
+# --- TEST IMAGE ---
+FROM python:3.8.6-slim-buster as test 
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+WORKDIR /usr/src/cic-eth
+
+RUN /usr/local/bin/python -m pip install --upgrade pip
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+# TODO we could use venv inside container to isolate the system and app deps further
+# COPY --from=compile /usr/src/cic-eth/ .
+# RUN . venv/bin/activate
+
+COPY cic-eth/test_requirements.txt .
+RUN	pip install $pip_extra_index_url_flag -r test_requirements.txt 
+
+COPY cic-eth . 
+
+ENV PYTHONPATH .
+
+ENTRYPOINT ["pytest"]
+
+# --- RUNTIME ---
+FROM python:3.8.6-slim-buster as runtime 
+
+RUN apt-get update && \
+	apt install -y gnupg libpq-dev procps
+
+WORKDIR /usr/src/cic-eth
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+
 COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
-COPY cic-eth/tests/ tests/
 
+COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
@@ -51,3 +65,4 @@ COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
 COPY util/liveness/health.sh /usr/local/bin/health.sh
+

apps/cic-eth/requirements.txt

@@ -22,3 +22,4 @@ sarafu-faucet==0.0.3a3
 erc20-faucet==0.2.1a4
 coincurve==15.0.0
 potaahto~=0.0.1a2
+pycryptodome==3.10.1

apps/cic-eth/setup.py

@@ -11,17 +11,6 @@ while True:
     requirements.append(l.rstrip())
 f.close()
 
-test_requirements = []
-f = open('test_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    test_requirements.append(l.rstrip())
-f.close()
-
-
 setup(
-    install_requires=requirements,
+    install_requires=requirements
-    tests_require=test_requirements,
         )

apps/cic-eth/test_requirements.txt

@@ -4,4 +4,3 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-giftable-erc20-token==0.0.8a9

ci_templates/.cic-template.yml

@@ -9,7 +9,7 @@ variables:
 .py_build_merge_request:
   stage: build
   variables:
-    - CI_DEBUG_TRACE: "true"
+    CI_DEBUG_TRACE: "true"
   script:
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
@@ -18,6 +18,18 @@ variables:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
 
+.py_build_target_test:
+  stage: build
+  variables:
+    CI_DEBUG_TRACE: "true"
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --tarPath $APP_NAME-test-image.tar --destination $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+
 .py_build_push:
   stage: build
   variables: