From 6a68d2ed323692c54b63eb3816f44d6c7eb87cd6 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sat, 28 Aug 2021 16:26:16 +0000 Subject: [PATCH] docker vm builds --- .gitlab-ci.yml | 30 +++++++++-- apps/cic-base-os/Dockerfile | 34 ------------ apps/cic-base-os/README.md | 1 - apps/cic-cache/.gitlab-ci.yml | 69 ++++++------------------- apps/cic-cache/README.md | 1 + apps/cic-cache/docker/Dockerfile_ci | 38 -------------- apps/cic-cache/docker/run_tests.sh | 10 ++++ apps/cic-eth/.gitlab-ci.yml | 68 ++++++------------------ apps/cic-eth/docker/Dockerfile_ci | 71 -------------------------- apps/cic-eth/docker/run_tests.sh | 11 ++++ apps/cic-eth/tests/run_tests.sh | 10 ++++ apps/cic-meta/.gitlab-ci.yml | 59 ++++++--------------- apps/cic-meta/docker/Dockerfile | 5 +- apps/cic-meta/docker/Dockerfile_ci | 32 ------------ apps/cic-meta/docker/run_tests.sh | 7 +++ apps/cic-notify/.gitlab-ci.yml | 69 ++++++------------------- apps/cic-notify/docker/Dockerfile_ci | 27 ---------- apps/cic-notify/docker/run_tests.sh | 9 ++++ apps/cic-ussd/.gitlab-ci.yml | 68 ++++++------------------ apps/cic-ussd/docker/Dockerfile_ci | 32 ------------ apps/cic-ussd/docker/run_tests.sh | 10 ++++ apps/contract-migration/.gitlab-ci.yml | 50 +++++++++--------- docker-compose.yml | 27 +++++----- scripts/build-push.sh | 9 ++++ scripts/build.sh | 9 ++++ scripts/test-local.sh | 15 ++++++ 26 files changed, 240 insertions(+), 531 deletions(-) delete mode 100644 apps/cic-base-os/Dockerfile delete mode 100644 apps/cic-base-os/README.md delete mode 100644 apps/cic-cache/docker/Dockerfile_ci create mode 100644 apps/cic-cache/docker/run_tests.sh delete mode 100644 apps/cic-eth/docker/Dockerfile_ci create mode 100644 apps/cic-eth/docker/run_tests.sh create mode 100644 apps/cic-eth/tests/run_tests.sh delete mode 100644 apps/cic-meta/docker/Dockerfile_ci create mode 100644 apps/cic-meta/docker/run_tests.sh delete mode 100644 apps/cic-notify/docker/Dockerfile_ci create mode 100644 apps/cic-notify/docker/run_tests.sh delete mode 100644 apps/cic-ussd/docker/Dockerfile_ci create mode 100644 apps/cic-ussd/docker/run_tests.sh create mode 100755 scripts/build-push.sh create mode 100755 scripts/build.sh create mode 100755 scripts/test-local.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52f8ffd6..6ddc1394 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,36 @@ include: - - local: 'ci_templates/.cic-template.yml' - - local: 'apps/contract-migration/.gitlab-ci.yml' + #- local: 'ci_templates/.cic-template.yml' #kaniko build templates + # these includes are app specific unit tests - local: 'apps/cic-eth/.gitlab-ci.yml' - local: 'apps/cic-ussd/.gitlab-ci.yml' - local: 'apps/cic-notify/.gitlab-ci.yml' - local: 'apps/cic-meta/.gitlab-ci.yml' - local: 'apps/cic-cache/.gitlab-ci.yml' - - local: 'apps/data-seeding/.gitlab-ci.yml' + #- local: 'apps/contract-migration/.gitlab-ci.yml' + #- local: 'apps/data-seeding/.gitlab-ci.yml' stages: - build - test - - release + - deploy + +image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest + +variables: + DOCKER_BUILDKIT: "1" + COMPOSE_DOCKER_CLI_BUILD: "1" + CI_DEBUG_TRACE: "true" + +before_script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + +# runs on protected branches and pushes to repo +build-push: + stage: build + tags: + - integration + script: + - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh + rules: + - if: $CI_COMMIT_REF_PROTECTED == "true" + when: always diff --git a/apps/cic-base-os/Dockerfile b/apps/cic-base-os/Dockerfile deleted file mode 100644 index 2af84ab5..00000000 --- a/apps/cic-base-os/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# The solc image messes up the alpine environment, so we have to go all over again -FROM python:3.8.6-slim-buster - -LABEL authors="Louis Holbrook 0826EDA1702D1E87C6E2875121D2E7BB88C2A746" -LABEL spdx-license-identifier="GPL-3.0-or-later" -LABEL description="Base layer for buiding development images for the cic component suite" - -RUN apt-get update && \ - apt-get install -y git gcc g++ libpq-dev && \ - apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client - - -RUN echo installing nodejs tooling - -COPY ./dev/nvm.sh /root/ - -# Install nvm with node and npm -# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker -ENV NVM_DIR /root/.nvm -ENV NODE_VERSION 15.3.0 -ENV BANCOR_NODE_VERSION 10.16.0 - -RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \ - && . $NVM_DIR/nvm.sh \ - && nvm install $NODE_VERSION \ - && nvm alias default $NODE_VERSION \ - && nvm use $NODE_VERSION \ -# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve -# owner of these files is "1001" by default - wtf - && chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION" - -ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH - diff --git a/apps/cic-base-os/README.md b/apps/cic-base-os/README.md deleted file mode 100644 index 6551bf40..00000000 --- a/apps/cic-base-os/README.md +++ /dev/null @@ -1 +0,0 @@ -## this is an example base image if we wanted one for all the other apps. Its just OS level things \ No newline at end of file diff --git a/apps/cic-cache/.gitlab-ci.yml b/apps/cic-cache/.gitlab-ci.yml index ea7ec6c0..7a120987 100644 --- a/apps/cic-cache/.gitlab-ci.yml +++ b/apps/cic-cache/.gitlab-ci.yml @@ -1,52 +1,17 @@ -.cic_cache_variables: - variables: - APP_NAME: cic-cache - DOCKERFILE_PATH: docker/Dockerfile_ci - CONTEXT: apps/$APP_NAME - -build-mr-cic-cache: - extends: - - .py_build_merge_request - - .cic_cache_variables - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/cic-cache/**/* - when: always - -test-mr-cic-cache: - stage: test - extends: - - .cic_cache_variables - cache: - key: - files: - - test_requirements.txt - paths: - - /root/.cache/pip - image: $MR_IMAGE_TAG - script: - - cd apps/$APP_NAME/ - - > - pip install --extra-index-url https://pip.grassrootseconomics.net:8433 - --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple - -r test_requirements.txt - - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests - needs: ["build-mr-cic-cache"] - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/$APP_NAME/**/* - when: always - -build-push-cic-cache: - extends: - - .py_build_push - - .cic_cache_variables - rules: - - if: $CI_COMMIT_BRANCH == "master" - changes: - - apps/cic-cache/**/* - when: always - - +build-test-cic-cache: + stage: test + tags: + - integration + variables: + APP_NAME: cic-cache + MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + script: + - cd apps/cic-cache + - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . + - docker run $MR_IMAGE_TAG sh docker/run_tests.sh + allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/$APP_NAME/**/* + when: always diff --git a/apps/cic-cache/README.md b/apps/cic-cache/README.md index e69de29b..3938e1af 100644 --- a/apps/cic-cache/README.md +++ b/apps/cic-cache/README.md @@ -0,0 +1 @@ +# CIC-CACHE diff --git a/apps/cic-cache/docker/Dockerfile_ci b/apps/cic-cache/docker/Dockerfile_ci deleted file mode 100644 index 67ab3c71..00000000 --- a/apps/cic-cache/docker/Dockerfile_ci +++ /dev/null @@ -1,38 +0,0 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev - -# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9 - -COPY requirements.txt . -#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt -#RUN pip install $pip_extra_index_url_flag . -#RUN pip install .[server] - -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -ARG EXTRA_PIP_ARGS="" -RUN pip install --index-url https://pypi.org/simple \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \ - -r requirements.txt - -COPY . . - -RUN python setup.py install - -# ini files in config directory defines the configurable parameters for the application -# they can all be overridden by environment variables -# to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -COPY config/ /usr/local/etc/cic-cache/ - -# for db migrations -RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/ -COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/ - -COPY /docker/start_tracker.sh ./start_tracker.sh -COPY /docker/db.sh ./db.sh -RUN chmod 755 ./*.sh -# Tracker -# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"] -# Server -# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ] -ENTRYPOINT [] diff --git a/apps/cic-cache/docker/run_tests.sh b/apps/cic-cache/docker/run_tests.sh new file mode 100644 index 00000000..62c1db72 --- /dev/null +++ b/apps/cic-cache/docker/run_tests.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +set -e + +pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ +--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ +-r test_requirements.txt + +export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests + diff --git a/apps/cic-eth/.gitlab-ci.yml b/apps/cic-eth/.gitlab-ci.yml index ef2b7462..2fcfde1e 100644 --- a/apps/cic-eth/.gitlab-ci.yml +++ b/apps/cic-eth/.gitlab-ci.yml @@ -1,52 +1,16 @@ -.cic_eth_variables: - variables: - APP_NAME: cic-eth - DOCKERFILE_PATH: docker/Dockerfile_ci - CONTEXT: apps/$APP_NAME - -build-mr-cic-eth: - extends: - - .cic_eth_variables - - .py_build_target_dev - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/cic-eth/**/* - when: always - -test-mr-cic-eth: - stage: test - extends: - - .cic_eth_variables - cache: - key: - files: - - test_requirements.txt - paths: - - /root/.cache/pip - image: $MR_IMAGE_TAG - script: - - cd apps/$APP_NAME/ - - > - pip install --extra-index-url https://pip.grassrootseconomics.net:8433 - --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple - -r admin_requirements.txt - -r services_requirements.txt - -r test_requirements.txt - - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests - needs: ["build-mr-cic-eth"] - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/cic-eth/**/* - when: always - -build-push-cic-eth: - extends: - - .py_build_push - - .cic_eth_variables - rules: - - if: $CI_COMMIT_BRANCH == "master" - changes: - - apps/cic-eth/**/* - when: always +build-test-cic-eth: + stage: test + tags: + - integration + variables: + APP_NAME: cic-eth + MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + script: + - cd apps/cic-eth + - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . + - docker run $MR_IMAGE_TAG sh docker/run_tests.sh + #rules: + #- if: $CI_PIPELINE_SOURCE == "merge_request_event" + # changes: + # - apps/$APP_NAME/**/* + # when: always diff --git a/apps/cic-eth/docker/Dockerfile_ci b/apps/cic-eth/docker/Dockerfile_ci deleted file mode 100644 index e7bcc831..00000000 --- a/apps/cic-eth/docker/Dockerfile_ci +++ /dev/null @@ -1,71 +0,0 @@ -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev - -WORKDIR /usr/src/cic-eth - -# Copy just the requirements and install....this _might_ give docker a hint on caching but we -# do load these all into setup.py later -# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py -#COPY cic-eth/requirements.txt . - -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -ARG EXTRA_PIP_ARGS="" -#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ -# pip install --index-url https://pypi.org/simple \ -# --force-reinstall \ -# --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ -# -r requirements.txt -COPY *requirements.txt . -RUN pip install --index-url https://pypi.org/simple \ - --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL \ - $EXTRA_PIP_ARGS \ - -r requirements.txt \ - -r services_requirements.txt \ - -r admin_requirements.txt - -COPY . . -RUN python setup.py install - -COPY docker/entrypoints/* ./ -RUN chmod 755 *.sh - -# # ini files in config directory defines the configurable parameters for the application -# # they can all be overridden by environment variables -# # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -COPY config/ /usr/local/etc/cic-eth/ -COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ -COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ - -# TODO this kind of code sharing across projects should be discouraged...can we make util a library? -#COPY util/liveness/health.sh /usr/local/bin/health.sh -ENTRYPOINT [] - -# ------------------ PRODUCTION CONTAINER ---------------------- -#FROM python:3.8.6-slim-buster as prod -# -#RUN apt-get update && \ -# apt install -y gnupg libpq-dev procps -# -#WORKDIR /root -# -#COPY --from=dev /usr/local/bin/ /usr/local/bin/ -#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \ -# /usr/local/lib/python3.8/site-packages/ -# -#COPY docker/entrypoints/* ./ -#RUN chmod 755 *.sh -# -## # ini files in config directory defines the configurable parameters for the application -## # they can all be overridden by environment variables -## # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -#COPY config/ /usr/local/etc/cic-eth/ -#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ -#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ -#COPY scripts/ scripts/ -# -## TODO this kind of code sharing across projects should be discouraged...can we make util a library? -##COPY util/liveness/health.sh /usr/local/bin/health.sh -# -#ENTRYPOINT [] -# diff --git a/apps/cic-eth/docker/run_tests.sh b/apps/cic-eth/docker/run_tests.sh new file mode 100644 index 00000000..ef434d5c --- /dev/null +++ b/apps/cic-eth/docker/run_tests.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +set -e + +pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ +-r admin_requirements.txt \ +-r services_requirements.txt \ +-r test_requirements.txt + +export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests + diff --git a/apps/cic-eth/tests/run_tests.sh b/apps/cic-eth/tests/run_tests.sh new file mode 100644 index 00000000..30500432 --- /dev/null +++ b/apps/cic-eth/tests/run_tests.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +set -e + +pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple +-r admin_requirements.txt +-r services_requirements.txt +-r test_requirements.txt + +export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests diff --git a/apps/cic-meta/.gitlab-ci.yml b/apps/cic-meta/.gitlab-ci.yml index 8ab7177d..c77d18d7 100644 --- a/apps/cic-meta/.gitlab-ci.yml +++ b/apps/cic-meta/.gitlab-ci.yml @@ -1,43 +1,16 @@ - -.cic_meta_variables: - variables: - APP_NAME: cic-meta - DOCKERFILE_PATH: docker/Dockerfile_ci - CONTEXT: apps/$APP_NAME - -build-mr-cic-meta: - extends: - - .py_build_merge_request - - .cic_meta_variables - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/cic-meta/**/* - when: always - -test-mr-cic-meta: - extends: - - .cic_meta_variables - stage: test - image: $MR_IMAGE_TAG - script: - - cd /root - - npm install --dev - - npm run test - - npm run test:coverage - needs: ["build-mr-cic-meta"] - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/cic-meta/**/* - when: always - -build-push-cic-meta: - extends: - - .py_build_push - - .cic_meta_variables - rules: - - if: $CI_COMMIT_BRANCH == "master" - changes: - - apps/cic-meta/**/* - when: always +build-test-cic-meta: + stage: test + tags: + - integration + variables: + APP_NAME: cic-meta + MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + script: + - cd apps/cic-meta + - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . + - docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh + #rules: + #- if: $CI_PIPELINE_SOURCE == "merge_request_event" + # changes: + # - apps/$APP_NAME/**/* + # when: always diff --git a/apps/cic-meta/docker/Dockerfile b/apps/cic-meta/docker/Dockerfile index 11eb5d9a..aa4c6a28 100644 --- a/apps/cic-meta/docker/Dockerfile +++ b/apps/cic-meta/docker/Dockerfile @@ -15,11 +15,10 @@ RUN --mount=type=cache,mode=0755,target=/root/.npm \ COPY webpack.config.js . COPY tsconfig.json . ## required to build the cic-client-meta module -COPY src/ src/ -COPY scripts/ scripts/ -COPY tests/ tests/ +COPY . . COPY tests/*.asc /root/pgp/ + ## copy runtime configs COPY .config/ /usr/local/etc/cic-meta/ # diff --git a/apps/cic-meta/docker/Dockerfile_ci b/apps/cic-meta/docker/Dockerfile_ci deleted file mode 100644 index 9215ab71..00000000 --- a/apps/cic-meta/docker/Dockerfile_ci +++ /dev/null @@ -1,32 +0,0 @@ -# syntax = docker/dockerfile:1.2 -#FROM node:15.3.0-alpine3.10 -FROM node:lts-alpine3.14 - -WORKDIR /root - -RUN apk add --no-cache postgresql bash - -# copy the dependencies -COPY package.json package-lock.json . -RUN npm set cache /root/.npm && \ - npm ci - -COPY webpack.config.js . -COPY tsconfig.json . -## required to build the cic-client-meta module -COPY src/ src/ -COPY scripts/ scripts/ -COPY tests/ tests/ -COPY tests/*.asc /root/pgp/ - -## copy runtime configs -COPY .config/ /usr/local/etc/cic-meta/ -# -## db migrations -COPY docker/db.sh ./db.sh -RUN chmod 755 ./db.sh -# -RUN alias tsc=node_modules/typescript/bin/tsc -COPY docker/start_server.sh ./start_server.sh -RUN chmod 755 ./start_server.sh -ENTRYPOINT ["sh", "./start_server.sh"] diff --git a/apps/cic-meta/docker/run_tests.sh b/apps/cic-meta/docker/run_tests.sh new file mode 100644 index 00000000..cf3965f2 --- /dev/null +++ b/apps/cic-meta/docker/run_tests.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +set -e + +npm install --dev +npm run test +npm run test:coverage diff --git a/apps/cic-notify/.gitlab-ci.yml b/apps/cic-notify/.gitlab-ci.yml index 1a4ef23e..e0c9597f 100644 --- a/apps/cic-notify/.gitlab-ci.yml +++ b/apps/cic-notify/.gitlab-ci.yml @@ -1,52 +1,17 @@ -.cic_notify_variables: - variables: - APP_NAME: cic-notify - DOCKERFILE_PATH: docker/Dockerfile_ci - CONTEXT: apps/$APP_NAME - -build-mr-cic-notify: - extends: - - .py_build_merge_request - - .cic_notify_variables - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/cic-notify/**/* - when: always - -test-mr-cic-notify: - stage: test - extends: - - .cic_notify_variables - cache: - key: - files: - - test_requirements.txt - paths: - - /root/.cache/pip - image: $MR_IMAGE_TAG - script: - - cd apps/$APP_NAME/ - - > - pip install --extra-index-url https://pip.grassrootseconomics.net:8433 - --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple - -r test_requirements.txt - - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests - needs: ["build-mr-cic-notify"] - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/$APP_NAME/**/* - when: always - -build-push-cic-notify: - extends: - - .py_build_push - - .cic_notify_variables - rules: - - if: $CI_COMMIT_BRANCH == "master" - changes: - - apps/cic-notify/**/* - when: always - - +build-test-cic-notify: + stage: test + tags: + - integration + variables: + APP_NAME: cic-notify + MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + script: + - cd apps/cic-notify + - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . + - docker run $MR_IMAGE_TAG sh docker/run_tests.sh + allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/$APP_NAME/**/* + when: always diff --git a/apps/cic-notify/docker/Dockerfile_ci b/apps/cic-notify/docker/Dockerfile_ci deleted file mode 100644 index 3db71bc9..00000000 --- a/apps/cic-notify/docker/Dockerfile_ci +++ /dev/null @@ -1,27 +0,0 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev - -#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 - -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -COPY requirements.txt . - -RUN pip install --index-url https://pypi.org/simple \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ - -r requirements.txt - -COPY . . - -RUN python setup.py install - -COPY docker/*.sh . -RUN chmod +x *.sh - -# ini files in config directory defines the configurable parameters for the application -# they can all be overridden by environment variables -# to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -COPY .config/ /usr/local/etc/cic-notify/ -COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/ - -ENTRYPOINT [] diff --git a/apps/cic-notify/docker/run_tests.sh b/apps/cic-notify/docker/run_tests.sh new file mode 100644 index 00000000..1c2e8678 --- /dev/null +++ b/apps/cic-notify/docker/run_tests.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +set -e + +pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ +--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ +-r test_requirements.txt + +export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests diff --git a/apps/cic-ussd/.gitlab-ci.yml b/apps/cic-ussd/.gitlab-ci.yml index e028dc6c..09923c75 100644 --- a/apps/cic-ussd/.gitlab-ci.yml +++ b/apps/cic-ussd/.gitlab-ci.yml @@ -1,52 +1,16 @@ -.cic_ussd_variables: - variables: - APP_NAME: cic-ussd - DOCKERFILE_PATH: docker/Dockerfile_ci - CONTEXT: apps/$APP_NAME - -build-mr-cic-ussd: - extends: - - .py_build_merge_request - - .cic_ussd_variables - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/cic-ussd/**/* - when: always - -test-mr-cic-ussd: - stage: test - extends: - - .cic_ussd_variables - cache: - key: - files: - - test_requirements.txt - paths: - - /root/.cache/pip - image: $MR_IMAGE_TAG - script: - - cd apps/$APP_NAME/ - - > - pip install --extra-index-url https://pip.grassrootseconomics.net:8433 - --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple - -r test_requirements.txt - - export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd - needs: ["build-mr-cic-ussd"] - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/$APP_NAME/**/* - when: always - -build-push-cic-ussd: - extends: - - .py_build_push - - .cic_ussd_variables - rules: - - if: $CI_COMMIT_BRANCH == "master" - changes: - - apps/cic-ussd/**/* - when: always - - +build-test-cic-ussd: + stage: test + tags: + - integration + variables: + APP_NAME: cic-ussd + MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA + script: + - cd apps/cic-ussd + - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . + - docker run $MR_IMAGE_TAG sh docker/run_tests.sh + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/$APP_NAME/**/* + when: always diff --git a/apps/cic-ussd/docker/Dockerfile_ci b/apps/cic-ussd/docker/Dockerfile_ci deleted file mode 100644 index 9811f563..00000000 --- a/apps/cic-ussd/docker/Dockerfile_ci +++ /dev/null @@ -1,32 +0,0 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev -RUN apt-get install -y redis-server - -# create secrets directory -RUN mkdir -vp pgp/keys - -# create application directory -RUN mkdir -vp cic-ussd -RUN mkdir -vp data - -COPY requirements.txt . - -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -RUN pip install --index-url https://pypi.org/simple \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ - -r requirements.txt - -COPY . . -RUN python setup.py install - -COPY cic_ussd/db/ussd_menu.json data/ - -COPY docker/*.sh . -RUN chmod +x /root/*.sh - -# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts -COPY config/ /usr/local/etc/cic-ussd/ -COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic - -ENTRYPOINT [] diff --git a/apps/cic-ussd/docker/run_tests.sh b/apps/cic-ussd/docker/run_tests.sh new file mode 100644 index 00000000..ab673411 --- /dev/null +++ b/apps/cic-ussd/docker/run_tests.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +set -e + + +pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ +--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ +-r test_requirements.txt + +export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd diff --git a/apps/contract-migration/.gitlab-ci.yml b/apps/contract-migration/.gitlab-ci.yml index d48f7933..555dfb31 100644 --- a/apps/contract-migration/.gitlab-ci.yml +++ b/apps/contract-migration/.gitlab-ci.yml @@ -1,25 +1,25 @@ -.contract_migration_variables: - variables: - APP_NAME: contract-migration - DOCKERFILE_PATH: docker/Dockerfile_ci - CONTEXT: apps/$APP_NAME - -build-mr-contract-migration: - extends: - - .py_build_merge_request - - .contract_migration_variables - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - apps/contract-migration/**/* - when: always - -build-push-contract-migration: - extends: - - .py_build_push - - .contract_migration_variables - rules: - - if: $CI_COMMIT_BRANCH == "master" - changes: - - apps/contract-migration/**/* - when: always +#.contract_migration_variables: +# variables: +# APP_NAME: contract-migration +# DOCKERFILE_PATH: docker/Dockerfile_ci +# CONTEXT: apps/$APP_NAME +# +#build-mr-contract-migration: +# extends: +# - .py_build_merge_request +# - .contract_migration_variables +# rules: +# - if: $CI_PIPELINE_SOURCE == "merge_request_event" +# changes: +# - apps/contract-migration/**/* +# when: always +# +#build-push-contract-migration: +# extends: +# - .py_build_push +# - .contract_migration_variables +# rules: +# - if: $CI_COMMIT_BRANCH == "master" +# changes: +# - apps/contract-migration/**/* +# when: always diff --git a/docker-compose.yml b/docker-compose.yml index 08010b38..d61cab55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,20 +13,8 @@ networks: name: cic-network services: - # eth: - # image: trufflesuite/ganache-cli - # ports: - # - ${HTTP_PORT_ETH:-8545} - # - ${WS_PORT_ETH:-8546} - # # Note! -e switch doesnt work, whatever you put there, it will be 100 - # command: "-i 8996 -e 1000 -l 90000000 \ - # -m '${DEV_MNEMONIC:-\"history stumble mystery avoid embark arrive mom foil pledge keep grain dice\"}' \ - # -v --db /tmp/cic/ganache/ganache.db \ - # --noVMErrorsOnRPCResponse --allowUnlimitedContractSize" - # volumes: - # - ganache-db:/tmp/cic/ganache - eth: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/bloxberg-node:${TAG:-latest} build: context: apps/bloxbergValidatorSetup restart: unless-stopped @@ -71,6 +59,7 @@ services: - bee-data:/tmp/cic/bee contract-migration: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest} profiles: - migrations build: @@ -129,6 +118,7 @@ services: - contract-config:/tmp/cic/config cic-cache-tracker: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} profiles: - cache build: @@ -170,6 +160,7 @@ services: - contract-config:/tmp/cic/config/:ro cic-cache-tasker: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} profiles: - cache build: @@ -210,6 +201,7 @@ services: - contract-config:/tmp/cic/config/:ro cic-cache-server: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} profiles: - cache build: @@ -245,6 +237,7 @@ services: cic-eth-tasker: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} build: context: apps/cic-eth dockerfile: docker/Dockerfile @@ -298,6 +291,7 @@ services: # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ] cic-eth-tracker: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} build: context: apps/cic-eth dockerfile: docker/Dockerfile @@ -342,6 +336,7 @@ services: cic-eth-dispatcher: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} build: context: apps/cic-eth dockerfile: docker/Dockerfile @@ -386,6 +381,7 @@ services: cic-eth-retrier: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} build: context: apps/cic-eth dockerfile: docker/Dockerfile @@ -433,6 +429,7 @@ services: cic-notify-tasker: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:${TAG:-latest} build: context: apps/cic-notify dockerfile: docker/Dockerfile @@ -461,6 +458,7 @@ services: cic-meta-server: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest} profiles: - custodial-meta hostname: meta @@ -496,6 +494,7 @@ services: # command: "/root/start_server.sh -vv" cic-user-ussd-server: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} profiles: - custodial-ussd build: @@ -528,6 +527,7 @@ services: command: "/root/start_cic_user_ussd_server.sh -vv" cic-user-server: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} profiles: - custodial-ussd build: @@ -553,6 +553,7 @@ services: command: "/root/start_cic_user_server.sh -vv" cic-user-tasker: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} profiles: - custodial-ussd build: diff --git a/scripts/build-push.sh b/scripts/build-push.sh new file mode 100755 index 00000000..141a9e6c --- /dev/null +++ b/scripts/build-push.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env sh + +# Exit in case of error +set -e + +TAG=${TAG?Variable not set} \ +sh ./scripts/build.sh + +docker-compose -f docker-compose.yml push diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000..b3da3a6a --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env sh + +# Exit in case of error +set -e + +TAG=${TAG?Variable not set} \ +docker-compose \ +-f docker-compose.yml \ +build diff --git a/scripts/test-local.sh b/scripts/test-local.sh new file mode 100755 index 00000000..4c180f21 --- /dev/null +++ b/scripts/test-local.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +# Exit in case of error +set -e + +docker-compose down -v --remove-orphans # Remove possibly previous broken stacks left hanging after an error + +if [ $(uname -s) = "Linux" ]; then + echo "Remove __pycache__ files" + sudo find . -type d -name __pycache__ -exec rm -r {} \+ +fi + +docker-compose build +docker-compose up -d +docker-compose exec -T backend bash /app/tests-start.sh "$@"