Merge branch 'bvander/contract-migration-caching' into 'master'

refactor contract-migration and add buildkit caching

See merge request grassrootseconomics/cic-internal-integration!211

apps/contract-migration/.gitlab-ci.yml

@@ -1,7 +1,7 @@
 .contract_migration_variables:
     variables:
         APP_NAME: contract-migration
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
 
 .contract_migration_changes_target:
     rules:

apps/contract-migration/docker/Dockerfile

@@ -1,24 +1,18 @@
 # syntax = docker/dockerfile:1.2
-FROM python:3.8.6-slim-buster as compile-image
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e
 
-RUN apt-get update
+WORKDIR /root
-RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
 
 RUN touch /etc/apt/sources.list.d/ethereum.list
 RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
 RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
-
+RUN cat /etc/apt/sources.list.d/ethereum.list
-RUN cat etc/apt/sources.list.d/ethereum.list
 RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
 
-RUN apt-get update
+#RUN apt-get install solc
-RUN apt-get install solc
-RUN pip install --upgrade pip
 
-WORKDIR /root
 RUN mkdir -vp /usr/local/etc/cic
 
-COPY contract-migration/nvm.sh .
 ENV CONFINI_DIR /usr/local/etc/cic/
 RUN mkdir -vp $CONFINI_DIR
 
@@ -31,81 +25,18 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-# Install nvm with node and npm
-# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
-ENV NVM_DIR /root/.nvm
-ENV NODE_VERSION 15.3.0
-ENV BANCOR_NODE_VERSION 10.16.0
-
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
-	&& . $NVM_DIR/nvm.sh \
-	&& nvm install $NODE_VERSION \
-	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION
-# && chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
-ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
-
-#RUN useradd --create-home grassroots
-# WORKDIR /home/grassroots
-# USER grassroots
-
 COPY contract-migration/requirements.txt . 
 
-ARG pip_extra_args=""
 ARG pip_index_url=https://pypi.org/simple
-ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 ARG pip_trusted_host=pypi.org
-RUN pip install --index-url $pip_index_url \
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    --trusted-host $pip_trusted_host \
+    pip install --index-url https://pypi.org/simple \
-    --extra-index-url $pip_extra_index_url -r requirements.txt
-
-# -------------- begin runtime container ---------------- 
-FROM python:3.8.6-slim-buster as runtime-image
-
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
-RUN apt-get install -y jq bash iputils-ping socat
-
-COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
-COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
-COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-
-ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
-# RUN useradd -u 1001 --create-home grassroots
-# RUN adduser grassroots sudo && \
-#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-# WORKDIR /home/grassroots
-
-COPY contract-migration/testdata/pgp testdata/pgp
-COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
-COPY contract-migration/keystore keystore
-COPY contract-migration/envlist .
-
-# A shared output dir for environment configs
-RUN mkdir -p /tmp/cic/config
-# RUN chown grassroots:grassroots /tmp/cic/config
-RUN chmod a+rwx /tmp/cic/config 
-
-COPY contract-migration/*.sh ./
-# RUN chown grassroots:grassroots -R .
-RUN chmod gu+x *.sh
-
-# we copied these from the root build container.
-# this is dumb though...I guess the compile image should have the same user
-# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
-
-# USER grassroots
-
-ARG pip_index_url=https://pypi.org/simple
-ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG pip_trusted_host=pypi.org
-COPY contract-migration/override_requirements.txt . 
-RUN pip install --index-url $pip_index_url \
     --force-reinstall \
     --trusted-host $pip_trusted_host \
-    --extra-index-url $pip_extra_index_url -r override_requirements.txt
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt 
 
-ENTRYPOINT [ ]
+COPY contract-migration/ .
+RUN chmod +x *.sh

apps/contract-migration/docker/Dockerfile_ci

@@ -0,0 +1,42 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e
+
+WORKDIR /root
+
+# solc install which we needed for bancor. Leaving as an artfact of HOW to do it. 
+#RUN touch /etc/apt/sources.list.d/ethereum.list
+#RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
+#RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
+#RUN cat /etc/apt/sources.list.d/ethereum.list
+#RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
+
+#RUN apt-get install solc
+
+RUN mkdir -vp /usr/local/etc/cic
+
+ENV CONFINI_DIR /usr/local/etc/cic/
+RUN mkdir -vp $CONFINI_DIR
+
+ARG cic_config_commit=0abe0867f18077907c7023bf0ef5e466a3984dd8
+ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
+RUN echo Install confini schema files && \
+	git clone --depth 1 $cic_config_url cic-config && \
+	cd cic-config && \
+	git fetch --depth 1 origin $cic_config_commit && \
+	git checkout $cic_config_commit && \	
+	cp -v *.ini $CONFINI_DIR
+
+COPY contract-migration/requirements.txt . 
+
+ARG pip_index_url=https://pypi.org/simple
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG pip_trusted_host=pypi.org
+RUN pip install --index-url https://pypi.org/simple \
+    --force-reinstall \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt 
+
+COPY contract-migration/ .
+RUN chmod +x *.sh