docker: multi tag builds

.gitlab-ci.yml

@@ -1,43 +1,14 @@
-include:
+default:
-  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
+  image: docker:latest
-  # these includes are app specific unit tests
+  tags:
-  - local: 'apps/cic-eth/.gitlab-ci.yml'
+    - cic-runner
-  - local: 'apps/cic-ussd/.gitlab-ci.yml'
-  - local: 'apps/cic-notify/.gitlab-ci.yml'
-  - local: 'apps/cic-meta/.gitlab-ci.yml'
-  - local: 'apps/cic-cache/.gitlab-ci.yml'
-    #- local: 'apps/contract-migration/.gitlab-ci.yml'
-    #- local: 'apps/data-seeding/.gitlab-ci.yml'
-
-stages:
-  - build
-  - test
-  - deploy
-
-image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
 
 variables:
+  REGISTRY: registry.gitlab.com/kamikazechaser/cic-monorepo
   DOCKER_BUILDKIT: "1"
-  COMPOSE_DOCKER_CLI_BUILD: "1"
-  CI_DEBUG_TRACE: "true"
 
 before_script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
 
-# runs on protected branches and pushes to repo
+include:
-build-push:
+  - local: 'apps/cic-eth/.gitlab-ci.yml'
-  stage: build
-  tags:
-    - integration
-  #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
-  script:
-    - TAG=latest sh ./scripts/build-push.sh
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-
-deploy-dev:
-  stage: deploy
-  trigger: grassrootseconomics/devops
-  when: manual

apps/cic-eth/.gitlab-ci.yml

@@ -1,16 +1,33 @@
-build-test-cic-eth:
+stages:
-  stage: test
+  - build-push
-  tags:
+  - test
-    - integration
+
-  variables:
+variables:
-    APP_NAME: cic-eth
+  APP_NAME: cic-eth
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+
+workflow:
+  rules:
+    - changes:
+      - apps/$APP_NAME/**/*
+
+cic-eth:build-push:
+  stage: build-push
   script:
-    - cd apps/cic-eth
+    - cd apps/$APP_NAME
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+    - docker pull $REGISTRY/$APP_NAME:latest || true
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    - docker build --cache-from $REGISTRY/$APP_NAME:latest -t $REGISTRY/$APP_NAME:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile .
+    - docker push $REGISTRY/$APP_NAME --all-tags
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_REF_PROTECTED == "true"
+
+cic-eth:test:
+  variables:
+    GIT_STRATEGY: none
+  stage: test
+  needs: ["cic-eth:build-push"]
+  variables:
+  allow_failure: true
+  script:
+    - docker run $REGISTRY/$APP_NAME:$CI_COMMIT_SHORT_SHA sh docker/run_tests.sh
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always

apps/contract-migration/config.sh

@@ -5,13 +5,7 @@ set -a
 if [ -z $DEV_DATA_DIR ]; then
 	export DEV_DATA_DIR=`mktemp -d`
 else
-	mkdir -vp $DEV_DATA_DIR
+	mkdir -p $DEV_DATA_DIR
-fi
-
-if [ -z $STATE_DIR ]; then
-	export STATE_DIR=/run/cic_init
-	mkdir -vp $STATE_DIR
-	echo 0 > $STATE_DIR/init
 fi
 
 if [ -z $DEV_CONFIG_RESET ]; then

apps/contract-migration/reset.sh

@@ -117,8 +117,6 @@ export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
 export TOKEN_NAME=$TOKEN_NAME
 " >> "${DEV_DATA_DIR}"/env_reset
 
-echo -n $CIC_REGISTRY_ADDRESS > $STATE_DIR/registry
-
 set +a
 set +e
 

apps/contract-migration/run_job.sh

@@ -16,7 +16,6 @@ then
 		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 1 - contract deployment"
 		exit 1;
 	fi
-	echo -n 1 > $STATE_DIR/init
 	>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 1 - contract deployment"
 fi
 
@@ -28,6 +27,5 @@ then
 		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 2 - custodial service initialization"
 		exit 1;
 	fi
-	echo -n 2 > $STATE_DIR/init
 	>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 2 - custodial service initialization"
 fi

apps/util/initness/MANIFEST.in

@@ -1 +0,0 @@
-include requirements.txt initness/data/config/**

apps/util/initness/initness/__init__.py

@@ -1,3 +0,0 @@
-"""Initness reports the initialization state and entry-point variables for the contract migrations process of the CIC services deployment.
-"""
-from .state import *

apps/util/initness/initness/data/config/httpd.ini

@@ -1,3 +0,0 @@
-[httpd]
-host = localhost 
-port = 8000

apps/util/initness/initness/data/config/state.ini

@@ -1,2 +0,0 @@
-[state]
-base_dir = /run/cic_init

apps/util/initness/initness/runnable/server.py

@@ -1,87 +0,0 @@
-# standard import
-import json
-import os
-import logging
-import argparse
-import sys
-from http.server import (
-        HTTPServer,
-        BaseHTTPRequestHandler,
-    )
-
-# external imports
-import confini
-
-# local imports
-from initness import get_state
-
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-data_dir = os.path.join(script_dir, '..', 'data')
-default_config_dir = os.path.join(data_dir, 'config')
-config_dir = os.environ.get('CONFINI_DIR', default_config_dir)
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', '--config', dest='c', type=str, help='configuration override directory')
-argparser.add_argument('--host', type=str, help='httpd host')
-argparser.add_argument('--port', type=str, help='httpd port')
-argparser.add_argument('--state-dir', dest='state_dir', type=str, help='directory to read state from')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-override_dirs = []
-if args.c:
-    override_dirs = [args.c]
-config = confini.Config(config_dir, args.env_prefix, override_dirs=override_dirs)
-config.process()
-# override args
-args_override = {
-        'HTTPD_HOST': getattr(args, 'host'),
-        'HTTPD_PORT': getattr(args, 'port'),
-        'STATE_BASE_DIR': getattr(args, 'state_dir'),
-        }
-config.dict_override(args_override, 'cli flag')
-logg.debug('loaded config: {}\n'.format(config))
-
-
-class StateRequestHandler(BaseHTTPRequestHandler):
-
-    state_store_dir = None
-
-    def do_GET(self):
-       
-        o = get_state(self.state_store_dir)
-        self.send_response(200, 'yarr')
-        self.end_headers()
-
-        self.wfile.write(json.dumps(o).encode('utf-8'))
-
-
-def run(store, host=None, port=None):
-    port = int(port, 10)
-    server_address = (host, port)
-    httpd = HTTPServer(server_address, StateRequestHandler)
-    httpd.serve_forever()
-
-
-def main():
-    try:
-        os.stat(config.get('STATE_BASE_DIR'))
-    except FileNotFoundError:
-        os.makedirs(config.get('STATE_BASE_DIR'))
-    store = StateRequestHandler.state_store_dir=config.get('STATE_BASE_DIR')
-    run(store, host=config.get('HTTPD_HOST'), port=config.get('HTTPD_PORT'))
-
-
-if __name__ == '__main__':
-    main()

apps/util/initness/initness/state.py

@@ -1,31 +0,0 @@
-# standard imports
-import os
-
-def get_state(state_store_dir):
-        init_path = os.path.join(state_store_dir, 'init')
-        init_level = 0
-        registry_address = None
-
-        try:
-            f = open(init_path, 'r')
-            init_level = f.read()
-            init_level = init_level.rstrip()
-            f.close()
-        except FileNotFoundError:
-            pass
-
-        registry_path = os.path.join(state_store_dir, 'registry')
-        try:
-            f = open(registry_path, 'r')
-            registry_address = f.read()
-            registry_address = registry_address.rstrip()
-            f.close()
-        except FileNotFoundError:
-            pass
-
-        o = {
-            'runlevel': init_level,
-            'registry': registry_address,
-                }
-
-        return o

apps/util/initness/requirements.txt

@@ -1 +0,0 @@
-confini>=0.3.6rc4,<0.5.0

apps/util/initness/setup.py

@@ -1,21 +0,0 @@
-from setuptools import setup
-
-f = open('requirements.txt', 'r')
-requirements = f.read()
-f.close()
-
-setup(
-        name='initness',
-        version='0.0.1a1',
-        install_requires=requirements,
-        packages=[
-            'initness',
-            'initness.runnable',
-            ],
-        include_package_data=True,
-        entry_points = {
-            'console_scripts': [
-                'cic-init-server=initness.runnable.server:main',
-            ],
-            },
-        )

apps/util/initness/tests/test_basic.py

@@ -1,27 +0,0 @@
-# standard imports
-import unittest
-import tempfile
-import os
-
-# local imports
-from initness.runnable.server import get_state
-
-class TestInitness(unittest.TestCase):
-
-    def setUp(self):
-        self.dir = tempfile.mkdtemp()
-        f = open(os.path.join(self.dir, 'init'), 'w')
-        f.write('42')
-        f.close()
-        f = open(os.path.join(self.dir, 'registry'), 'w')
-        f.write('0xdeadbeef')
-        f.close()
-
-    def test_state(self):
-        o = get_state(self.dir)
-        self.assertEqual(o['runlevel'], '42')
-        self.assertEqual(o['registry'], '0xdeadbeef')
-
-
-if __name__ == '__main__':
-    unittest.main()