Bump aux version

.gitignore

@@ -13,4 +13,3 @@ build/
 **/coverage
 **/.venv
 .idea
-**/.vim

apps/cic-cache/.coveragerc

@@ -2,5 +2,4 @@
 omit = 
 	.venv/*
 	scripts/*
-	cic_cache/db/migrations/*
+	cic_cache/db/postgres/*
-	cic_cache/version.py

apps/cic-cache/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-cache/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_cache_variables:
     variables:
         APP_NAME: cic-cache
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_cache_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-cache:
     extends:
+        - .cic_cache_changes_target
         - .py_build_merge_request
         - .cic_cache_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-
-test-mr-cic-cache:
-    stage: test
-    extends:
-        - .cic_cache_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-cache"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 build-push-cic-cache:
     extends:
         - .py_build_push
         - .cic_cache_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
         
 

apps/cic-cache/cic_cache/cache.py

@@ -15,8 +15,6 @@ from cic_cache.db.list import (
 logg = logging.getLogger()
 
 
-DEFAULT_FILTER_SIZE = 8192 * 8
-
 class Cache:
 
     def __init__(self, session):
@@ -27,7 +25,7 @@ class BloomCache(Cache):
 
     @staticmethod
     def __get_filter_size(n):
-        n = DEFAULT_FILTER_SIZE
+        n = 8192 * 8
         logg.warning('filter size hardcoded to {}'.format(n))
         return n
 

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -4,9 +4,6 @@ import json
 import re
 import base64
 
-# external imports
-from hexathon import add_0x
-
 # local imports
 from cic_cache.cache import (
         BloomCache,
@@ -14,11 +11,10 @@ from cic_cache.cache import (
     )
 
 logg = logging.getLogger(__name__)
-#logg = logging.getLogger()
 
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
+re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
 
 DEFAULT_LIMIT = 100
 
@@ -30,13 +26,13 @@ def process_transactions_account_bloom(session, env):
 
     address = r[1]
     if r[2] == None:
-        address = add_0x(address)
+        address = '0x' + address
-    offset = 0
+    offset = DEFAULT_LIMIT
     if r.lastindex > 2:
-        offset = r[4]
+        offset = r[3]
-    limit = DEFAULT_LIMIT
+    limit = 0
-    if r.lastindex > 4:
+    if r.lastindex > 3:
-        limit = r[6]
+        limit = r[4]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)

apps/cic-cache/docker/Dockerfile

@@ -1,38 +1,52 @@
-# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-COPY requirements.txt .
+#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+WORKDIR /usr/src/cic-cache
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
 
-COPY . . 
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+ARG root_requirement_file='requirements.txt'
 
-RUN python setup.py install
+#RUN apk update && \
+#	apk add gcc musl-dev gnupg libpq
+#RUN apk add postgresql-dev
+#RUN apk add linux-headers
+#RUN apk add libffi-dev
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+# Copy shared requirements from top of mono-repo
+RUN echo "copying root req file ${root_requirement_file}"
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY cic-cache/requirements.txt ./
+COPY cic-cache/setup.cfg \
+	 cic-cache/setup.py \
+ 	 ./
+COPY cic-cache/cic_cache/ ./cic_cache/
+COPY cic-cache/scripts/ ./scripts/
+COPY cic-cache/test_requirements.txt ./
+RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+RUN pip install $pip_extra_index_url_flag .
+RUN pip install .[server]
+
+COPY cic-cache/tests/ ./tests/
+#COPY db/ cic-cache/db
+#RUN apk add postgresql-client
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
+COPY cic-cache/config/ /usr/local/etc/cic-cache/
 
 # for db migrations
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
+COPY cic-cache/docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server
 # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/Dockerfile_ci

@@ -1,37 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/tests/conftest.py

@@ -5,12 +5,9 @@ import datetime
 
 # external imports
 import pytest
-import moolb
 
 # local imports
 from cic_cache import db
-from cic_cache import BloomCache
-from cic_cache.cache import DEFAULT_FILTER_SIZE
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -104,7 +101,3 @@ def tag_txs(
 
     db.tag_transaction(init_database, txs[1], 'taag', domain='test')
 
-
-@pytest.fixture(scope='session')
-def zero_filter():
-    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/test_erc20.py

@@ -10,7 +10,6 @@ from sqlalchemy import text
 from chainlib.eth.tx import Tx
 from chainlib.eth.block import Block
 from chainlib.chain import ChainSpec
-from chainlib.eth.error import RequestMismatchException
 from hexathon import (
         strip_0x,
         add_0x,
@@ -19,21 +18,10 @@ from hexathon import (
 # local imports
 from cic_cache.db import add_tag
 from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
-from cic_cache.runnable.daemons.filters.base import TagSyncFilter
 
 logg = logging.getLogger()
 
 
-def test_base_filter_str(
-        init_database,
-        ):
-        f = TagSyncFilter('foo')
-        assert 'foo' == str(f)
-        f = TagSyncFilter('foo', domain='bar')
-        assert 'bar.foo' == str(f)
-
-
-
 def test_erc20_filter(
         eth_rpc,
         foo_token,
@@ -79,95 +67,3 @@ def test_erc20_filter(
     s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
     r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
     assert r[0] == tx.hash
-
-
-def test_erc20_filter_nocontract(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': os.urandom(20).hex(),
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-
-@pytest.mark.parametrize(
-        'contract_method,contract_input,expected_exception',
-        [
-            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
-            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
-            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
-            ],
-        )
-def test_erc20_filter_bogus(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        contract_method,
-        contract_input,
-        expected_exception,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = contract_method
-    data += contract_input 
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/test_query.py

@@ -1,230 +0,0 @@
-# standard imports
-import logging
-import json
-import base64
-import copy
-import re
-
-# external imports
-import pytest
-from hexathon import strip_0x
-
-# local imports
-from cic_cache.runnable.daemons.query import *
-
-logg = logging.getLogger()
-
-
-
-@pytest.mark.parametrize(
-        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
-        [
-            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
-            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
-            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
-            ],
-        )
-def test_query_regex(
-        list_actors,
-        query_path_prefix,
-        query_role,
-        query_address_index,
-        query_offset,
-        query_offset_index,
-        query_limit,
-        query_limit_index,
-        match_re,
-        ):
-
-        paths = []
-        path = query_path_prefix
-        query_address = None
-        if query_role != None:
-            query_address = strip_0x(list_actors[query_role])
-            paths.append(path + '0x' + query_address)
-            paths.append(path + query_address)
-        if query_offset != None:
-            if query_limit != None:
-                for i in range(len(paths)-1):
-                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
-            else:
-                for i in range(len(paths)-1):
-                    paths[i] += '/' + str(query_offset)
-
-        for i in range(len(paths)):
-            paths.append(paths[i] + '/')
-
-        for p in paths:
-            logg.debug('testing path {} against {}'.format(p, match_re))
-            m = re.match(match_re, p)
-            l = len(m.groups())
-            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
-            for i in range(l+1):
-                logg.debug('group {} {}'.format(i, m[i]))
-            if m.lastindex >= query_offset_index:
-                assert query_offset == int(m[query_offset_index + 1])
-            if m.lastindex >= query_limit_index:
-                assert query_limit == int(m[query_limit_index + 1])
-            if query_address_index != None:
-                match_address = strip_0x(m[query_address_index + 1])
-                assert query_address == match_address
-
-
-
-@pytest.mark.parametrize(
-        'role_name, query_offset, query_limit, query_match',
-        [ 
-            ('alice', None, None, [(420000, 13), (419999, 42)]),
-            ('alice', None, 1, [(420000, 13)]),
-            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
-            ('alice', 2, None, []), # 420000 == list_defaults['block']
-            ],
-        )
-def test_query_process_txs_account(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        role_name,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    actor = None
-    try:
-        actor = list_actors[role_name]
-    except KeyError:
-        actor = os.urandom(20).hex()
-    path_info = '/tx/user/0x' + strip_0x(actor)
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_account_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_offset, query_limit, query_match',
-        [ 
-            (None, 2, [(420000, 13), (419999, 42)]),
-            (0, 1, [(420000, 13)]),
-            (1, 1, [(419999, 42)]),
-            (2, 0, []),
-            ],
-        )
-def test_query_process_txs_bloom(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    path_info = '/tx'
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_block_start, query_block_end, query_match_count',
-        [ 
-            (None, 42, 0),
-            (420000, 420001, 1),
-            (419999, 419999, 1), # matches are inclusive
-            (419999, 420000, 2),
-            (419999, 420001, 2),
-            ],
-        )
-def test_query_process_txs_data(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_block_start,
-        query_block_end,
-        query_match_count,
-        ):
-
-    path_info = '/txa'
-    if query_block_start != None:
-        path_info += '/' + str(query_block_start)
-    if query_block_end != None:
-        if query_block_start == None:
-            path_info += '/0'
-        path_info += '/' + str(query_block_end)
-    env = {
-            'PATH_INFO': path_info,
-            'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_data(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    assert len(o['data']) == query_match_count

apps/cic-eth/.dockerignore

@@ -1,6 +0,0 @@
-.git
-.cache
-.dot
-**/doc
-**/.venv
-**/venv

apps/cic-eth/.gitlab-ci.yml

@@ -1,52 +1,31 @@
 .cic_eth_variables:
     variables:
         APP_NAME: cic-eth
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_eth_mr_changes_target:
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - $CONTEXT/$APP_NAME/**/*
+      when: always
 
 build-mr-cic-eth:
     extends:
         - .cic_eth_variables
-        - .py_build_target_dev
+        - .cic_eth_mr_changes_target
-    rules:
+        - .py_build_target_test
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 test-mr-cic-eth:
-    stage: test
     extends:
         - .cic_eth_variables
-    cache:
+        - .cic_eth_mr_changes_target
-        key:
+    stage: test
-            files:
+    image: $IMAGE_TAG_BASE
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
     script:
         - cd apps/$APP_NAME/
-        - >
+        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r admin_requirements.txt
-          -r services_requirements.txt
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-eth"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 build-push-cic-eth:
     extends:
         - .py_build_push
-        - .cic_eth_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-eth/**/*
-      when: always

apps/cic-eth/docker/Dockerfile

@@ -1,39 +1,58 @@
-# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster as compile
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+WORKDIR /usr/src
-# do load these all into setup.py later
+
-# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+RUN apt-get update && \
-#COPY cic-eth/requirements.txt .
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+#RUN python -m venv venv && . venv/bin/activate
+
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN /usr/local/bin/python -m pip install --upgrade pip
+RUN pip install semver
+
+COPY cic-eth-aux/  ./cic-eth-aux/
+WORKDIR /usr/src/cic-eth-aux/erc20-demurrage-token
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    	--extra-index-url $EXTRA_INDEX_URL .
+
+WORKDIR /usr/src/cic-eth
+
+COPY cic-eth/ .
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    	--extra-index-url $EXTRA_INDEX_URL .[services]
+
+# --- TEST IMAGE ---
+FROM python:3.8.6-slim-buster as test 
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+WORKDIR /usr/src/cic-eth
+
+RUN /usr/local/bin/python -m pip install --upgrade pip
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+# TODO we could use venv inside container to isolate the system and app deps further
+# COPY --from=compile /usr/src/cic-eth/ .
+# RUN . venv/bin/activate
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-#    pip install --index-url https://pypi.org/simple \
-#    --force-reinstall \
-#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-#    -r requirements.txt 
 
-COPY . .
+COPY cic-eth/test_requirements.txt .
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    pip install --index-url https://pypi.org/simple  \
+    	--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL .
+
+COPY cic-eth . 
 
 ENV PYTHONPATH .
 
-COPY docker/entrypoints/* ./ 
+ENTRYPOINT ["pytest"]
-RUN chmod 755 *.sh
-
-# # ini files in config directory defines the configurable parameters for the application
-# # they can all be overridden by environment variables
-# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-ENTRYPOINT []
 
 # --- RUNTIME ---
 FROM python:3.8.6-slim-buster as runtime 
@@ -43,22 +62,20 @@ RUN apt-get update && \
 
 WORKDIR /usr/src/cic-eth
 
-COPY --from=dev /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
                           /usr/local/lib/python3.8/site-packages/
 
-COPY docker/entrypoints/* ./ 
+COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
 
+COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
+COPY cic-eth/config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+COPY util/liveness/health.sh /usr/local/bin/health.sh
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-
-ENTRYPOINT []
 

apps/cic-eth/docker/Dockerfile_ci

@@ -1,62 +0,0 @@
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-WORKDIR /usr/src/cic-eth
-
-# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
-# do load these all into setup.py later
-# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
-#COPY cic-eth/requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-#    pip install --index-url https://pypi.org/simple \
-#    --force-reinstall \
-#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-#    -r requirements.txt 
-
-COPY . .
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL .
-
-COPY docker/entrypoints/* ./ 
-RUN chmod 755 *.sh
-
-# # ini files in config directory defines the configurable parameters for the application
-# # they can all be overridden by environment variables
-# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-ENTRYPOINT []
-
-# --- RUNTIME ---
-FROM python:3.8.6-slim-buster as runtime 
-
-RUN apt-get update && \
-	apt install -y gnupg libpq-dev procps
-
-WORKDIR /usr/src/cic-eth
-
-COPY --from=dev /usr/local/bin/ /usr/local/bin/
-COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-
-COPY docker/entrypoints/* ./ 
-RUN chmod 755 *.sh
-
-# # ini files in config directory defines the configurable parameters for the application
-# # they can all be overridden by environment variables
-# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-
-ENTRYPOINT []
-

apps/cic-eth/services_requirements.txt

@@ -10,6 +10,5 @@ eth-address-index~=0.1.2a1
 eth-accounts-index~=0.0.12a1
 cic-eth-registry~=0.5.6a1
 erc20-faucet~=0.2.2a1
-erc20-transfer-authorization~=0.3.2a1
 sarafu-faucet~=0.0.4a1
 moolb~=0.1.1b2

apps/cic-meta/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-meta/.gitlab-ci.yml

@@ -3,41 +3,42 @@
     variables:
         APP_NAME: cic-meta
         DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps
+        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
 
-build-mr-cic-meta:
+.cic_meta_changes_target:
-    extends:
-        - .py_build_merge_request
-        - .cic_meta_variables
     rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
+#        - changes:
-          - apps/cic-meta/**/*
+#            - $CONTEXT/$APP_NAME/*
-      when: always
+        - when: always
+
+cic-meta-build-mr:
+  stage: build
+  extends:
+    - .cic_meta_variables
+    - .cic_meta_changes_target
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
 
 test-mr-cic-meta:
     extends:
         - .cic_meta_variables
+        - .cic_meta_changes_target
     stage: test
-    image: $MR_IMAGE_TAG 
+    image: $IMAGE_TAG 
     script:
         - cd /tmp/src/cic-meta
         - npm install --dev 
         - npm run test
         - npm run test:coverage
-    needs: ["build-mr-cic-meta"]
+    needs: ["cic-meta-build-mr"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-meta/**/*
-      when: always
 
 build-push-cic-meta:
     extends:
         - .py_build_push
         - .cic_meta_variables
-    rules:
+        
-    - if:  $CI_COMMIT_BRANCH == "master"
+
-      changes:
-          - apps/cic-meta/**/*
-      when: always

apps/cic-notify/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_notify_variables:
     variables:
         APP_NAME: cic-notify
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_notify_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-notify:
     extends:
+        - .cic_notify_changes_target
         - .py_build_merge_request
         - .cic_notify_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-notify/**/*
-      when: always
-
-test-mr-cic-notify:
-    stage: test
-    extends:
-        - .cic_notify_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-notify"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 build-push-cic-notify:
     extends:
         - .py_build_push
         - .cic_notify_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-notify/**/*
-      when: always
         
 

apps/cic-notify/docker/Dockerfile

@@ -1,30 +1,38 @@
-# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+WORKDIR /usr/src/cic-notify
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-COPY requirements.txt .
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-    pip install --index-url https://pypi.org/simple  \
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
 
-COPY . . 
+COPY cic-notify/setup.cfg \
+	 cic-notify/setup.py \
+     ./
 
-RUN python setup.py install
+COPY cic-notify/cic_notify/ ./cic_notify/
 
-# TODO please review..can this go into requirements?
+COPY cic-notify/requirements.txt \
+	 cic-notify/test_requirements.txt \
+     ./
+
+COPY cic-notify/scripts/ scripts/
 RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
 
-COPY docker/*.sh .
+COPY cic-notify/tests/ tests/
+COPY cic-notify/docker/db.sh \
+     cic-notify/docker/start_tasker.sh \
+	 /root/
+
+#RUN apk add postgresql-client
+#RUN apk add bash
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY .config/ /usr/local/etc/cic-notify/
+COPY cic-notify/.config/ /usr/local/etc/cic-notify/
-COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+COPY cic-notify/cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
 
-ENTRYPOINT []
+WORKDIR /root

apps/cic-notify/docker/Dockerfile_ci

@@ -1,29 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-COPY requirements.txt .
-
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# TODO please review..can this go into requirements?
-RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
-
-COPY docker/*.sh .
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY .config/ /usr/local/etc/cic-notify/
-COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
-
-ENTRYPOINT []

apps/cic-ussd/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-ussd/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_ussd_variables:
     variables:
         APP_NAME: cic-ussd
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_ussd_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-ussd:
     extends:
+        - .cic_ussd_changes_target
         - .py_build_merge_request
         - .cic_ussd_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-ussd/**/*
-      when: always
-
-test-mr-cic-ussd:
-    stage: test
-    extends:
-        - .cic_ussd_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-ussd"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 build-push-cic-ussd:
     extends:
         - .py_build_push
         - .cic_ussd_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-ussd/**/*
-      when: always
 
 

apps/cic-ussd/docker/Dockerfile

@@ -1,5 +1,14 @@
-# syntax = docker/dockerfile:1.2
+# FROM python:3.8.5-alpine
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+FROM python:3.8.6-slim-buster
+
+# set working directory
+WORKDIR /usr/src
+
+# add args for installing from self-hosted packages
+ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
 # create secrets directory
 RUN mkdir -vp pgp/keys
@@ -8,25 +17,39 @@ RUN mkdir -vp pgp/keys
 RUN mkdir -vp cic-ussd
 RUN mkdir -vp data
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+COPY cic-ussd/setup.cfg \
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+     cic-ussd/setup.py \
-COPY requirements.txt .
+     cic-ussd/
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+COPY cic-ussd/requirements.txt \
-    pip install --index-url https://pypi.org/simple  \
+     cic-ussd/test_requirements.txt \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+     cic-ussd/
-    -r requirements.txt
 
-COPY . .
+# install requirements
-RUN python setup.py install
+RUN cd cic-ussd && \
+    pip install -r requirements.txt $pip_extra_index_url_flag
 
-COPY cic_ussd/db/ussd_menu.json data/
+# copy all necessary files
+COPY cic-ussd/cic_ussd/ cic-ussd/cic_ussd/
+COPY cic-ussd/cic_ussd/db/ussd_menu.json data/
+COPY cic-ussd/scripts/ cic-ussd/scripts/
+COPY cic-ussd/states/ cic-ussd/states/
+COPY cic-ussd/transitions/ cic-ussd/transitions/
+COPY cic-ussd/var/ cic-ussd/var/
+
+COPY cic-ussd/docker/db.sh \
+     cic-ussd/docker/start_cic_user_tasker.sh \
+     cic-ussd/docker/start_cic_user_ussd_server.sh\
+     cic-ussd/docker/start_cic_user_server.sh\
+     /root/
 
-COPY docker/*.sh .
 RUN chmod +x /root/*.sh
+RUN cd cic-ussd && \
+    pip install $pip_extra_index_url_flag .
+
 
 # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY .config/ /usr/local/etc/cic-ussd/
+COPY cic-ussd/.config/ /usr/local/etc/cic-ussd/
-COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+COPY cic-ussd/cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
 
-ENTRYPOINT []
+WORKDIR /root

apps/cic-ussd/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-
-# create secrets directory
-RUN mkdir -vp pgp/keys
-
-# create application directory
-RUN mkdir -vp cic-ussd
-RUN mkdir -vp data
-
-COPY requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . .
-RUN python setup.py install
-
-COPY cic_ussd/db/ussd_menu.json data/
-
-COPY docker/*.sh .
-RUN chmod +x /root/*.sh
-
-# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY .config/ /usr/local/etc/cic-ussd/
-COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
-
-ENTRYPOINT []

apps/contract-migration/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/contract-migration/.gitlab-ci.yml

@@ -1,25 +1,20 @@
 .contract_migration_variables:
     variables:
         APP_NAME: contract-migration
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
+
+.contract_migration_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-contract-migration:
     extends:
+        - .contract_migration_changes_target
         - .py_build_merge_request
         - .contract_migration_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/contract-migration/**/*
-      when: always
 
 build-push-contract-migration:
     extends:
         - .py_build_push
         - .contract_migration_variables
-    rules:
-    - if: $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/contract-migration/**/*
-      when: always

apps/contract-migration/docker/Dockerfile

@@ -25,7 +25,7 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-COPY requirements.txt . 
+COPY contract-migration/requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@@ -38,5 +38,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
 
-COPY . .
+COPY contract-migration/ .
 RUN chmod +x *.sh

apps/contract-migration/docker/Dockerfile_ci

@@ -3,11 +3,12 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55
 
 WORKDIR /root
 
-RUN touch /etc/apt/sources.list.d/ethereum.list
+# solc install which we needed for bancor. Leaving as an artfact of HOW to do it. 
-RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
+#RUN touch /etc/apt/sources.list.d/ethereum.list
-RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
+#RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
-RUN cat /etc/apt/sources.list.d/ethereum.list
+#RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
-RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
+#RUN cat /etc/apt/sources.list.d/ethereum.list
+#RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
 
 #RUN apt-get install solc
 
@@ -25,7 +26,7 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-COPY requirements.txt . 
+COPY contract-migration/requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@@ -37,5 +38,5 @@ RUN pip install --index-url https://pypi.org/simple \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
 
-COPY . .
+COPY contract-migration/ .
 RUN chmod +x *.sh

apps/data-seeding/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/data-seeding/.gitlab-ci.yml

@@ -1,26 +1,21 @@
 .data_seeding_variables:
     variables:
         APP_NAME: data-seeding
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
+
+.data_seeding_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-data-seeding:
     extends:
+        - .data_seeding_changes_target
         - .py_build_merge_request
         - .data_seeding_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/data-seeding/**/*
-      when: always
 
 build-push-data-seeding:
     extends:
         - .py_build_push
         - .data_seeding_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/data-seeding/**/*
-      when: always
 

apps/data-seeding/docker/Dockerfile

@@ -5,13 +5,13 @@ WORKDIR /root
 
 RUN mkdir -vp /usr/local/etc/cic
 
-COPY package.json \
+COPY data-seeding/package.json \
-     package-lock.json \
+     data-seeding/package-lock.json \
      .
 
 RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
 
-COPY requirements.txt . 
+COPY data-seeding/requirements.txt . 
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
@@ -19,6 +19,6 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
     --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL -r requirements.txt
 
-COPY . . 
+COPY data-seeding/ . 
 
 ENTRYPOINT [ ]

apps/data-seeding/docker/Dockerfile_ci

@@ -5,13 +5,13 @@ WORKDIR /root
 
 RUN mkdir -vp /usr/local/etc/cic
 
-COPY package.json \
+COPY data-seeding/package.json \
-     package-lock.json \
+     data-seeding/package-lock.json \
      .
 
 RUN npm install
 
-COPY requirements.txt . 
+COPY data-seeding/requirements.txt . 
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
@@ -19,6 +19,6 @@ RUN pip install \
     --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL -r requirements.txt
 
-COPY . . 
+COPY data-seeding/ . 
 
 ENTRYPOINT [ ]

ci_templates/.cic-template.yml

@@ -3,29 +3,30 @@ image:
   entrypoint: [""]
 
 variables:
-  KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
+  KANIKO_CACHE_ARGS: "--cache=false --cache-copy-layers=true --cache-ttl=24h"
-  MR_IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA
+  CONTEXT: $CI_PROJECT_DIR/apps/
 
 .py_build_merge_request:
   stage: build
+  variables:
+    CI_DEBUG_TRACE: "true"
   script:
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    - > 
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push
-      /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS
+  rules:
-      --cache-repo $CI_REGISTRY_IMAGE --destination $MR_IMAGE_TAG 
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
 
-.py_build_target_dev:
+.py_build_target_test:
   stage: build
   variables:
     IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA
   script:
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    - > 
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --destination $IMAGE_TAG_BASE
-      /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS
+
-      --cache-repo $CI_REGISTRY_IMAGE --target dev  --skip-unused-stages 
-      --destination $MR_IMAGE_TAG
 
 .py_build_push:
   stage: build
@@ -38,6 +39,9 @@ variables:
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
     # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
     - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
+  rules:
+      - if: $CI_COMMIT_BRANCH == "master"
+        when: always
 
 
 

docker-compose.yml

@@ -72,11 +72,12 @@ services:
 
   contract-migration:
     build:
-      context: apps/contract-migration
+      dockerfile: contract-migration/docker/Dockerfile
-      dockerfile: docker/Dockerfile
+        #      target: compile-image
       args:
         pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
         pip_extra_args: $PIP_EXTRA_ARGS
+      context: apps/
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
       # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
@@ -124,8 +125,8 @@ services:
 
   cic-cache-tracker:
     build:
-      context: apps/cic-cache
+      context: apps
-      dockerfile: docker/Dockerfile
+      dockerfile: cic-cache/docker/Dockerfile
     environment:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
       ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
@@ -160,8 +161,8 @@ services:
 
   cic-cache-tasker:
     build:
-      context: apps/cic-cache
+      context: apps
-      dockerfile: docker/Dockerfile
+      dockerfile: cic-cache/docker/Dockerfile
     environment:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
       ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
@@ -196,9 +197,9 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-server:
-    build:
+    build: 
-      context: apps/cic-cache
+        context: apps
-      dockerfile: docker/Dockerfile
+        dockerfile: cic-cache/docker/Dockerfile
     environment:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
@@ -229,9 +230,8 @@ services:
   cic-eth-tasker:
     # image: grassrootseconomics:cic-eth-service
     build: 
-        context: apps/cic-eth
+        context: apps/
-        dockerfile: docker/Dockerfile
+        dockerfile: cic-eth/docker/Dockerfile
-        target: dev
     environment:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
       ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER
@@ -279,9 +279,8 @@ services:
 
   cic-eth-tracker:
     build:
-        context: apps/cic-eth
+        context: apps/
-        dockerfile: docker/Dockerfile
+        dockerfile: cic-eth/docker/Dockerfile
-        target: dev
     environment:
       ETH_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
@@ -318,9 +317,8 @@ services:
 
   cic-eth-dispatcher:
     build:
-        context: apps/cic-eth
+        context: apps/
-        dockerfile: docker/Dockerfile
+        dockerfile: cic-eth/docker/Dockerfile
-        target: dev
     environment:
       ETH_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
@@ -359,9 +357,8 @@ services:
 
   cic-eth-retrier:
     build:
-        context: apps/cic-eth
+        context: apps/
-        dockerfile: docker/Dockerfile
+        dockerfile: cic-eth/docker/Dockerfile
-        target: dev
     environment:
       ETH_PROVIDER: http://eth:8545
       DATABASE_USER: ${DATABASE_USER:-grassroots}
@@ -402,8 +399,8 @@ services:
 
   cic-notify-tasker:
     build:
-      context: apps/cic-notify
+      context: apps/
-      dockerfile: docker/Dockerfile
+      dockerfile: cic-notify/docker/Dockerfile
     environment:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
@@ -431,8 +428,8 @@ services:
   cic-meta-server:
     hostname: meta
     build:
-      context: apps/cic-meta
+      context: apps/
-      dockerfile: docker/Dockerfile
+      dockerfile: cic-meta/docker/Dockerfile
     environment:
       DATABASE_NAME: ${DATABASE_NAME:-cic_meta}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
@@ -463,8 +460,8 @@ services:
 
   cic-user-ussd-server:
     build:
-      context: apps/cic-ussd
+      context: apps/
-      dockerfile: docker/Dockerfile
+      dockerfile: cic-ussd/docker/Dockerfile
     environment:
       DATABASE_USER: grassroots
       DATABASE_HOST: postgres
@@ -492,8 +489,8 @@ services:
 
   cic-user-server:
     build:
-      context: apps/cic-ussd
+      context: apps
-      dockerfile: docker/Dockerfile
+      dockerfile: cic-ussd/docker/Dockerfile
     environment:
       DATABASE_USER: grassroots
       DATABASE_HOST: postgres
@@ -514,8 +511,8 @@ services:
 
   cic-user-tasker:
     build:
-      context: apps/cic-ussd/
+      context: apps
-      dockerfile: docker/Dockerfile
+      dockerfile: cic-ussd/docker/Dockerfile
     environment:
       DATABASE_USER: grassroots
       DATABASE_HOST: postgres