diff --git a/.gitignore b/.gitignore index dabaefe2..3fb25a37 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ build/ **/coverage **/.venv .idea +**/.vim diff --git a/apps/cic-cache/.coveragerc b/apps/cic-cache/.coveragerc index c6fef2de..a8162630 100644 --- a/apps/cic-cache/.coveragerc +++ b/apps/cic-cache/.coveragerc @@ -2,4 +2,5 @@ omit = .venv/* scripts/* - cic_cache/db/postgres/* + cic_cache/db/migrations/* + cic_cache/version.py diff --git a/apps/cic-cache/.dockerignore b/apps/cic-cache/.dockerignore new file mode 100644 index 00000000..a58c5dc8 --- /dev/null +++ b/apps/cic-cache/.dockerignore @@ -0,0 +1,4 @@ +.git +.cache +.dot +**/doc \ No newline at end of file diff --git a/apps/cic-cache/.gitlab-ci.yml b/apps/cic-cache/.gitlab-ci.yml index e39cb189..c88a72d8 100644 --- a/apps/cic-cache/.gitlab-ci.yml +++ b/apps/cic-cache/.gitlab-ci.yml @@ -1,22 +1,52 @@ .cic_cache_variables: variables: APP_NAME: cic-cache - DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile - -.cic_cache_changes_target: - rules: - - changes: - - $CONTEXT/$APP_NAME/* + DOCKERFILE_PATH: docker/Dockerfile_ci + CONTEXT: apps/$APP_NAME build-mr-cic-cache: extends: - - .cic_cache_changes_target - .py_build_merge_request - .cic_cache_variables + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-cache/**/* + when: always + +test-mr-cic-cache: + stage: test + extends: + - .cic_cache_variables + cache: + key: + files: + - test_requirements.txt + paths: + - /root/.cache/pip + image: $MR_IMAGE_TAG + script: + - cd apps/$APP_NAME/ + - > + pip install --extra-index-url https://pip.grassrootseconomics.net:8433 + --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple + -r test_requirements.txt + - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests + needs: ["build-mr-cic-cache"] + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-eth/**/* + when: always build-push-cic-cache: extends: - .py_build_push - .cic_cache_variables + rules: + - if: $CI_COMMIT_BRANCH == "master" + changes: + - apps/cic-cache/**/* + when: always diff --git a/apps/cic-cache/cic_cache/cache.py b/apps/cic-cache/cic_cache/cache.py index 99a99984..7c4d1040 100644 --- a/apps/cic-cache/cic_cache/cache.py +++ b/apps/cic-cache/cic_cache/cache.py @@ -15,6 +15,8 @@ from cic_cache.db.list import ( logg = logging.getLogger() +DEFAULT_FILTER_SIZE = 8192 * 8 + class Cache: def __init__(self, session): @@ -25,7 +27,7 @@ class BloomCache(Cache): @staticmethod def __get_filter_size(n): - n = 8192 * 8 + n = DEFAULT_FILTER_SIZE logg.warning('filter size hardcoded to {}'.format(n)) return n diff --git a/apps/cic-cache/cic_cache/runnable/__init__.py b/apps/cic-cache/cic_cache/runnable/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/cic-cache/cic_cache/runnable/daemons/__init__.py b/apps/cic-cache/cic_cache/runnable/daemons/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/cic-cache/cic_cache/runnable/daemons/query.py b/apps/cic-cache/cic_cache/runnable/daemons/query.py index b2576690..a698a692 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/query.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py @@ -4,6 +4,9 @@ import json import re import base64 +# external imports +from hexathon import add_0x + # local imports from cic_cache.cache import ( BloomCache, @@ -11,10 +14,11 @@ from cic_cache.cache import ( ) logg = logging.getLogger(__name__) +#logg = logging.getLogger() re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?' -re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?' -re_transactions_all_data = r'/txa/(\d+)/(\d+)/?' +re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?' +re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?' DEFAULT_LIMIT = 100 @@ -26,13 +30,13 @@ def process_transactions_account_bloom(session, env): address = r[1] if r[2] == None: - address = '0x' + address - offset = DEFAULT_LIMIT + address = add_0x(address) + offset = 0 if r.lastindex > 2: - offset = r[3] - limit = 0 - if r.lastindex > 3: - limit = r[4] + offset = r[4] + limit = DEFAULT_LIMIT + if r.lastindex > 4: + limit = r[6] c = BloomCache(session) (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit) diff --git a/apps/cic-cache/docker/Dockerfile b/apps/cic-cache/docker/Dockerfile index c4c783dd..2aa7e311 100644 --- a/apps/cic-cache/docker/Dockerfile +++ b/apps/cic-cache/docker/Dockerfile @@ -1,52 +1,38 @@ -FROM python:3.8.6-slim-buster +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev + +# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9 -#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/ +COPY requirements.txt . +#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt +#RUN pip install $pip_extra_index_url_flag . +#RUN pip install .[server] -WORKDIR /usr/src/cic-cache +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + -r requirements.txt -ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433' -ARG root_requirement_file='requirements.txt' +COPY . . -#RUN apk update && \ -# apk add gcc musl-dev gnupg libpq -#RUN apk add postgresql-dev -#RUN apk add linux-headers -#RUN apk add libffi-dev -RUN apt-get update && \ - apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git - -# Copy shared requirements from top of mono-repo -RUN echo "copying root req file ${root_requirement_file}" -RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9 - -COPY cic-cache/requirements.txt ./ -COPY cic-cache/setup.cfg \ - cic-cache/setup.py \ - ./ -COPY cic-cache/cic_cache/ ./cic_cache/ -COPY cic-cache/scripts/ ./scripts/ -COPY cic-cache/test_requirements.txt ./ -RUN pip install $pip_extra_index_url_flag -r test_requirements.txt -RUN pip install $pip_extra_index_url_flag . -RUN pip install .[server] - -COPY cic-cache/tests/ ./tests/ -#COPY db/ cic-cache/db -#RUN apk add postgresql-client +RUN python setup.py install # ini files in config directory defines the configurable parameters for the application # they can all be overridden by environment variables # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -COPY cic-cache/config/ /usr/local/etc/cic-cache/ +COPY config/ /usr/local/etc/cic-cache/ # for db migrations RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/ -COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/ +COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/ -COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh -COPY cic-cache/docker/db.sh ./db.sh +COPY /docker/start_tracker.sh ./start_tracker.sh +COPY /docker/db.sh ./db.sh RUN chmod 755 ./*.sh # Tracker # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"] # Server # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ] +ENTRYPOINT [] diff --git a/apps/cic-cache/docker/Dockerfile_ci b/apps/cic-cache/docker/Dockerfile_ci new file mode 100644 index 00000000..9f49cac0 --- /dev/null +++ b/apps/cic-cache/docker/Dockerfile_ci @@ -0,0 +1,37 @@ +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev + +# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9 + +COPY requirements.txt . +#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt +#RUN pip install $pip_extra_index_url_flag . +#RUN pip install .[server] + +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +RUN pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + -r requirements.txt + +COPY . . + +RUN python setup.py install + +# ini files in config directory defines the configurable parameters for the application +# they can all be overridden by environment variables +# to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) +COPY config/ /usr/local/etc/cic-cache/ + +# for db migrations +RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/ +COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/ + +COPY /docker/start_tracker.sh ./start_tracker.sh +COPY /docker/db.sh ./db.sh +RUN chmod 755 ./*.sh +# Tracker +# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"] +# Server +# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ] +ENTRYPOINT [] diff --git a/apps/cic-cache/tests/conftest.py b/apps/cic-cache/tests/conftest.py index f5e8edd4..61db702f 100644 --- a/apps/cic-cache/tests/conftest.py +++ b/apps/cic-cache/tests/conftest.py @@ -5,9 +5,12 @@ import datetime # external imports import pytest +import moolb # local imports from cic_cache import db +from cic_cache import BloomCache +from cic_cache.cache import DEFAULT_FILTER_SIZE script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.dirname(script_dir) @@ -101,3 +104,7 @@ def tag_txs( db.tag_transaction(init_database, txs[1], 'taag', domain='test') + +@pytest.fixture(scope='session') +def zero_filter(): + return moolb.Bloom(DEFAULT_FILTER_SIZE, 3) diff --git a/apps/cic-cache/tests/filters/test_erc20.py b/apps/cic-cache/tests/filters/test_erc20.py index fb44e29a..0a11e2f6 100644 --- a/apps/cic-cache/tests/filters/test_erc20.py +++ b/apps/cic-cache/tests/filters/test_erc20.py @@ -10,6 +10,7 @@ from sqlalchemy import text from chainlib.eth.tx import Tx from chainlib.eth.block import Block from chainlib.chain import ChainSpec +from chainlib.eth.error import RequestMismatchException from hexathon import ( strip_0x, add_0x, @@ -18,10 +19,21 @@ from hexathon import ( # local imports from cic_cache.db import add_tag from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter +from cic_cache.runnable.daemons.filters.base import TagSyncFilter logg = logging.getLogger() +def test_base_filter_str( + init_database, + ): + f = TagSyncFilter('foo') + assert 'foo' == str(f) + f = TagSyncFilter('foo', domain='bar') + assert 'bar.foo' == str(f) + + + def test_erc20_filter( eth_rpc, foo_token, @@ -67,3 +79,95 @@ def test_erc20_filter( s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b") r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone() assert r[0] == tx.hash + + +def test_erc20_filter_nocontract( + eth_rpc, + foo_token, + init_database, + list_defaults, + list_actors, + tags, + ): + + chain_spec = ChainSpec('foo', 'bar', 42, 'baz') + + fltr = ERC20TransferFilter(chain_spec) + add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain) + + # incomplete args + data = 'a9059cbb' + data += strip_0x(list_actors['alice']) + data += '1000'.ljust(64, '0') + block = Block({ + 'hash': os.urandom(32).hex(), + 'number': 42, + 'timestamp': datetime.datetime.utcnow().timestamp(), + 'transactions': [], + }) + + tx = Tx({ + 'to': os.urandom(20).hex(), + 'from': list_actors['bob'], + 'data': data, + 'value': 0, + 'hash': os.urandom(32).hex(), + 'nonce': 13, + 'gasPrice': 10000000, + 'gas': 123456, + }) + block.txs.append(tx) + tx.block = block + + assert not fltr.filter(eth_rpc, block, tx, db_session=init_database) + + +@pytest.mark.parametrize( + 'contract_method,contract_input,expected_exception', + [ + ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args + ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary + ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method + ], + ) +def test_erc20_filter_bogus( + eth_rpc, + foo_token, + init_database, + list_defaults, + list_actors, + tags, + contract_method, + contract_input, + expected_exception, + ): + + chain_spec = ChainSpec('foo', 'bar', 42, 'baz') + + fltr = ERC20TransferFilter(chain_spec) + add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain) + + # incomplete args + data = contract_method + data += contract_input + block = Block({ + 'hash': os.urandom(32).hex(), + 'number': 42, + 'timestamp': datetime.datetime.utcnow().timestamp(), + 'transactions': [], + }) + + tx = Tx({ + 'to': foo_token, + 'from': list_actors['bob'], + 'data': data, + 'value': 0, + 'hash': os.urandom(32).hex(), + 'nonce': 13, + 'gasPrice': 10000000, + 'gas': 123456, + }) + block.txs.append(tx) + tx.block = block + + assert not fltr.filter(eth_rpc, block, tx, db_session=init_database) diff --git a/apps/cic-cache/tests/test_query.py b/apps/cic-cache/tests/test_query.py new file mode 100644 index 00000000..fcc06e2f --- /dev/null +++ b/apps/cic-cache/tests/test_query.py @@ -0,0 +1,230 @@ +# standard imports +import logging +import json +import base64 +import copy +import re + +# external imports +import pytest +from hexathon import strip_0x + +# local imports +from cic_cache.runnable.daemons.query import * + +logg = logging.getLogger() + + + +@pytest.mark.parametrize( + 'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re', + [ + ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom), + ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom), + ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom), + ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom), + ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom), + ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom), + ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data), + ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data), + ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data), + ], + ) +def test_query_regex( + list_actors, + query_path_prefix, + query_role, + query_address_index, + query_offset, + query_offset_index, + query_limit, + query_limit_index, + match_re, + ): + + paths = [] + path = query_path_prefix + query_address = None + if query_role != None: + query_address = strip_0x(list_actors[query_role]) + paths.append(path + '0x' + query_address) + paths.append(path + query_address) + if query_offset != None: + if query_limit != None: + for i in range(len(paths)-1): + paths[i] += '/{}/{}'.format(query_offset, query_limit) + else: + for i in range(len(paths)-1): + paths[i] += '/' + str(query_offset) + + for i in range(len(paths)): + paths.append(paths[i] + '/') + + for p in paths: + logg.debug('testing path {} against {}'.format(p, match_re)) + m = re.match(match_re, p) + l = len(m.groups()) + logg.debug('laast index match {} groups {}'.format(m.lastindex, l)) + for i in range(l+1): + logg.debug('group {} {}'.format(i, m[i])) + if m.lastindex >= query_offset_index: + assert query_offset == int(m[query_offset_index + 1]) + if m.lastindex >= query_limit_index: + assert query_limit == int(m[query_limit_index + 1]) + if query_address_index != None: + match_address = strip_0x(m[query_address_index + 1]) + assert query_address == match_address + + + +@pytest.mark.parametrize( + 'role_name, query_offset, query_limit, query_match', + [ + ('alice', None, None, [(420000, 13), (419999, 42)]), + ('alice', None, 1, [(420000, 13)]), + ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block'] + ('alice', 2, None, []), # 420000 == list_defaults['block'] + ], + ) +def test_query_process_txs_account( + init_database, + list_defaults, + list_actors, + list_tokens, + txs, + zero_filter, + role_name, + query_offset, + query_limit, + query_match, + ): + + actor = None + try: + actor = list_actors[role_name] + except KeyError: + actor = os.urandom(20).hex() + path_info = '/tx/user/0x' + strip_0x(actor) + if query_offset != None: + path_info += '/' + str(query_offset) + if query_limit != None: + if query_offset == None: + path_info += '/0' + path_info += '/' + str(query_limit) + env = { + 'PATH_INFO': path_info, + } + logg.debug('using path {}'.format(path_info)) + r = process_transactions_account_bloom(init_database, env) + assert r != None + + o = json.loads(r[1]) + block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8')) + zero_filter_data = zero_filter.to_bytes() + if len(query_match) == 0: + assert block_filter_data == zero_filter_data + return + + assert block_filter_data != zero_filter_data + block_filter = copy.copy(zero_filter) + block_filter.merge(block_filter_data) + block_filter_data = block_filter.to_bytes() + assert block_filter_data != zero_filter_data + + for (block, tx) in query_match: + block = block.to_bytes(4, byteorder='big') + assert block_filter.check(block) + + +@pytest.mark.parametrize( + 'query_offset, query_limit, query_match', + [ + (None, 2, [(420000, 13), (419999, 42)]), + (0, 1, [(420000, 13)]), + (1, 1, [(419999, 42)]), + (2, 0, []), + ], + ) +def test_query_process_txs_bloom( + init_database, + list_defaults, + list_actors, + list_tokens, + txs, + zero_filter, + query_offset, + query_limit, + query_match, + ): + + path_info = '/tx' + if query_offset != None: + path_info += '/' + str(query_offset) + if query_limit != None: + if query_offset == None: + path_info += '/0' + path_info += '/' + str(query_limit) + env = { + 'PATH_INFO': path_info, + } + logg.debug('using path {}'.format(path_info)) + r = process_transactions_all_bloom(init_database, env) + assert r != None + + o = json.loads(r[1]) + block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8')) + zero_filter_data = zero_filter.to_bytes() + if len(query_match) == 0: + assert block_filter_data == zero_filter_data + return + + assert block_filter_data != zero_filter_data + block_filter = copy.copy(zero_filter) + block_filter.merge(block_filter_data) + block_filter_data = block_filter.to_bytes() + assert block_filter_data != zero_filter_data + + for (block, tx) in query_match: + block = block.to_bytes(4, byteorder='big') + assert block_filter.check(block) + + +@pytest.mark.parametrize( + 'query_block_start, query_block_end, query_match_count', + [ + (None, 42, 0), + (420000, 420001, 1), + (419999, 419999, 1), # matches are inclusive + (419999, 420000, 2), + (419999, 420001, 2), + ], + ) +def test_query_process_txs_data( + init_database, + list_defaults, + list_actors, + list_tokens, + txs, + zero_filter, + query_block_start, + query_block_end, + query_match_count, + ): + + path_info = '/txa' + if query_block_start != None: + path_info += '/' + str(query_block_start) + if query_block_end != None: + if query_block_start == None: + path_info += '/0' + path_info += '/' + str(query_block_end) + env = { + 'PATH_INFO': path_info, + 'HTTP_X_CIC_CACHE_MODE': 'all', + } + logg.debug('using path {}'.format(path_info)) + r = process_transactions_all_data(init_database, env) + assert r != None + + o = json.loads(r[1]) + assert len(o['data']) == query_match_count diff --git a/apps/cic-eth/.dockerignore b/apps/cic-eth/.dockerignore new file mode 100644 index 00000000..9aae4e0e --- /dev/null +++ b/apps/cic-eth/.dockerignore @@ -0,0 +1,6 @@ +.git +.cache +.dot +**/doc +**/.venv +**/venv diff --git a/apps/cic-eth/.gitlab-ci.yml b/apps/cic-eth/.gitlab-ci.yml index e613f6b7..ef2b7462 100644 --- a/apps/cic-eth/.gitlab-ci.yml +++ b/apps/cic-eth/.gitlab-ci.yml @@ -1,31 +1,52 @@ .cic_eth_variables: variables: APP_NAME: cic-eth - DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile - -.cic_eth_mr_changes_target: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - $CONTEXT/$APP_NAME/**/* - when: always + DOCKERFILE_PATH: docker/Dockerfile_ci + CONTEXT: apps/$APP_NAME build-mr-cic-eth: extends: - .cic_eth_variables - - .cic_eth_mr_changes_target - - .py_build_target_test + - .py_build_target_dev + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-eth/**/* + when: always test-mr-cic-eth: + stage: test extends: - .cic_eth_variables - - .cic_eth_mr_changes_target - stage: test - image: $IMAGE_TAG_BASE + cache: + key: + files: + - test_requirements.txt + paths: + - /root/.cache/pip + image: $MR_IMAGE_TAG script: - cd apps/$APP_NAME/ - - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests + - > + pip install --extra-index-url https://pip.grassrootseconomics.net:8433 + --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple + -r admin_requirements.txt + -r services_requirements.txt + -r test_requirements.txt + - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests + needs: ["build-mr-cic-eth"] + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-eth/**/* + when: always build-push-cic-eth: extends: - .py_build_push + - .cic_eth_variables + rules: + - if: $CI_COMMIT_BRANCH == "master" + changes: + - apps/cic-eth/**/* + when: always diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 8e43464b..7e117c63 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -1,58 +1,39 @@ -FROM python:3.8.6-slim-buster as compile +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev -WORKDIR /usr/src - -RUN apt-get update && \ - apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git - -#RUN python -m venv venv && . venv/bin/activate - -ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433' -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -RUN /usr/local/bin/python -m pip install --upgrade pip -RUN pip install semver - -COPY cic-eth-aux/ ./cic-eth-aux/ -WORKDIR /usr/src/cic-eth-aux/erc20-demurrage-token -RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL . - -WORKDIR /usr/src/cic-eth - -COPY cic-eth/ . -RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL .[services] - -# --- TEST IMAGE --- -FROM python:3.8.6-slim-buster as test - -RUN apt-get update && \ - apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git - -WORKDIR /usr/src/cic-eth - -RUN /usr/local/bin/python -m pip install --upgrade pip - -COPY --from=compile /usr/local/bin/ /usr/local/bin/ -COPY --from=compile /usr/local/lib/python3.8/site-packages/ \ - /usr/local/lib/python3.8/site-packages/ -# TODO we could use venv inside container to isolate the system and app deps further -# COPY --from=compile /usr/src/cic-eth/ . -# RUN . venv/bin/activate +# Copy just the requirements and install....this _might_ give docker a hint on caching but we +# do load these all into setup.py later +# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py +#COPY cic-eth/requirements.txt . ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ +# pip install --index-url https://pypi.org/simple \ +# --force-reinstall \ +# --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ +# -r requirements.txt -COPY cic-eth/test_requirements.txt . -RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt - -COPY cic-eth . +COPY . . +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL . ENV PYTHONPATH . -ENTRYPOINT ["pytest"] +COPY docker/entrypoints/* ./ +RUN chmod 755 *.sh + +# # ini files in config directory defines the configurable parameters for the application +# # they can all be overridden by environment variables +# # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) +COPY config/ /usr/local/etc/cic-eth/ +COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ +COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ + +# TODO this kind of code sharing across projects should be discouraged...can we make util a library? +#COPY util/liveness/health.sh /usr/local/bin/health.sh +ENTRYPOINT [] # --- RUNTIME --- FROM python:3.8.6-slim-buster as runtime @@ -62,20 +43,22 @@ RUN apt-get update && \ WORKDIR /usr/src/cic-eth -COPY --from=compile /usr/local/bin/ /usr/local/bin/ -COPY --from=compile /usr/local/lib/python3.8/site-packages/ \ +COPY --from=dev /usr/local/bin/ /usr/local/bin/ +COPY --from=dev /usr/local/lib/python3.8/site-packages/ \ /usr/local/lib/python3.8/site-packages/ -COPY cic-eth/docker/* ./ +COPY docker/entrypoints/* ./ RUN chmod 755 *.sh -COPY cic-eth/scripts/ scripts/ # # ini files in config directory defines the configurable parameters for the application # # they can all be overridden by environment variables # # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -COPY cic-eth/config/ /usr/local/etc/cic-eth/ -COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ -COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ +COPY config/ /usr/local/etc/cic-eth/ +COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ +COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ -COPY util/liveness/health.sh /usr/local/bin/health.sh +# TODO this kind of code sharing across projects should be discouraged...can we make util a library? +#COPY util/liveness/health.sh /usr/local/bin/health.sh + +ENTRYPOINT [] diff --git a/apps/cic-eth/docker/Dockerfile_ci b/apps/cic-eth/docker/Dockerfile_ci new file mode 100644 index 00000000..cbdb7fb0 --- /dev/null +++ b/apps/cic-eth/docker/Dockerfile_ci @@ -0,0 +1,62 @@ +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev + +WORKDIR /usr/src/cic-eth + +# Copy just the requirements and install....this _might_ give docker a hint on caching but we +# do load these all into setup.py later +# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py +#COPY cic-eth/requirements.txt . + +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ +# pip install --index-url https://pypi.org/simple \ +# --force-reinstall \ +# --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ +# -r requirements.txt + +COPY . . +RUN pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL . + +COPY docker/entrypoints/* ./ +RUN chmod 755 *.sh + +# # ini files in config directory defines the configurable parameters for the application +# # they can all be overridden by environment variables +# # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) +COPY config/ /usr/local/etc/cic-eth/ +COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ +COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ + +# TODO this kind of code sharing across projects should be discouraged...can we make util a library? +#COPY util/liveness/health.sh /usr/local/bin/health.sh +ENTRYPOINT [] + +# --- RUNTIME --- +FROM python:3.8.6-slim-buster as runtime + +RUN apt-get update && \ + apt install -y gnupg libpq-dev procps + +WORKDIR /usr/src/cic-eth + +COPY --from=dev /usr/local/bin/ /usr/local/bin/ +COPY --from=dev /usr/local/lib/python3.8/site-packages/ \ + /usr/local/lib/python3.8/site-packages/ + +COPY docker/entrypoints/* ./ +RUN chmod 755 *.sh + +# # ini files in config directory defines the configurable parameters for the application +# # they can all be overridden by environment variables +# # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) +COPY config/ /usr/local/etc/cic-eth/ +COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ +COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ + +# TODO this kind of code sharing across projects should be discouraged...can we make util a library? +#COPY util/liveness/health.sh /usr/local/bin/health.sh + +ENTRYPOINT [] + diff --git a/apps/cic-eth/docker/db.sh b/apps/cic-eth/docker/entrypoints/db.sh similarity index 100% rename from apps/cic-eth/docker/db.sh rename to apps/cic-eth/docker/entrypoints/db.sh diff --git a/apps/cic-eth/docker/start_dispatcher.sh b/apps/cic-eth/docker/entrypoints/start_dispatcher.sh similarity index 100% rename from apps/cic-eth/docker/start_dispatcher.sh rename to apps/cic-eth/docker/entrypoints/start_dispatcher.sh diff --git a/apps/cic-eth/docker/start_retry.sh b/apps/cic-eth/docker/entrypoints/start_retry.sh similarity index 100% rename from apps/cic-eth/docker/start_retry.sh rename to apps/cic-eth/docker/entrypoints/start_retry.sh diff --git a/apps/cic-eth/docker/start_tasker.sh b/apps/cic-eth/docker/entrypoints/start_tasker.sh similarity index 100% rename from apps/cic-eth/docker/start_tasker.sh rename to apps/cic-eth/docker/entrypoints/start_tasker.sh diff --git a/apps/cic-eth/docker/start_tracker.sh b/apps/cic-eth/docker/entrypoints/start_tracker.sh similarity index 100% rename from apps/cic-eth/docker/start_tracker.sh rename to apps/cic-eth/docker/entrypoints/start_tracker.sh diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index e7842d75..cab48e40 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -10,5 +10,6 @@ eth-address-index~=0.1.2a1 eth-accounts-index~=0.0.12a1 cic-eth-registry~=0.5.6a1 erc20-faucet~=0.2.2a1 +erc20-transfer-authorization~=0.3.2a1 sarafu-faucet~=0.0.4a1 moolb~=0.1.1b2 diff --git a/apps/cic-meta/.dockerignore b/apps/cic-meta/.dockerignore new file mode 100644 index 00000000..a58c5dc8 --- /dev/null +++ b/apps/cic-meta/.dockerignore @@ -0,0 +1,4 @@ +.git +.cache +.dot +**/doc \ No newline at end of file diff --git a/apps/cic-meta/.gitlab-ci.yml b/apps/cic-meta/.gitlab-ci.yml index d1ed979c..ac157031 100644 --- a/apps/cic-meta/.gitlab-ci.yml +++ b/apps/cic-meta/.gitlab-ci.yml @@ -3,42 +3,41 @@ variables: APP_NAME: cic-meta DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile - IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA + CONTEXT: apps -.cic_meta_changes_target: +build-mr-cic-meta: + extends: + - .py_build_merge_request + - .cic_meta_variables rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" -# - changes: -# - $CONTEXT/$APP_NAME/* - - when: always - -cic-meta-build-mr: - stage: build - extends: - - .cic_meta_variables - - .cic_meta_changes_target - script: - - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" - # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG - - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-meta/**/* + when: always test-mr-cic-meta: extends: - .cic_meta_variables - - .cic_meta_changes_target stage: test - image: $IMAGE_TAG + image: $MR_IMAGE_TAG script: - cd /tmp/src/cic-meta - npm install --dev - npm run test - npm run test:coverage - needs: ["cic-meta-build-mr"] + needs: ["build-mr-cic-meta"] + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-meta/**/* + when: always build-push-cic-meta: extends: - .py_build_push - .cic_meta_variables - - + rules: + - if: $CI_COMMIT_BRANCH == "master" + changes: + - apps/cic-meta/**/* + when: always diff --git a/apps/cic-notify/.gitlab-ci.yml b/apps/cic-notify/.gitlab-ci.yml index 34695d7f..a414b9bf 100644 --- a/apps/cic-notify/.gitlab-ci.yml +++ b/apps/cic-notify/.gitlab-ci.yml @@ -1,22 +1,52 @@ .cic_notify_variables: variables: APP_NAME: cic-notify - DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile - -.cic_notify_changes_target: - rules: - - changes: - - $CONTEXT/$APP_NAME/* + DOCKERFILE_PATH: docker/Dockerfile_ci + CONTEXT: apps/$APP_NAME build-mr-cic-notify: extends: - - .cic_notify_changes_target - .py_build_merge_request - .cic_notify_variables + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-notify/**/* + when: always + +test-mr-cic-notify: + stage: test + extends: + - .cic_notify_variables + cache: + key: + files: + - test_requirements.txt + paths: + - /root/.cache/pip + image: $MR_IMAGE_TAG + script: + - cd apps/$APP_NAME/ + - > + pip install --extra-index-url https://pip.grassrootseconomics.net:8433 + --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple + -r test_requirements.txt + - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests + needs: ["build-mr-cic-notify"] + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-eth/**/* + when: always build-push-cic-notify: extends: - .py_build_push - .cic_notify_variables + rules: + - if: $CI_COMMIT_BRANCH == "master" + changes: + - apps/cic-notify/**/* + when: always diff --git a/apps/cic-notify/docker/Dockerfile b/apps/cic-notify/docker/Dockerfile index ea00a238..36b09e18 100644 --- a/apps/cic-notify/docker/Dockerfile +++ b/apps/cic-notify/docker/Dockerfile @@ -1,38 +1,30 @@ -FROM python:3.8.6-slim-buster +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev -RUN apt-get update && \ - apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps +#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 -WORKDIR /usr/src/cic-notify +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +COPY requirements.txt . -ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433' -RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + -r requirements.txt -COPY cic-notify/setup.cfg \ - cic-notify/setup.py \ - ./ +COPY . . -COPY cic-notify/cic_notify/ ./cic_notify/ +RUN python setup.py install -COPY cic-notify/requirements.txt \ - cic-notify/test_requirements.txt \ - ./ - -COPY cic-notify/scripts/ scripts/ +# TODO please review..can this go into requirements? RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog] -COPY cic-notify/tests/ tests/ -COPY cic-notify/docker/db.sh \ - cic-notify/docker/start_tasker.sh \ - /root/ - -#RUN apk add postgresql-client -#RUN apk add bash +COPY docker/*.sh . # ini files in config directory defines the configurable parameters for the application # they can all be overridden by environment variables # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -COPY cic-notify/.config/ /usr/local/etc/cic-notify/ -COPY cic-notify/cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/ +COPY .config/ /usr/local/etc/cic-notify/ +COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/ -WORKDIR /root +ENTRYPOINT [] diff --git a/apps/cic-notify/docker/Dockerfile_ci b/apps/cic-notify/docker/Dockerfile_ci new file mode 100644 index 00000000..5c14c05d --- /dev/null +++ b/apps/cic-notify/docker/Dockerfile_ci @@ -0,0 +1,29 @@ +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev + +#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 + +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +COPY requirements.txt . + +RUN pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + -r requirements.txt + +COPY . . + +RUN python setup.py install + +# TODO please review..can this go into requirements? +RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog] + +COPY docker/*.sh . + +# ini files in config directory defines the configurable parameters for the application +# they can all be overridden by environment variables +# to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) +COPY .config/ /usr/local/etc/cic-notify/ +COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/ + +ENTRYPOINT [] diff --git a/apps/cic-ussd/.dockerignore b/apps/cic-ussd/.dockerignore new file mode 100644 index 00000000..a58c5dc8 --- /dev/null +++ b/apps/cic-ussd/.dockerignore @@ -0,0 +1,4 @@ +.git +.cache +.dot +**/doc \ No newline at end of file diff --git a/apps/cic-ussd/.gitlab-ci.yml b/apps/cic-ussd/.gitlab-ci.yml index 983fa19b..5a1b8742 100644 --- a/apps/cic-ussd/.gitlab-ci.yml +++ b/apps/cic-ussd/.gitlab-ci.yml @@ -1,22 +1,52 @@ .cic_ussd_variables: variables: APP_NAME: cic-ussd - DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile - -.cic_ussd_changes_target: - rules: - - changes: - - $CONTEXT/$APP_NAME/* + DOCKERFILE_PATH: docker/Dockerfile_ci + CONTEXT: apps/$APP_NAME build-mr-cic-ussd: extends: - - .cic_ussd_changes_target - .py_build_merge_request - .cic_ussd_variables + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-ussd/**/* + when: always + +test-mr-cic-ussd: + stage: test + extends: + - .cic_ussd_variables + cache: + key: + files: + - test_requirements.txt + paths: + - /root/.cache/pip + image: $MR_IMAGE_TAG + script: + - cd apps/$APP_NAME/ + - > + pip install --extra-index-url https://pip.grassrootseconomics.net:8433 + --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple + -r test_requirements.txt + - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests + needs: ["build-mr-cic-ussd"] + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/cic-eth/**/* + when: always build-push-cic-ussd: extends: - .py_build_push - .cic_ussd_variables + rules: + - if: $CI_COMMIT_BRANCH == "master" + changes: + - apps/cic-ussd/**/* + when: always diff --git a/apps/cic-ussd/docker/Dockerfile b/apps/cic-ussd/docker/Dockerfile index ecaa14dc..81ceb733 100644 --- a/apps/cic-ussd/docker/Dockerfile +++ b/apps/cic-ussd/docker/Dockerfile @@ -1,14 +1,5 @@ -# FROM python:3.8.5-alpine -FROM python:3.8.6-slim-buster - -# set working directory -WORKDIR /usr/src - -# add args for installing from self-hosted packages -ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433' - -RUN apt-get update && \ - apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev # create secrets directory RUN mkdir -vp pgp/keys @@ -17,39 +8,25 @@ RUN mkdir -vp pgp/keys RUN mkdir -vp cic-ussd RUN mkdir -vp data -COPY cic-ussd/setup.cfg \ - cic-ussd/setup.py \ - cic-ussd/ +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +COPY requirements.txt . -COPY cic-ussd/requirements.txt \ - cic-ussd/test_requirements.txt \ - cic-ussd/ +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + -r requirements.txt -# install requirements -RUN cd cic-ussd && \ - pip install -r requirements.txt $pip_extra_index_url_flag +COPY . . +RUN python setup.py install -# copy all necessary files -COPY cic-ussd/cic_ussd/ cic-ussd/cic_ussd/ -COPY cic-ussd/cic_ussd/db/ussd_menu.json data/ -COPY cic-ussd/scripts/ cic-ussd/scripts/ -COPY cic-ussd/states/ cic-ussd/states/ -COPY cic-ussd/transitions/ cic-ussd/transitions/ -COPY cic-ussd/var/ cic-ussd/var/ - -COPY cic-ussd/docker/db.sh \ - cic-ussd/docker/start_cic_user_tasker.sh \ - cic-ussd/docker/start_cic_user_ussd_server.sh\ - cic-ussd/docker/start_cic_user_server.sh\ - /root/ +COPY cic_ussd/db/ussd_menu.json data/ +COPY docker/*.sh . RUN chmod +x /root/*.sh -RUN cd cic-ussd && \ - pip install $pip_extra_index_url_flag . - # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts -COPY cic-ussd/.config/ /usr/local/etc/cic-ussd/ -COPY cic-ussd/cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic +COPY .config/ /usr/local/etc/cic-ussd/ +COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic -WORKDIR /root +ENTRYPOINT [] diff --git a/apps/cic-ussd/docker/Dockerfile_ci b/apps/cic-ussd/docker/Dockerfile_ci new file mode 100644 index 00000000..e431b264 --- /dev/null +++ b/apps/cic-ussd/docker/Dockerfile_ci @@ -0,0 +1,32 @@ +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev + + +# create secrets directory +RUN mkdir -vp pgp/keys + +# create application directory +RUN mkdir -vp cic-ussd +RUN mkdir -vp data + +COPY requirements.txt . + +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +RUN pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + -r requirements.txt + +COPY . . +RUN python setup.py install + +COPY cic_ussd/db/ussd_menu.json data/ + +COPY docker/*.sh . +RUN chmod +x /root/*.sh + +# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts +COPY .config/ /usr/local/etc/cic-ussd/ +COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic + +ENTRYPOINT [] diff --git a/apps/contract-migration/.dockerignore b/apps/contract-migration/.dockerignore new file mode 100644 index 00000000..a58c5dc8 --- /dev/null +++ b/apps/contract-migration/.dockerignore @@ -0,0 +1,4 @@ +.git +.cache +.dot +**/doc \ No newline at end of file diff --git a/apps/contract-migration/.gitlab-ci.yml b/apps/contract-migration/.gitlab-ci.yml index 39efdbfb..d48f7933 100644 --- a/apps/contract-migration/.gitlab-ci.yml +++ b/apps/contract-migration/.gitlab-ci.yml @@ -1,20 +1,25 @@ .contract_migration_variables: variables: APP_NAME: contract-migration - DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci - -.contract_migration_changes_target: - rules: - - changes: - - $CONTEXT/$APP_NAME/* + DOCKERFILE_PATH: docker/Dockerfile_ci + CONTEXT: apps/$APP_NAME build-mr-contract-migration: extends: - - .contract_migration_changes_target - .py_build_merge_request - .contract_migration_variables + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/contract-migration/**/* + when: always build-push-contract-migration: extends: - .py_build_push - .contract_migration_variables + rules: + - if: $CI_COMMIT_BRANCH == "master" + changes: + - apps/contract-migration/**/* + when: always diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index ffeb8dc5..2c588fd2 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -25,7 +25,7 @@ RUN echo Install confini schema files && \ git checkout $cic_config_commit && \ cp -v *.ini $CONFINI_DIR -COPY contract-migration/requirements.txt . +COPY requirements.txt . ARG pip_index_url=https://pypi.org/simple ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" @@ -38,5 +38,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ -r requirements.txt -COPY contract-migration/ . +COPY . . RUN chmod +x *.sh diff --git a/apps/contract-migration/docker/Dockerfile_ci b/apps/contract-migration/docker/Dockerfile_ci index cce1c3f8..4f384a87 100644 --- a/apps/contract-migration/docker/Dockerfile_ci +++ b/apps/contract-migration/docker/Dockerfile_ci @@ -3,12 +3,11 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55 WORKDIR /root -# solc install which we needed for bancor. Leaving as an artfact of HOW to do it. -#RUN touch /etc/apt/sources.list.d/ethereum.list -#RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list -#RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list -#RUN cat /etc/apt/sources.list.d/ethereum.list -#RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9 +RUN touch /etc/apt/sources.list.d/ethereum.list +RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list +RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list +RUN cat /etc/apt/sources.list.d/ethereum.list +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9 #RUN apt-get install solc @@ -26,7 +25,7 @@ RUN echo Install confini schema files && \ git checkout $cic_config_commit && \ cp -v *.ini $CONFINI_DIR -COPY contract-migration/requirements.txt . +COPY requirements.txt . ARG pip_index_url=https://pypi.org/simple ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" @@ -38,5 +37,5 @@ RUN pip install --index-url https://pypi.org/simple \ --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ -r requirements.txt -COPY contract-migration/ . +COPY . . RUN chmod +x *.sh diff --git a/apps/data-seeding/.dockerignore b/apps/data-seeding/.dockerignore new file mode 100644 index 00000000..a58c5dc8 --- /dev/null +++ b/apps/data-seeding/.dockerignore @@ -0,0 +1,4 @@ +.git +.cache +.dot +**/doc \ No newline at end of file diff --git a/apps/data-seeding/.gitlab-ci.yml b/apps/data-seeding/.gitlab-ci.yml index b0073e9e..1554b922 100644 --- a/apps/data-seeding/.gitlab-ci.yml +++ b/apps/data-seeding/.gitlab-ci.yml @@ -1,21 +1,26 @@ .data_seeding_variables: variables: APP_NAME: data-seeding - DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci - -.data_seeding_changes_target: - rules: - - changes: - - $CONTEXT/$APP_NAME/* + DOCKERFILE_PATH: docker/Dockerfile_ci + CONTEXT: apps/$APP_NAME build-mr-data-seeding: extends: - - .data_seeding_changes_target - .py_build_merge_request - .data_seeding_variables + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/data-seeding/**/* + when: always build-push-data-seeding: extends: - .py_build_push - .data_seeding_variables + rules: + - if: $CI_COMMIT_BRANCH == "master" + changes: + - apps/data-seeding/**/* + when: always diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index e965af1c..2d032960 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -5,13 +5,13 @@ WORKDIR /root RUN mkdir -vp /usr/local/etc/cic -COPY data-seeding/package.json \ - data-seeding/package-lock.json \ +COPY package.json \ + package-lock.json \ . RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install -COPY data-seeding/requirements.txt . +COPY requirements.txt . ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" @@ -19,6 +19,6 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \ --extra-index-url $GITLAB_PYTHON_REGISTRY \ --extra-index-url $EXTRA_INDEX_URL -r requirements.txt -COPY data-seeding/ . +COPY . . ENTRYPOINT [ ] diff --git a/apps/data-seeding/docker/Dockerfile_ci b/apps/data-seeding/docker/Dockerfile_ci index fbb02707..8169b4db 100644 --- a/apps/data-seeding/docker/Dockerfile_ci +++ b/apps/data-seeding/docker/Dockerfile_ci @@ -5,13 +5,13 @@ WORKDIR /root RUN mkdir -vp /usr/local/etc/cic -COPY data-seeding/package.json \ - data-seeding/package-lock.json \ +COPY package.json \ + package-lock.json \ . RUN npm install -COPY data-seeding/requirements.txt . +COPY requirements.txt . ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" @@ -19,6 +19,6 @@ RUN pip install \ --extra-index-url $GITLAB_PYTHON_REGISTRY \ --extra-index-url $EXTRA_INDEX_URL -r requirements.txt -COPY data-seeding/ . +COPY . . ENTRYPOINT [ ] diff --git a/ci_templates/.cic-template.yml b/ci_templates/.cic-template.yml index c1d7d9f9..5ba47394 100644 --- a/ci_templates/.cic-template.yml +++ b/ci_templates/.cic-template.yml @@ -3,30 +3,29 @@ image: entrypoint: [""] variables: - KANIKO_CACHE_ARGS: "--cache=false --cache-copy-layers=true --cache-ttl=24h" - CONTEXT: $CI_PROJECT_DIR/apps/ + KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h" + MR_IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA .py_build_merge_request: stage: build - variables: - CI_DEBUG_TRACE: "true" script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" - - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: always + - > + /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS + --cache-repo $CI_REGISTRY_IMAGE --destination $MR_IMAGE_TAG -.py_build_target_test: +.py_build_target_dev: stage: build variables: IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" - - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --destination $IMAGE_TAG_BASE - + - > + /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS + --cache-repo $CI_REGISTRY_IMAGE --target dev --skip-unused-stages + --destination $MR_IMAGE_TAG .py_build_push: stage: build @@ -39,9 +38,6 @@ variables: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always diff --git a/docker-compose.yml b/docker-compose.yml index d0e40931..f3f78c80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,12 +72,11 @@ services: contract-migration: build: - dockerfile: contract-migration/docker/Dockerfile - # target: compile-image + context: apps/contract-migration + dockerfile: docker/Dockerfile args: pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} pip_extra_args: $PIP_EXTRA_ARGS - context: apps/ # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest environment: # ETH_PROVIDER should be broken out into host/port but cic-eth expects this @@ -125,8 +124,8 @@ services: cic-cache-tracker: build: - context: apps - dockerfile: cic-cache/docker/Dockerfile + context: apps/cic-cache + dockerfile: docker/Dockerfile environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} @@ -161,8 +160,8 @@ services: cic-cache-tasker: build: - context: apps - dockerfile: cic-cache/docker/Dockerfile + context: apps/cic-cache + dockerfile: docker/Dockerfile environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} @@ -197,9 +196,9 @@ services: - contract-config:/tmp/cic/config/:ro cic-cache-server: - build: - context: apps - dockerfile: cic-cache/docker/Dockerfile + build: + context: apps/cic-cache + dockerfile: docker/Dockerfile environment: DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -230,8 +229,9 @@ services: cic-eth-tasker: # image: grassrootseconomics:cic-eth-service build: - context: apps/ - dockerfile: cic-eth/docker/Dockerfile + context: apps/cic-eth + dockerfile: docker/Dockerfile + target: dev environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER @@ -279,8 +279,9 @@ services: cic-eth-tracker: build: - context: apps/ - dockerfile: cic-eth/docker/Dockerfile + context: apps/cic-eth + dockerfile: docker/Dockerfile + target: dev environment: ETH_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} @@ -317,8 +318,9 @@ services: cic-eth-dispatcher: build: - context: apps/ - dockerfile: cic-eth/docker/Dockerfile + context: apps/cic-eth + dockerfile: docker/Dockerfile + target: dev environment: ETH_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} @@ -357,8 +359,9 @@ services: cic-eth-retrier: build: - context: apps/ - dockerfile: cic-eth/docker/Dockerfile + context: apps/cic-eth + dockerfile: docker/Dockerfile + target: dev environment: ETH_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} @@ -399,8 +402,8 @@ services: cic-notify-tasker: build: - context: apps/ - dockerfile: cic-notify/docker/Dockerfile + context: apps/cic-notify + dockerfile: docker/Dockerfile environment: DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -428,8 +431,8 @@ services: cic-meta-server: hostname: meta build: - context: apps/ - dockerfile: cic-meta/docker/Dockerfile + context: apps/cic-meta + dockerfile: docker/Dockerfile environment: DATABASE_NAME: ${DATABASE_NAME:-cic_meta} DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} @@ -460,8 +463,8 @@ services: cic-user-ussd-server: build: - context: apps/ - dockerfile: cic-ussd/docker/Dockerfile + context: apps/cic-ussd + dockerfile: docker/Dockerfile environment: DATABASE_USER: grassroots DATABASE_HOST: postgres @@ -489,8 +492,8 @@ services: cic-user-server: build: - context: apps - dockerfile: cic-ussd/docker/Dockerfile + context: apps/cic-ussd + dockerfile: docker/Dockerfile environment: DATABASE_USER: grassroots DATABASE_HOST: postgres @@ -511,8 +514,8 @@ services: cic-user-tasker: build: - context: apps - dockerfile: cic-ussd/docker/Dockerfile + context: apps/cic-ussd/ + dockerfile: docker/Dockerfile environment: DATABASE_USER: grassroots DATABASE_HOST: postgres