Merge branch 'bvander/cic-eth-base-image-and-testing' into 'master'

Faster local builds with base image and buildkit and run all unit tests

See merge request grassrootseconomics/cic-internal-integration!217
This commit is contained in:
Blair Vanderlugt 2021-07-10 15:46:15 +00:00
commit fe017d2b0f
42 changed files with 887 additions and 295 deletions

1
.gitignore vendored
View File

@ -13,3 +13,4 @@ build/
**/coverage
**/.venv
.idea
**/.vim

View File

@ -2,4 +2,5 @@
omit =
.venv/*
scripts/*
cic_cache/db/postgres/*
cic_cache/db/migrations/*
cic_cache/version.py

View File

@ -0,0 +1,4 @@
.git
.cache
.dot
**/doc

View File

@ -1,22 +1,52 @@
.cic_cache_variables:
variables:
APP_NAME: cic-cache
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
.cic_cache_changes_target:
rules:
- changes:
- $CONTEXT/$APP_NAME/*
DOCKERFILE_PATH: docker/Dockerfile_ci
CONTEXT: apps/$APP_NAME
build-mr-cic-cache:
extends:
- .cic_cache_changes_target
- .py_build_merge_request
- .cic_cache_variables
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-cache/**/*
when: always
test-mr-cic-cache:
stage: test
extends:
- .cic_cache_variables
cache:
key:
files:
- test_requirements.txt
paths:
- /root/.cache/pip
image: $MR_IMAGE_TAG
script:
- cd apps/$APP_NAME/
- >
pip install --extra-index-url https://pip.grassrootseconomics.net:8433
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-r test_requirements.txt
- export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
needs: ["build-mr-cic-cache"]
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-eth/**/*
when: always
build-push-cic-cache:
extends:
- .py_build_push
- .cic_cache_variables
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
- apps/cic-cache/**/*
when: always

View File

@ -15,6 +15,8 @@ from cic_cache.db.list import (
logg = logging.getLogger()
DEFAULT_FILTER_SIZE = 8192 * 8
class Cache:
def __init__(self, session):
@ -25,7 +27,7 @@ class BloomCache(Cache):
@staticmethod
def __get_filter_size(n):
n = 8192 * 8
n = DEFAULT_FILTER_SIZE
logg.warning('filter size hardcoded to {}'.format(n))
return n

View File

@ -4,6 +4,9 @@ import json
import re
import base64
# external imports
from hexathon import add_0x
# local imports
from cic_cache.cache import (
BloomCache,
@ -11,10 +14,11 @@ from cic_cache.cache import (
)
logg = logging.getLogger(__name__)
#logg = logging.getLogger()
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
DEFAULT_LIMIT = 100
@ -26,13 +30,13 @@ def process_transactions_account_bloom(session, env):
address = r[1]
if r[2] == None:
address = '0x' + address
offset = DEFAULT_LIMIT
address = add_0x(address)
offset = 0
if r.lastindex > 2:
offset = r[3]
limit = 0
if r.lastindex > 3:
limit = r[4]
offset = r[4]
limit = DEFAULT_LIMIT
if r.lastindex > 4:
limit = r[6]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)

View File

@ -1,52 +1,38 @@
FROM python:3.8.6-slim-buster
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
COPY requirements.txt .
#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
#RUN pip install $pip_extra_index_url_flag .
#RUN pip install .[server]
WORKDIR /usr/src/cic-cache
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
ARG root_requirement_file='requirements.txt'
COPY . .
#RUN apk update && \
# apk add gcc musl-dev gnupg libpq
#RUN apk add postgresql-dev
#RUN apk add linux-headers
#RUN apk add libffi-dev
RUN apt-get update && \
apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
# Copy shared requirements from top of mono-repo
RUN echo "copying root req file ${root_requirement_file}"
RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
COPY cic-cache/requirements.txt ./
COPY cic-cache/setup.cfg \
cic-cache/setup.py \
./
COPY cic-cache/cic_cache/ ./cic_cache/
COPY cic-cache/scripts/ ./scripts/
COPY cic-cache/test_requirements.txt ./
RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
RUN pip install $pip_extra_index_url_flag .
RUN pip install .[server]
COPY cic-cache/tests/ ./tests/
#COPY db/ cic-cache/db
#RUN apk add postgresql-client
RUN python setup.py install
# ini files in config directory defines the configurable parameters for the application
# they can all be overridden by environment variables
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY cic-cache/config/ /usr/local/etc/cic-cache/
COPY config/ /usr/local/etc/cic-cache/
# for db migrations
RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
COPY cic-cache/docker/db.sh ./db.sh
COPY /docker/start_tracker.sh ./start_tracker.sh
COPY /docker/db.sh ./db.sh
RUN chmod 755 ./*.sh
# Tracker
# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
# Server
# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
ENTRYPOINT []

View File

@ -0,0 +1,37 @@
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
COPY requirements.txt .
#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
#RUN pip install $pip_extra_index_url_flag .
#RUN pip install .[server]
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
RUN pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
COPY . .
RUN python setup.py install
# ini files in config directory defines the configurable parameters for the application
# they can all be overridden by environment variables
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY config/ /usr/local/etc/cic-cache/
# for db migrations
RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
COPY /docker/start_tracker.sh ./start_tracker.sh
COPY /docker/db.sh ./db.sh
RUN chmod 755 ./*.sh
# Tracker
# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
# Server
# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
ENTRYPOINT []

View File

@ -5,9 +5,12 @@ import datetime
# external imports
import pytest
import moolb
# local imports
from cic_cache import db
from cic_cache import BloomCache
from cic_cache.cache import DEFAULT_FILTER_SIZE
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
@ -101,3 +104,7 @@ def tag_txs(
db.tag_transaction(init_database, txs[1], 'taag', domain='test')
@pytest.fixture(scope='session')
def zero_filter():
return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

View File

@ -10,6 +10,7 @@ from sqlalchemy import text
from chainlib.eth.tx import Tx
from chainlib.eth.block import Block
from chainlib.chain import ChainSpec
from chainlib.eth.error import RequestMismatchException
from hexathon import (
strip_0x,
add_0x,
@ -18,10 +19,21 @@ from hexathon import (
# local imports
from cic_cache.db import add_tag
from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
from cic_cache.runnable.daemons.filters.base import TagSyncFilter
logg = logging.getLogger()
def test_base_filter_str(
init_database,
):
f = TagSyncFilter('foo')
assert 'foo' == str(f)
f = TagSyncFilter('foo', domain='bar')
assert 'bar.foo' == str(f)
def test_erc20_filter(
eth_rpc,
foo_token,
@ -67,3 +79,95 @@ def test_erc20_filter(
s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
assert r[0] == tx.hash
def test_erc20_filter_nocontract(
eth_rpc,
foo_token,
init_database,
list_defaults,
list_actors,
tags,
):
chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
fltr = ERC20TransferFilter(chain_spec)
add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
# incomplete args
data = 'a9059cbb'
data += strip_0x(list_actors['alice'])
data += '1000'.ljust(64, '0')
block = Block({
'hash': os.urandom(32).hex(),
'number': 42,
'timestamp': datetime.datetime.utcnow().timestamp(),
'transactions': [],
})
tx = Tx({
'to': os.urandom(20).hex(),
'from': list_actors['bob'],
'data': data,
'value': 0,
'hash': os.urandom(32).hex(),
'nonce': 13,
'gasPrice': 10000000,
'gas': 123456,
})
block.txs.append(tx)
tx.block = block
assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
@pytest.mark.parametrize(
'contract_method,contract_input,expected_exception',
[
('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
],
)
def test_erc20_filter_bogus(
eth_rpc,
foo_token,
init_database,
list_defaults,
list_actors,
tags,
contract_method,
contract_input,
expected_exception,
):
chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
fltr = ERC20TransferFilter(chain_spec)
add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
# incomplete args
data = contract_method
data += contract_input
block = Block({
'hash': os.urandom(32).hex(),
'number': 42,
'timestamp': datetime.datetime.utcnow().timestamp(),
'transactions': [],
})
tx = Tx({
'to': foo_token,
'from': list_actors['bob'],
'data': data,
'value': 0,
'hash': os.urandom(32).hex(),
'nonce': 13,
'gasPrice': 10000000,
'gas': 123456,
})
block.txs.append(tx)
tx.block = block
assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

View File

@ -0,0 +1,230 @@
# standard imports
import logging
import json
import base64
import copy
import re
# external imports
import pytest
from hexathon import strip_0x
# local imports
from cic_cache.runnable.daemons.query import *
logg = logging.getLogger()
@pytest.mark.parametrize(
'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
[
('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
],
)
def test_query_regex(
list_actors,
query_path_prefix,
query_role,
query_address_index,
query_offset,
query_offset_index,
query_limit,
query_limit_index,
match_re,
):
paths = []
path = query_path_prefix
query_address = None
if query_role != None:
query_address = strip_0x(list_actors[query_role])
paths.append(path + '0x' + query_address)
paths.append(path + query_address)
if query_offset != None:
if query_limit != None:
for i in range(len(paths)-1):
paths[i] += '/{}/{}'.format(query_offset, query_limit)
else:
for i in range(len(paths)-1):
paths[i] += '/' + str(query_offset)
for i in range(len(paths)):
paths.append(paths[i] + '/')
for p in paths:
logg.debug('testing path {} against {}'.format(p, match_re))
m = re.match(match_re, p)
l = len(m.groups())
logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
for i in range(l+1):
logg.debug('group {} {}'.format(i, m[i]))
if m.lastindex >= query_offset_index:
assert query_offset == int(m[query_offset_index + 1])
if m.lastindex >= query_limit_index:
assert query_limit == int(m[query_limit_index + 1])
if query_address_index != None:
match_address = strip_0x(m[query_address_index + 1])
assert query_address == match_address
@pytest.mark.parametrize(
'role_name, query_offset, query_limit, query_match',
[
('alice', None, None, [(420000, 13), (419999, 42)]),
('alice', None, 1, [(420000, 13)]),
('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
('alice', 2, None, []), # 420000 == list_defaults['block']
],
)
def test_query_process_txs_account(
init_database,
list_defaults,
list_actors,
list_tokens,
txs,
zero_filter,
role_name,
query_offset,
query_limit,
query_match,
):
actor = None
try:
actor = list_actors[role_name]
except KeyError:
actor = os.urandom(20).hex()
path_info = '/tx/user/0x' + strip_0x(actor)
if query_offset != None:
path_info += '/' + str(query_offset)
if query_limit != None:
if query_offset == None:
path_info += '/0'
path_info += '/' + str(query_limit)
env = {
'PATH_INFO': path_info,
}
logg.debug('using path {}'.format(path_info))
r = process_transactions_account_bloom(init_database, env)
assert r != None
o = json.loads(r[1])
block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
zero_filter_data = zero_filter.to_bytes()
if len(query_match) == 0:
assert block_filter_data == zero_filter_data
return
assert block_filter_data != zero_filter_data
block_filter = copy.copy(zero_filter)
block_filter.merge(block_filter_data)
block_filter_data = block_filter.to_bytes()
assert block_filter_data != zero_filter_data
for (block, tx) in query_match:
block = block.to_bytes(4, byteorder='big')
assert block_filter.check(block)
@pytest.mark.parametrize(
'query_offset, query_limit, query_match',
[
(None, 2, [(420000, 13), (419999, 42)]),
(0, 1, [(420000, 13)]),
(1, 1, [(419999, 42)]),
(2, 0, []),
],
)
def test_query_process_txs_bloom(
init_database,
list_defaults,
list_actors,
list_tokens,
txs,
zero_filter,
query_offset,
query_limit,
query_match,
):
path_info = '/tx'
if query_offset != None:
path_info += '/' + str(query_offset)
if query_limit != None:
if query_offset == None:
path_info += '/0'
path_info += '/' + str(query_limit)
env = {
'PATH_INFO': path_info,
}
logg.debug('using path {}'.format(path_info))
r = process_transactions_all_bloom(init_database, env)
assert r != None
o = json.loads(r[1])
block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
zero_filter_data = zero_filter.to_bytes()
if len(query_match) == 0:
assert block_filter_data == zero_filter_data
return
assert block_filter_data != zero_filter_data
block_filter = copy.copy(zero_filter)
block_filter.merge(block_filter_data)
block_filter_data = block_filter.to_bytes()
assert block_filter_data != zero_filter_data
for (block, tx) in query_match:
block = block.to_bytes(4, byteorder='big')
assert block_filter.check(block)
@pytest.mark.parametrize(
'query_block_start, query_block_end, query_match_count',
[
(None, 42, 0),
(420000, 420001, 1),
(419999, 419999, 1), # matches are inclusive
(419999, 420000, 2),
(419999, 420001, 2),
],
)
def test_query_process_txs_data(
init_database,
list_defaults,
list_actors,
list_tokens,
txs,
zero_filter,
query_block_start,
query_block_end,
query_match_count,
):
path_info = '/txa'
if query_block_start != None:
path_info += '/' + str(query_block_start)
if query_block_end != None:
if query_block_start == None:
path_info += '/0'
path_info += '/' + str(query_block_end)
env = {
'PATH_INFO': path_info,
'HTTP_X_CIC_CACHE_MODE': 'all',
}
logg.debug('using path {}'.format(path_info))
r = process_transactions_all_data(init_database, env)
assert r != None
o = json.loads(r[1])
assert len(o['data']) == query_match_count

View File

@ -0,0 +1,6 @@
.git
.cache
.dot
**/doc
**/.venv
**/venv

View File

@ -1,31 +1,52 @@
.cic_eth_variables:
variables:
APP_NAME: cic-eth
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
.cic_eth_mr_changes_target:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- $CONTEXT/$APP_NAME/**/*
when: always
DOCKERFILE_PATH: docker/Dockerfile_ci
CONTEXT: apps/$APP_NAME
build-mr-cic-eth:
extends:
- .cic_eth_variables
- .cic_eth_mr_changes_target
- .py_build_target_test
- .py_build_target_dev
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-eth/**/*
when: always
test-mr-cic-eth:
stage: test
extends:
- .cic_eth_variables
- .cic_eth_mr_changes_target
stage: test
image: $IMAGE_TAG_BASE
cache:
key:
files:
- test_requirements.txt
paths:
- /root/.cache/pip
image: $MR_IMAGE_TAG
script:
- cd apps/$APP_NAME/
- pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
- >
pip install --extra-index-url https://pip.grassrootseconomics.net:8433
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-r admin_requirements.txt
-r services_requirements.txt
-r test_requirements.txt
- export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
needs: ["build-mr-cic-eth"]
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-eth/**/*
when: always
build-push-cic-eth:
extends:
- .py_build_push
- .cic_eth_variables
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
- apps/cic-eth/**/*
when: always

View File

@ -1,58 +1,39 @@
FROM python:3.8.6-slim-buster as compile
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
WORKDIR /usr/src
RUN apt-get update && \
apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
#RUN python -m venv venv && . venv/bin/activate
ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
RUN /usr/local/bin/python -m pip install --upgrade pip
RUN pip install semver
COPY cic-eth-aux/ ./cic-eth-aux/
WORKDIR /usr/src/cic-eth-aux/erc20-demurrage-token
RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL .
WORKDIR /usr/src/cic-eth
COPY cic-eth/ .
RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL .[services]
# --- TEST IMAGE ---
FROM python:3.8.6-slim-buster as test
RUN apt-get update && \
apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
WORKDIR /usr/src/cic-eth
RUN /usr/local/bin/python -m pip install --upgrade pip
COPY --from=compile /usr/local/bin/ /usr/local/bin/
COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
/usr/local/lib/python3.8/site-packages/
# TODO we could use venv inside container to isolate the system and app deps further
# COPY --from=compile /usr/src/cic-eth/ .
# RUN . venv/bin/activate
# Copy just the requirements and install....this _might_ give docker a hint on caching but we
# do load these all into setup.py later
# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
#COPY cic-eth/requirements.txt .
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
# pip install --index-url https://pypi.org/simple \
# --force-reinstall \
# --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
# -r requirements.txt
COPY cic-eth/test_requirements.txt .
RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
COPY cic-eth .
COPY . .
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL .
ENV PYTHONPATH .
ENTRYPOINT ["pytest"]
COPY docker/entrypoints/* ./
RUN chmod 755 *.sh
# # ini files in config directory defines the configurable parameters for the application
# # they can all be overridden by environment variables
# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY config/ /usr/local/etc/cic-eth/
COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
#COPY util/liveness/health.sh /usr/local/bin/health.sh
ENTRYPOINT []
# --- RUNTIME ---
FROM python:3.8.6-slim-buster as runtime
@ -62,20 +43,22 @@ RUN apt-get update && \
WORKDIR /usr/src/cic-eth
COPY --from=compile /usr/local/bin/ /usr/local/bin/
COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
COPY --from=dev /usr/local/bin/ /usr/local/bin/
COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
/usr/local/lib/python3.8/site-packages/
COPY cic-eth/docker/* ./
COPY docker/entrypoints/* ./
RUN chmod 755 *.sh
COPY cic-eth/scripts/ scripts/
# # ini files in config directory defines the configurable parameters for the application
# # they can all be overridden by environment variables
# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY cic-eth/config/ /usr/local/etc/cic-eth/
COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
COPY config/ /usr/local/etc/cic-eth/
COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
COPY util/liveness/health.sh /usr/local/bin/health.sh
# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
#COPY util/liveness/health.sh /usr/local/bin/health.sh
ENTRYPOINT []

View File

@ -0,0 +1,62 @@
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
WORKDIR /usr/src/cic-eth
# Copy just the requirements and install....this _might_ give docker a hint on caching but we
# do load these all into setup.py later
# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
#COPY cic-eth/requirements.txt .
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
# pip install --index-url https://pypi.org/simple \
# --force-reinstall \
# --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
# -r requirements.txt
COPY . .
RUN pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL .
COPY docker/entrypoints/* ./
RUN chmod 755 *.sh
# # ini files in config directory defines the configurable parameters for the application
# # they can all be overridden by environment variables
# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY config/ /usr/local/etc/cic-eth/
COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
#COPY util/liveness/health.sh /usr/local/bin/health.sh
ENTRYPOINT []
# --- RUNTIME ---
FROM python:3.8.6-slim-buster as runtime
RUN apt-get update && \
apt install -y gnupg libpq-dev procps
WORKDIR /usr/src/cic-eth
COPY --from=dev /usr/local/bin/ /usr/local/bin/
COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
/usr/local/lib/python3.8/site-packages/
COPY docker/entrypoints/* ./
RUN chmod 755 *.sh
# # ini files in config directory defines the configurable parameters for the application
# # they can all be overridden by environment variables
# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY config/ /usr/local/etc/cic-eth/
COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
#COPY util/liveness/health.sh /usr/local/bin/health.sh
ENTRYPOINT []

View File

@ -10,5 +10,6 @@ eth-address-index~=0.1.2a1
eth-accounts-index~=0.0.12a1
cic-eth-registry~=0.5.6a1
erc20-faucet~=0.2.2a1
erc20-transfer-authorization~=0.3.2a1
sarafu-faucet~=0.0.4a1
moolb~=0.1.1b2

View File

@ -0,0 +1,4 @@
.git
.cache
.dot
**/doc

View File

@ -3,42 +3,41 @@
variables:
APP_NAME: cic-meta
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
CONTEXT: apps
.cic_meta_changes_target:
build-mr-cic-meta:
extends:
- .py_build_merge_request
- .cic_meta_variables
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
# - changes:
# - $CONTEXT/$APP_NAME/*
- when: always
cic-meta-build-mr:
stage: build
extends:
- .cic_meta_variables
- .cic_meta_changes_target
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
# - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-meta/**/*
when: always
test-mr-cic-meta:
extends:
- .cic_meta_variables
- .cic_meta_changes_target
stage: test
image: $IMAGE_TAG
image: $MR_IMAGE_TAG
script:
- cd /tmp/src/cic-meta
- npm install --dev
- npm run test
- npm run test:coverage
needs: ["cic-meta-build-mr"]
needs: ["build-mr-cic-meta"]
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-meta/**/*
when: always
build-push-cic-meta:
extends:
- .py_build_push
- .cic_meta_variables
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
- apps/cic-meta/**/*
when: always

View File

@ -1,22 +1,52 @@
.cic_notify_variables:
variables:
APP_NAME: cic-notify
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
.cic_notify_changes_target:
rules:
- changes:
- $CONTEXT/$APP_NAME/*
DOCKERFILE_PATH: docker/Dockerfile_ci
CONTEXT: apps/$APP_NAME
build-mr-cic-notify:
extends:
- .cic_notify_changes_target
- .py_build_merge_request
- .cic_notify_variables
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-notify/**/*
when: always
test-mr-cic-notify:
stage: test
extends:
- .cic_notify_variables
cache:
key:
files:
- test_requirements.txt
paths:
- /root/.cache/pip
image: $MR_IMAGE_TAG
script:
- cd apps/$APP_NAME/
- >
pip install --extra-index-url https://pip.grassrootseconomics.net:8433
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-r test_requirements.txt
- export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
needs: ["build-mr-cic-notify"]
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-eth/**/*
when: always
build-push-cic-notify:
extends:
- .py_build_push
- .cic_notify_variables
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
- apps/cic-notify/**/*
when: always

View File

@ -1,38 +1,30 @@
FROM python:3.8.6-slim-buster
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
RUN apt-get update && \
apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps
#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
WORKDIR /usr/src/cic-notify
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
COPY requirements.txt .
ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
COPY cic-notify/setup.cfg \
cic-notify/setup.py \
./
COPY . .
COPY cic-notify/cic_notify/ ./cic_notify/
RUN python setup.py install
COPY cic-notify/requirements.txt \
cic-notify/test_requirements.txt \
./
COPY cic-notify/scripts/ scripts/
# TODO please review..can this go into requirements?
RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
COPY cic-notify/tests/ tests/
COPY cic-notify/docker/db.sh \
cic-notify/docker/start_tasker.sh \
/root/
#RUN apk add postgresql-client
#RUN apk add bash
COPY docker/*.sh .
# ini files in config directory defines the configurable parameters for the application
# they can all be overridden by environment variables
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY cic-notify/.config/ /usr/local/etc/cic-notify/
COPY cic-notify/cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
COPY .config/ /usr/local/etc/cic-notify/
COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
WORKDIR /root
ENTRYPOINT []

View File

@ -0,0 +1,29 @@
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
COPY requirements.txt .
RUN pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
COPY . .
RUN python setup.py install
# TODO please review..can this go into requirements?
RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
COPY docker/*.sh .
# ini files in config directory defines the configurable parameters for the application
# they can all be overridden by environment variables
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY .config/ /usr/local/etc/cic-notify/
COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
ENTRYPOINT []

View File

@ -0,0 +1,4 @@
.git
.cache
.dot
**/doc

View File

@ -1,22 +1,52 @@
.cic_ussd_variables:
variables:
APP_NAME: cic-ussd
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
.cic_ussd_changes_target:
rules:
- changes:
- $CONTEXT/$APP_NAME/*
DOCKERFILE_PATH: docker/Dockerfile_ci
CONTEXT: apps/$APP_NAME
build-mr-cic-ussd:
extends:
- .cic_ussd_changes_target
- .py_build_merge_request
- .cic_ussd_variables
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-ussd/**/*
when: always
test-mr-cic-ussd:
stage: test
extends:
- .cic_ussd_variables
cache:
key:
files:
- test_requirements.txt
paths:
- /root/.cache/pip
image: $MR_IMAGE_TAG
script:
- cd apps/$APP_NAME/
- >
pip install --extra-index-url https://pip.grassrootseconomics.net:8433
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-r test_requirements.txt
- export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
needs: ["build-mr-cic-ussd"]
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/cic-eth/**/*
when: always
build-push-cic-ussd:
extends:
- .py_build_push
- .cic_ussd_variables
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
- apps/cic-ussd/**/*
when: always

View File

@ -1,14 +1,5 @@
# FROM python:3.8.5-alpine
FROM python:3.8.6-slim-buster
# set working directory
WORKDIR /usr/src
# add args for installing from self-hosted packages
ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
RUN apt-get update && \
apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
# create secrets directory
RUN mkdir -vp pgp/keys
@ -17,39 +8,25 @@ RUN mkdir -vp pgp/keys
RUN mkdir -vp cic-ussd
RUN mkdir -vp data
COPY cic-ussd/setup.cfg \
cic-ussd/setup.py \
cic-ussd/
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
COPY requirements.txt .
COPY cic-ussd/requirements.txt \
cic-ussd/test_requirements.txt \
cic-ussd/
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
# install requirements
RUN cd cic-ussd && \
pip install -r requirements.txt $pip_extra_index_url_flag
COPY . .
RUN python setup.py install
# copy all necessary files
COPY cic-ussd/cic_ussd/ cic-ussd/cic_ussd/
COPY cic-ussd/cic_ussd/db/ussd_menu.json data/
COPY cic-ussd/scripts/ cic-ussd/scripts/
COPY cic-ussd/states/ cic-ussd/states/
COPY cic-ussd/transitions/ cic-ussd/transitions/
COPY cic-ussd/var/ cic-ussd/var/
COPY cic-ussd/docker/db.sh \
cic-ussd/docker/start_cic_user_tasker.sh \
cic-ussd/docker/start_cic_user_ussd_server.sh\
cic-ussd/docker/start_cic_user_server.sh\
/root/
COPY cic_ussd/db/ussd_menu.json data/
COPY docker/*.sh .
RUN chmod +x /root/*.sh
RUN cd cic-ussd && \
pip install $pip_extra_index_url_flag .
# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
COPY cic-ussd/.config/ /usr/local/etc/cic-ussd/
COPY cic-ussd/cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
COPY .config/ /usr/local/etc/cic-ussd/
COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
WORKDIR /root
ENTRYPOINT []

View File

@ -0,0 +1,32 @@
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
# create secrets directory
RUN mkdir -vp pgp/keys
# create application directory
RUN mkdir -vp cic-ussd
RUN mkdir -vp data
COPY requirements.txt .
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
RUN pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
COPY . .
RUN python setup.py install
COPY cic_ussd/db/ussd_menu.json data/
COPY docker/*.sh .
RUN chmod +x /root/*.sh
# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
COPY .config/ /usr/local/etc/cic-ussd/
COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
ENTRYPOINT []

View File

@ -0,0 +1,4 @@
.git
.cache
.dot
**/doc

View File

@ -1,20 +1,25 @@
.contract_migration_variables:
variables:
APP_NAME: contract-migration
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
.contract_migration_changes_target:
rules:
- changes:
- $CONTEXT/$APP_NAME/*
DOCKERFILE_PATH: docker/Dockerfile_ci
CONTEXT: apps/$APP_NAME
build-mr-contract-migration:
extends:
- .contract_migration_changes_target
- .py_build_merge_request
- .contract_migration_variables
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/contract-migration/**/*
when: always
build-push-contract-migration:
extends:
- .py_build_push
- .contract_migration_variables
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
- apps/contract-migration/**/*
when: always

View File

@ -25,7 +25,7 @@ RUN echo Install confini schema files && \
git checkout $cic_config_commit && \
cp -v *.ini $CONFINI_DIR
COPY contract-migration/requirements.txt .
COPY requirements.txt .
ARG pip_index_url=https://pypi.org/simple
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@ -38,5 +38,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
COPY contract-migration/ .
COPY . .
RUN chmod +x *.sh

View File

@ -3,12 +3,11 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55
WORKDIR /root
# solc install which we needed for bancor. Leaving as an artfact of HOW to do it.
#RUN touch /etc/apt/sources.list.d/ethereum.list
#RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
#RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
#RUN cat /etc/apt/sources.list.d/ethereum.list
#RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
RUN touch /etc/apt/sources.list.d/ethereum.list
RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
RUN cat /etc/apt/sources.list.d/ethereum.list
RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
#RUN apt-get install solc
@ -26,7 +25,7 @@ RUN echo Install confini schema files && \
git checkout $cic_config_commit && \
cp -v *.ini $CONFINI_DIR
COPY contract-migration/requirements.txt .
COPY requirements.txt .
ARG pip_index_url=https://pypi.org/simple
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@ -38,5 +37,5 @@ RUN pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-r requirements.txt
COPY contract-migration/ .
COPY . .
RUN chmod +x *.sh

View File

@ -0,0 +1,4 @@
.git
.cache
.dot
**/doc

View File

@ -1,21 +1,26 @@
.data_seeding_variables:
variables:
APP_NAME: data-seeding
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
.data_seeding_changes_target:
rules:
- changes:
- $CONTEXT/$APP_NAME/*
DOCKERFILE_PATH: docker/Dockerfile_ci
CONTEXT: apps/$APP_NAME
build-mr-data-seeding:
extends:
- .data_seeding_changes_target
- .py_build_merge_request
- .data_seeding_variables
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/data-seeding/**/*
when: always
build-push-data-seeding:
extends:
- .py_build_push
- .data_seeding_variables
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
- apps/data-seeding/**/*
when: always

View File

@ -5,13 +5,13 @@ WORKDIR /root
RUN mkdir -vp /usr/local/etc/cic
COPY data-seeding/package.json \
data-seeding/package-lock.json \
COPY package.json \
package-lock.json \
.
RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
COPY data-seeding/requirements.txt .
COPY requirements.txt .
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
@ -19,6 +19,6 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
--extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL -r requirements.txt
COPY data-seeding/ .
COPY . .
ENTRYPOINT [ ]

View File

@ -5,13 +5,13 @@ WORKDIR /root
RUN mkdir -vp /usr/local/etc/cic
COPY data-seeding/package.json \
data-seeding/package-lock.json \
COPY package.json \
package-lock.json \
.
RUN npm install
COPY data-seeding/requirements.txt .
COPY requirements.txt .
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
@ -19,6 +19,6 @@ RUN pip install \
--extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL -r requirements.txt
COPY data-seeding/ .
COPY . .
ENTRYPOINT [ ]

View File

@ -3,30 +3,29 @@ image:
entrypoint: [""]
variables:
KANIKO_CACHE_ARGS: "--cache=false --cache-copy-layers=true --cache-ttl=24h"
CONTEXT: $CI_PROJECT_DIR/apps/
KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
MR_IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA
.py_build_merge_request:
stage: build
variables:
CI_DEBUG_TRACE: "true"
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
when: always
- >
/kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS
--cache-repo $CI_REGISTRY_IMAGE --destination $MR_IMAGE_TAG
.py_build_target_test:
.py_build_target_dev:
stage: build
variables:
IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --destination $IMAGE_TAG_BASE
- >
/kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS
--cache-repo $CI_REGISTRY_IMAGE --target dev --skip-unused-stages
--destination $MR_IMAGE_TAG
.py_build_push:
stage: build
@ -39,9 +38,6 @@ variables:
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
# - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
rules:
- if: $CI_COMMIT_BRANCH == "master"
when: always

View File

@ -72,12 +72,11 @@ services:
contract-migration:
build:
dockerfile: contract-migration/docker/Dockerfile
# target: compile-image
context: apps/contract-migration
dockerfile: docker/Dockerfile
args:
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
pip_extra_args: $PIP_EXTRA_ARGS
context: apps/
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
environment:
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
@ -125,8 +124,8 @@ services:
cic-cache-tracker:
build:
context: apps
dockerfile: cic-cache/docker/Dockerfile
context: apps/cic-cache
dockerfile: docker/Dockerfile
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
@ -161,8 +160,8 @@ services:
cic-cache-tasker:
build:
context: apps
dockerfile: cic-cache/docker/Dockerfile
context: apps/cic-cache
dockerfile: docker/Dockerfile
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
@ -197,9 +196,9 @@ services:
- contract-config:/tmp/cic/config/:ro
cic-cache-server:
build:
context: apps
dockerfile: cic-cache/docker/Dockerfile
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
environment:
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_HOST: ${DATABASE_HOST:-postgres}
@ -230,8 +229,9 @@ services:
cic-eth-tasker:
# image: grassrootseconomics:cic-eth-service
build:
context: apps/
dockerfile: cic-eth/docker/Dockerfile
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER
@ -279,8 +279,9 @@ services:
cic-eth-tracker:
build:
context: apps/
dockerfile: cic-eth/docker/Dockerfile
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
ETH_PROVIDER: http://eth:8545
DATABASE_USER: ${DATABASE_USER:-grassroots}
@ -317,8 +318,9 @@ services:
cic-eth-dispatcher:
build:
context: apps/
dockerfile: cic-eth/docker/Dockerfile
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
ETH_PROVIDER: http://eth:8545
DATABASE_USER: ${DATABASE_USER:-grassroots}
@ -357,8 +359,9 @@ services:
cic-eth-retrier:
build:
context: apps/
dockerfile: cic-eth/docker/Dockerfile
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
ETH_PROVIDER: http://eth:8545
DATABASE_USER: ${DATABASE_USER:-grassroots}
@ -399,8 +402,8 @@ services:
cic-notify-tasker:
build:
context: apps/
dockerfile: cic-notify/docker/Dockerfile
context: apps/cic-notify
dockerfile: docker/Dockerfile
environment:
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_HOST: ${DATABASE_HOST:-postgres}
@ -428,8 +431,8 @@ services:
cic-meta-server:
hostname: meta
build:
context: apps/
dockerfile: cic-meta/docker/Dockerfile
context: apps/cic-meta
dockerfile: docker/Dockerfile
environment:
DATABASE_NAME: ${DATABASE_NAME:-cic_meta}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
@ -460,8 +463,8 @@ services:
cic-user-ussd-server:
build:
context: apps/
dockerfile: cic-ussd/docker/Dockerfile
context: apps/cic-ussd
dockerfile: docker/Dockerfile
environment:
DATABASE_USER: grassroots
DATABASE_HOST: postgres
@ -489,8 +492,8 @@ services:
cic-user-server:
build:
context: apps
dockerfile: cic-ussd/docker/Dockerfile
context: apps/cic-ussd
dockerfile: docker/Dockerfile
environment:
DATABASE_USER: grassroots
DATABASE_HOST: postgres
@ -511,8 +514,8 @@ services:
cic-user-tasker:
build:
context: apps
dockerfile: cic-ussd/docker/Dockerfile
context: apps/cic-ussd/
dockerfile: docker/Dockerfile
environment:
DATABASE_USER: grassroots
DATABASE_HOST: postgres