kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into lash/update-aux-deps

Browse Source
This commit is contained in:
nolash 2021-09-15 21:21:37 +02:00
commit dcdff46b34
14 changed files with 142 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitlab-ci.yml
View File

@ -36,3 +36,8 @@ build-push:
rules:
- if: $CI_COMMIT_REF_PROTECTED == "true"
when: always
deploy-dev:
stage: deploy
trigger: grassrootseconomics/devops
when: manual

10
apps/cic-eth/.gitlab-ci.yml
View File

@ -9,8 +9,8 @@ build-test-cic-eth:
- cd apps/cic-eth
- docker build -t $MR_IMAGE_TAG -f docker/Dockerfile .
- docker run $MR_IMAGE_TAG sh docker/run_tests.sh
#rules:
#- if: $CI_PIPELINE_SOURCE == "merge_request_event"
# changes:
# - apps/$APP_NAME/**/*
# when: always
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/$APP_NAME/**/*
when: always

29
apps/cic-eth/docker/Dockerfile
View File

@ -13,7 +13,16 @@ ARG EXTRA_PIP_ARGS=""
# pip install --index-url https://pypi.org/simple \
# --force-reinstall \
# --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
# -r requirements.txt
# -r requirements.txt
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL \
$EXTRA_PIP_ARGS \
cic-eth-aux-erc20-demurrage-token~=0.0.2a6
COPY *requirements.txt ./
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
@ -22,19 +31,21 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
$EXTRA_PIP_ARGS \
-r requirements.txt \
-r services_requirements.txt \
-r admin_requirements.txt
COPY . .
RUN python setup.py install
ENV PYTHONPATH .
-r admin_requirements.txt
# always install the latest signer
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL \
$EXTRA_PIP_ARGS \
cic-eth-aux-erc20-demurrage-token~=0.0.2a6
crypto-dev-signer
COPY . .
RUN python setup.py install
ENV PYTHONPATH .
COPY docker/entrypoints/* ./
RUN chmod 755 *.sh

36
apps/cic-eth/docker/entrypoints/start_tasker.sh
View File

@ -5,27 +5,27 @@ set -e
# set CONFINI_ENV_PREFIX to override the env prefix to override env vars
echo "!!! starting signer"
python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
#echo "!!! starting signer"
#python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
echo "!!! starting taskerd"
/usr/local/bin/cic-eth-taskerd $@
# thanks! https://docs.docker.com/config/containers/multi-service_container/
sleep 1;
echo "!!! entering monitor loop"
while true; do
ps aux | grep crypto-dev-daemon | grep -q -v grep
PROCESS_1_STATUS=$?
ps aux | grep cic-eth-tasker |grep -q -v grep
PROCESS_2_STATUS=$?
# If the greps above find anything, they exit with 0 status
# If they are not both 0, then something is wrong
if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
echo "One of the processes has already exited."
exit 1
fi
sleep 15;
done
#sleep 1;
#echo "!!! entering monitor loop"
#while true; do
# ps aux | grep crypto-dev-daemon | grep -q -v grep
# PROCESS_1_STATUS=$?
# ps aux | grep cic-eth-tasker |grep -q -v grep
# PROCESS_2_STATUS=$?
# # If the greps above find anything, they exit with 0 status
# # If they are not both 0, then something is wrong
# if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
# echo "One of the processes has already exited."
# exit 1
# fi
# sleep 15;
#done
#
set +e

1
apps/cic-eth/requirements.txt
View File

@ -1,3 +1,4 @@
celery==4.4.7
chainlib-eth>=0.0.9a9,<0.1.0
chainlib-eth>=0.0.9a13,<0.1.0
semver==2.13.0

2
apps/cic-eth/tools_requirements.txt
View File

@ -1,4 +1,4 @@
crypto-dev-signer>=0.4.15a1,<=0.4.15
crypto-dev-signer>=0.4.15a4,<=0.4.15
chainqueue>=0.0.5a1,<0.1.0
cic-eth-registry>=0.6.1a2,<0.7.0
redis==3.5.3

10
apps/cic-meta/.gitlab-ci.yml
View File

@ -9,8 +9,8 @@ build-test-cic-meta:
- cd apps/cic-meta
- docker build -t $MR_IMAGE_TAG -f docker/Dockerfile .
- docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh
#rules:
#- if: $CI_PIPELINE_SOURCE == "merge_request_event"
# changes:
# - apps/$APP_NAME/**/*
# when: always
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- apps/$APP_NAME/**/*
when: always

1
apps/cic-meta/README.md Normal file
View File

@ -0,0 +1 @@
# CIC-Meta

44
apps/contract-migration/docker/Dockerfile_ci
View File

@ -1,44 +0,0 @@
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e
WORKDIR /root
RUN touch /etc/apt/sources.list.d/ethereum.list
RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
RUN cat /etc/apt/sources.list.d/ethereum.list
RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
#RUN apt-get install solc
RUN mkdir -vp /usr/local/etc/cic
ENV CONFINI_DIR /usr/local/etc/cic/
COPY config_template/ /usr/local/etc/cic/
COPY requirements.txt .
COPY override_requirements.txt .
ARG pip_index_url=https://pypi.org/simple
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG EXTRA_PIP_ARGS=""
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
ARG pip_trusted_host=pypi.org
RUN pip install --index-url https://pypi.org/simple \
pip install --index-url https://pypi.org/simple \
--pre \
--force-reinstall \
--trusted-host $pip_trusted_host \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt
RUN pip install --index-url https://pypi.org/simple \
--force-reinstall \
--pre \
--trusted-host $pip_trusted_host \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-r override_requirements.txt
COPY . .
RUN chmod +x *.sh

1
apps/contract-migration/requirements.txt
View File

@ -9,3 +9,4 @@ erc20-transfer-authorization>=0.3.5a2,<0.4.0
erc20-faucet>=0.3.2a2,<0.4.0
sarafu-faucet>=0.0.7a2,<0.1.0
confini>=0.4.2rc3,<1.0.0
crypto-dev-signer>=0.4.15a4,<=0.4.15

9
apps/contract-migration/reset.sh
View File

@ -3,6 +3,8 @@
set -a
. ${DEV_DATA_DIR}/env_reset
WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60}
set -e
@ -24,7 +26,7 @@ if [ ! -z "$DEV_USE_DOCKER_WAIT_SCRIPT" ]; then
read -i "/" rpc_provider_port <<< "${p[2]}"
rpc_provider_host=${p[1]:2}
echo "waiting for provider host $rpc_provider_host port $rpc_provider_port..."
./wait-for-it.sh "$rpc_provider_host:$rpc_provider_port"
./wait-for-it.sh "$rpc_provider_host:$rpc_provider_port" -t $WAIT_FOR_TIMEOUT
fi
if [ "$TOKEN_TYPE" == "giftable_erc20_token" ]; then
@ -96,7 +98,7 @@ eth-token-index-add $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_S
DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s`
>&2 echo "set token faucet amount"
sarafu-faucet-set $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
sarafu-faucet-set $fee_price_arg -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
>&2 echo "register faucet in registry"
eth-contract-registry-set -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS
@ -105,7 +107,10 @@ eth-contract-registry-set -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $CIC_RE
giftable-token-minter -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv $DEV_FAUCET_ADDRESS
#echo "export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL" >> ${DEV_DATA_DIR}/env_reset
export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset
confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config
set +a
set +e

2
apps/contract-migration/seed_cic_eth.sh
View File

@ -82,5 +82,7 @@ cic-eth-ctl -i $CHAIN_SPEC unlock INIT
cic-eth-ctl -i $CHAIN_SPEC unlock SEND
cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE
confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config
set +a
set +e

90
docker-compose.yml
View File

@ -73,6 +73,7 @@ services:
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
environment:
RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
DEV_USE_DOCKER_WAIT_SCRIPT: 1
CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
@ -101,6 +102,7 @@ services:
TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL
TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
TOKEN_TYPE: $TOKEN_TYPE
SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
command: ["./run_job.sh"]
#command: ["./reset.sh"]
depends_on:
@ -148,7 +150,7 @@ services:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
./start_tracker.sh -c /usr/local/etc/cic-cache -vv
volumes:
- contract-config:/tmp/cic/config/:ro
@ -189,7 +191,7 @@ services:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
/usr/local/bin/cic-cache-taskerd -vv
volumes:
- contract-config:/tmp/cic/config/:ro
@ -223,7 +225,7 @@ services:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
"/usr/local/bin/uwsgi" \
--wsgi-file /root/cic_cache/runnable/daemons/server.py \
--http :8000 \
@ -260,8 +262,64 @@ services:
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
CELERY_DEBUG: ${CELERY_DEBUG:-1}
SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc}
SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc}
SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
depends_on:
- eth
- postgres
- redis
- cic-eth-signer
deploy:
restart_policy:
condition: on-failure
volumes:
- signer-data:/run/crypto-dev-signer
- contract-config:/tmp/cic/config/:ro
#command: ["/usr/local/bin/cic-eth-taskerd"]
#command: ["sleep", "3600"]
command:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
./start_tasker.sh --aux-all -q cic-eth -vv
cic-eth-signer:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
args:
EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER
ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
DATABASE_NAME: ${DATABASE_NAME_CIC_ETH:-cic_eth}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
REDIS_PORT: 6379
REDIS_HOST: redis
PGPASSWORD: ${DATABASE_PASSWORD:-tralala}
CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
CELERY_DEBUG: ${CELERY_DEBUG:-1}
SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000}
SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000}
SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
@ -274,14 +332,15 @@ services:
restart_policy:
condition: on-failure
volumes:
- signer-data:/tmp/cic/signer
- signer-data:/run/crypto-dev-signer
- contract-config:/tmp/cic/config/:ro
command:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
./start_tasker.sh --aux-all -q cic-eth -vv
command: ["python", "/usr/local/bin/crypto-dev-daemon", "-c", "/usr/local/etc/crypto-dev-signer", "-vv"]
#command:
# - /bin/bash
# - -c
# - |
# if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
# ./start_tasker.sh --aux-all -q cic-eth -vv
# command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
cic-eth-tracker:
@ -323,7 +382,7 @@ services:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
#./start_tracker.sh -vv -c /usr/local/etc/cic-eth
./start_tracker.sh -vv
# command: "/root/start_manager.sh head -vv"
@ -369,9 +428,10 @@ services:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
./start_dispatcher.sh -q cic-eth -vv
# command: "/root/start_dispatcher.sh -q cic-eth -vv"
cic-eth-retrier:
@ -416,7 +476,7 @@ services:
- /bin/bash
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
./start_retry.sh -vv
# command: "/root/start_retry.sh -q cic-eth -vv"

1
dump.sh Normal file
View File

@ -0,0 +1 @@
docker run -t -v cic-internal-integration_contract-config:/tmp/cic/config cic-internal-integration_contract-migration cat /tmp/cic/config/env_reset