Compare commits

...

9 Commits

10 changed files with 252 additions and 120 deletions

View File

@ -1,14 +1,44 @@
include:
- local: 'ci_templates/.cic-template.yml'
- local: 'apps/contract-migration/.gitlab-ci.yml'
- local: 'apps/cic-eth/.gitlab-ci.yml'
- local: 'apps/cic-ussd/.gitlab-ci.yml'
- local: 'apps/cic-notify/.gitlab-ci.yml'
- local: 'apps/cic-meta/.gitlab-ci.yml'
- local: 'apps/cic-cache/.gitlab-ci.yml'
- local: 'apps/data-seeding/.gitlab-ci.yml'
#include:
# - local: 'ci_templates/.cic-template.yml'
# - local: 'apps/contract-migration/.gitlab-ci.yml'
# - local: 'apps/cic-eth/.gitlab-ci.yml'
# - local: 'apps/cic-ussd/.gitlab-ci.yml'
# - local: 'apps/cic-notify/.gitlab-ci.yml'
# - local: 'apps/cic-meta/.gitlab-ci.yml'
# - local: 'apps/cic-cache/.gitlab-ci.yml'
# - local: 'apps/data-seeding/.gitlab-ci.yml'
image: tiangolo/docker-with-compose
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
stages:
- build
- test
- release
- deploy
variables:
DOCKER_BUILDKIT: 1
COMPOSE_DOCKER_CLI_BUILD: 1
#services:
# - docker:stable-dind
build-prod:
tags:
- integration
stage: build
script:
- docker-compose build
only:
- bvander/integration-tests-on-docker
deploy-prod:
tags:
- integration
stage: deploy
script:
- docker stack deploy -c docker-compose.yml --with-registry-auth my-stack
only:
- bvander/integration-tests-on-docker

View File

@ -8,3 +8,7 @@ REDIS_PORT=6379
USER_USSD_HOST=cic-user-ussd-server
USER_USSD_PORT=9000
KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
CIC_DATA_DIR=config/
OUT_DIR=out
NUMBER_OF_USERS=10
META_URL=http://cic-meta-server:63380

View File

@ -9,7 +9,7 @@ COPY package.json \
package-lock.json \
.
RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
RUN npm ci --production
COPY requirements.txt .
@ -20,5 +20,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
--extra-index-url $EXTRA_INDEX_URL -r requirements.txt
COPY . .
RUN chmod +x scripts/*
ENTRYPOINT [ ]

View File

@ -0,0 +1,36 @@
#! /bin/bash
if [[ $((RUN_MASK & 3)) -eq 3 ]]
then
>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 3 - data seeding"
contract_migration_complete=0
retry_count=0
retry_sleep=30 #seconds
retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))"
while [[ $contract_migration_complete -ne 1 ]]
do
if [[ -f "$CIC_DATA_DIR/.env" ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env
then
echo "🤜💥🤛 data-seeding found the output of contract-migration!"
source /tmp/cic/config/.env
env
contract_migration_complete=1
./scripts/run_ussd_user_imports.sh
elif [[ $retry_count -ge $retry_limit ]]
then
echo "😢 data-seeding timeout waiting for contract migration to finish." >&2
exit 1
else
echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..."
((retry_count= $retry_count + $retry_sleep))
sleep $retry_sleep
fi
done
if [ $? -ne "0" ]; then
>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 3 - data seeding"
exit 1;
fi
>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 3 - data seeding"
fi

View File

@ -0,0 +1,3 @@
#! /bin/bash
python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR

View File

@ -1,8 +1,9 @@
#! /bin/sh
#! /bin/bash
set -u
set -e
while getopts ":n:o:g:" opt; do
case $opt in
n) NUMBER_OF_USERS="$OPTARG"
@ -41,14 +42,24 @@ echo "purging existing ussd tasks..."
celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f
echo "running import_balance in the background..."
python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
-r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 2>&1 &
#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
# -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR > nohup.out 2> nohup.err < /dev/null &
#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
# -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR
echo "import_balance pid: $!"
echo "importing accounts"
python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out
sleep 3600
echo "data importing finished!"
echo "run verify manually:"
echo "python verify.py -v -c config -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER --token-symbol $TOKEN_SYMBOL $OUT_DIR"

View File

@ -154,21 +154,25 @@ for t in custodial_tests:
logg.info('activating custodial module'.format(t))
break
cols = os.get_terminal_size().columns
def to_terminalwidth(s):
ss = s.ljust(int(cols)-1)
ss += "\r"
return ss
def default_outfunc(s):
ss = to_terminalwidth(s)
sys.stdout.write(ss)
outfunc = default_outfunc
if logg.isEnabledFor(logging.DEBUG):
outfunc = logg.debug
#cols = os.get_terminal_size().columns
##disabled because:
#data-seeding_1 | Traceback (most recent call last):
#data-seeding_1 | File "verify.py", line 157, in <module>
#data-seeding_1 | cols = os.get_terminal_size().columns
#data-seeding_1 | OSError: [Errno 25] Inappropriate ioctl for device
#contract-migration_1 | DEBUG:create_account_script:register False
#def to_terminalwidth(s):
# ss = s.ljust(int(cols)-1)
# ss += "\r"
# return ss
#
#def default_outfunc(s):
# ss = to_terminalwidth(s)
# sys.stdout.write(ss)
#outfunc = default_outfunc
#if logg.isEnabledFor(logging.DEBUG):
outfunc = logg.debug
def send_ussd_request(address, data_dir):
upper_address = strip_0x(address).upper()

View File

120
docker-compose.test.yml Normal file
View File

@ -0,0 +1,120 @@
version: "3"
services:
contract-migration:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration
build:
context: apps/contract-migration
dockerfile: docker/Dockerfile
args:
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
pip_extra_args: $PIP_EXTRA_ARGS
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
ETH_PROVIDER: http://eth:8545
# And these two are for wait-for-it (could parse this)
DEV_USE_DOCKER_WAIT_SCRIPT: 1
ETH_PROVIDER_HOST: eth
ETH_PROVIDER_PORT: 8545
CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_USER: ${DATABASE_USER:-postgres}
REDIS_HOST: ${REDIS_HOST:-redis}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DB: ${REDIS_DB:-0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
RUN_MASK: ${RUN_MASK:-0}
DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
#DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
TOKEN_NAME: $TOKEN_NAME
TOKEN_DECIMALS: $TOKEN_DECIMALS
TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
TOKEN_TYPE: $TOKEN_TYPE
#CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
command: ["./run_job.sh"]
#command: ["./reset.sh"]
volumes:
- contract-config:/tmp/cic/config
data-seeding-tasker:
build:
context: apps/data-seeding
dockerfile: docker/Dockerfile
args:
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
pip_extra_args: $PIP_EXTRA_ARGS
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
ETH_PROVIDER: http://eth:8545
# And these two are for wait-for-it (could parse this)
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
DEV_USE_DOCKER_WAIT_SCRIPT: 1
ETH_PROVIDER_HOST: eth
ETH_PROVIDER_PORT: 8545
REDIS_HOST: ${REDIS_HOST:-redis}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DB: ${REDIS_DB:-0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
RUN_MASK: ${RUN_MASK:-0} # 7
TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
USER_USSD_HOST: cic-user-ussd-server
USER_USSD_PORT: 9000
KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
OUT_DIR: out
NUMBER_OF_USERS: 10
#command: "python cic_ussd/import_balance.py -v -c config -p $$ETH_PROVIDER -r $$CIC_REGISTRY_ADDRESS --token-symbol $$TOKEN_SYMBOL -y $$KEYSTORE_FILE_PATH $$OUT_DIR"
command: "bash scripts/run_tasker.sh"
volumes:
- contract-config:/tmp/cic/config
data-seeding:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding
build:
context: apps/data-seeding
dockerfile: docker/Dockerfile
args:
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
pip_extra_args: $PIP_EXTRA_ARGS
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
ETH_PROVIDER: http://eth:8545
# And these two are for wait-for-it (could parse this)
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
DEV_USE_DOCKER_WAIT_SCRIPT: 1
ETH_PROVIDER_HOST: eth
ETH_PROVIDER_PORT: 8545
REDIS_HOST: ${REDIS_HOST:-redis}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DB: ${REDIS_DB:-0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
RUN_MASK: ${RUN_MASK:-0} # 7
TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
USER_USSD_HOST: cic-user-ussd-server
USER_USSD_PORT: 9000
KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
OUT_DIR: out
NUMBER_OF_USERS: 10
command: ["bash", "./scripts/run_job.sh"]
volumes:
- contract-config:/tmp/cic/config

View File

@ -8,25 +8,10 @@ volumes:
bloxberg-data: {}
contract-config: {}
networks:
default:
name: cic-network
services:
# eth:
# image: trufflesuite/ganache-cli
# ports:
# - ${HTTP_PORT_ETH:-8545}
# - ${WS_PORT_ETH:-8546}
# # Note! -e switch doesnt work, whatever you put there, it will be 100
# command: "-i 8996 -e 1000 -l 90000000 \
# -m '${DEV_MNEMONIC:-\"history stumble mystery avoid embark arrive mom foil pledge keep grain dice\"}' \
# -v --db /tmp/cic/ganache/ganache.db \
# --noVMErrorsOnRPCResponse --allowUnlimitedContractSize"
# volumes:
# - ganache-db:/tmp/cic/ganache
eth:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/bloxberg-node
build:
context: apps/bloxbergValidatorSetup
restart: unless-stopped
@ -70,64 +55,8 @@ services:
volumes:
- bee-data:/tmp/cic/bee
contract-migration:
profiles:
- migrations
build:
context: apps/contract-migration
dockerfile: docker/Dockerfile
args:
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
pip_extra_args: $PIP_EXTRA_ARGS
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
ETH_PROVIDER: http://eth:8545
# And these two are for wait-for-it (could parse this)
DEV_USE_DOCKER_WAIT_SCRIPT: 1
ETH_PROVIDER_HOST: eth
ETH_PROVIDER_PORT: 8545
CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_USER: ${DATABASE_USER:-postgres}
REDIS_HOST: ${REDIS_HOST:-redis}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DB: ${REDIS_DB:-0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
#DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
TOKEN_NAME: $TOKEN_NAME
TOKEN_DECIMALS: $TOKEN_DECIMALS
TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
TOKEN_TYPE: $TOKEN_TYPE
#CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
command: ["./run_job.sh"]
#command: ["./reset.sh"]
depends_on:
- eth
- postgres
- redis
- cic-eth-tasker
volumes:
- contract-config:/tmp/cic/config
cic-cache-tracker:
profiles:
- cache
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
@ -164,8 +93,7 @@ services:
- contract-config:/tmp/cic/config/:ro
cic-cache-tasker:
profiles:
- cache
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
@ -203,8 +131,7 @@ services:
- contract-config:/tmp/cic/config/:ro
cic-cache-server:
profiles:
- cache
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
@ -236,10 +163,10 @@ services:
cic-eth-tasker:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER
@ -286,10 +213,10 @@ services:
# command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
cic-eth-tracker:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
ETH_PROVIDER: http://eth:8545
DATABASE_USER: ${DATABASE_USER:-grassroots}
@ -325,10 +252,10 @@ services:
cic-eth-dispatcher:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
ETH_PROVIDER: http://eth:8545
DATABASE_USER: ${DATABASE_USER:-grassroots}
@ -365,10 +292,10 @@ services:
cic-eth-retrier:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
environment:
ETH_PROVIDER: http://eth:8545
DATABASE_USER: ${DATABASE_USER:-grassroots}
@ -408,6 +335,7 @@ services:
cic-notify-tasker:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify
build:
context: apps/cic-notify
dockerfile: docker/Dockerfile
@ -436,8 +364,7 @@ services:
cic-meta-server:
profiles:
- custodial-meta
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta
hostname: meta
build:
context: apps/cic-meta
@ -471,8 +398,7 @@ services:
# command: "/root/start_server.sh -vv"
cic-user-ussd-server:
profiles:
- custodial-ussd
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd
build:
context: apps/cic-ussd
dockerfile: docker/Dockerfile
@ -502,8 +428,7 @@ services:
command: "/root/start_cic_user_ussd_server.sh -vv"
cic-user-server:
profiles:
- custodial-ussd
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd
build:
context: apps/cic-ussd
dockerfile: docker/Dockerfile
@ -526,8 +451,7 @@ services:
command: "/root/start_cic_user_server.sh -vv"
cic-user-tasker:
profiles:
- custodial-ussd
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd
build:
context: apps/cic-ussd/
dockerfile: docker/Dockerfile