adding changes
This commit is contained in:
parent
6d957eea6e
commit
68f4ed383a
@ -8,3 +8,7 @@ REDIS_PORT=6379
|
||||
USER_USSD_HOST=cic-user-ussd-server
|
||||
USER_USSD_PORT=9000
|
||||
KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
|
||||
CIC_DATA_DIR=config/
|
||||
OUT_DIR=out
|
||||
NUMBER_OF_USERS=10
|
||||
META_URL=http://cic-meta-server:63380
|
||||
|
||||
@ -9,7 +9,7 @@ COPY package.json \
|
||||
package-lock.json \
|
||||
.
|
||||
|
||||
RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
|
||||
RUN npm ci --production
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
|
||||
@ -3,7 +3,31 @@
|
||||
if [[ $((RUN_MASK & 3)) -eq 3 ]]
|
||||
then
|
||||
>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 3 - data seeding"
|
||||
./scripts/run_ussd_user_imports.sh
|
||||
|
||||
contract_migration_complete=0
|
||||
retry_count=0
|
||||
retry_sleep=30 #seconds
|
||||
retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))"
|
||||
while [[ $contract_migration_complete -ne 1 ]]
|
||||
do
|
||||
if [[ -f "$CIC_DATA_DIR/.env" ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env
|
||||
then
|
||||
echo "🤜💥🤛 data-seeding found the output of contract-migration!"
|
||||
source /tmp/cic/config/.env
|
||||
env
|
||||
contract_migration_complete=1
|
||||
./scripts/run_ussd_user_imports.sh
|
||||
elif [[ $retry_count -ge $retry_limit ]]
|
||||
then
|
||||
echo "😢 data-seeding timeout waiting for contract migration to finish." >&2
|
||||
exit 1
|
||||
else
|
||||
echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..."
|
||||
((retry_count= $retry_count + $retry_sleep))
|
||||
sleep $retry_sleep
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $? -ne "0" ]; then
|
||||
>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 3 - data seeding"
|
||||
exit 1;
|
||||
|
||||
3
apps/data-seeding/scripts/run_tasker.sh
Normal file
3
apps/data-seeding/scripts/run_tasker.sh
Normal file
@ -0,0 +1,3 @@
|
||||
#! /bin/bash
|
||||
|
||||
python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR
|
||||
@ -3,30 +3,6 @@
|
||||
set -u
|
||||
set -e
|
||||
|
||||
contract_migration_complete=0
|
||||
retry_count=0
|
||||
retry_sleep=30 #seconds
|
||||
retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))"
|
||||
while [[ $contract_migration_complete -ne 1 ]]
|
||||
do
|
||||
if [[ -f "$CIC_DATA_DIR/.env" ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env
|
||||
then
|
||||
echo "🤜💥🤛 data-seeding found the output of contract-migration!"
|
||||
source /tmp/cic/config/.env
|
||||
env
|
||||
contract_migration_complete=1
|
||||
elif [[ $retry_count -ge $retry_limit ]]
|
||||
then
|
||||
echo "😢 data-seeding timeout waiting for contract migration to finish." >&2
|
||||
exit 1
|
||||
else
|
||||
echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..."
|
||||
((retry_count= $retry_count + $retry_sleep))
|
||||
sleep $retry_sleep
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
|
||||
while getopts ":n:o:g:" opt; do
|
||||
case $opt in
|
||||
@ -66,14 +42,24 @@ echo "purging existing ussd tasks..."
|
||||
|
||||
celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f
|
||||
|
||||
|
||||
echo "running import_balance in the background..."
|
||||
|
||||
python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
|
||||
-r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 2>&1 &
|
||||
#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
|
||||
# -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR > nohup.out 2> nohup.err < /dev/null &
|
||||
|
||||
#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
|
||||
# -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR
|
||||
|
||||
echo "import_balance pid: $!"
|
||||
|
||||
echo "importing accounts"
|
||||
|
||||
python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out
|
||||
|
||||
sleep 3600
|
||||
|
||||
echo "data importing finished!"
|
||||
|
||||
echo "run verify manually:"
|
||||
echo "python verify.py -v -c config -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER --token-symbol $TOKEN_SYMBOL $OUT_DIR"
|
||||
|
||||
|
||||
@ -154,21 +154,25 @@ for t in custodial_tests:
|
||||
logg.info('activating custodial module'.format(t))
|
||||
break
|
||||
|
||||
cols = os.get_terminal_size().columns
|
||||
|
||||
|
||||
def to_terminalwidth(s):
|
||||
ss = s.ljust(int(cols)-1)
|
||||
ss += "\r"
|
||||
return ss
|
||||
|
||||
def default_outfunc(s):
|
||||
ss = to_terminalwidth(s)
|
||||
sys.stdout.write(ss)
|
||||
outfunc = default_outfunc
|
||||
if logg.isEnabledFor(logging.DEBUG):
|
||||
outfunc = logg.debug
|
||||
#cols = os.get_terminal_size().columns
|
||||
|
||||
##disabled because:
|
||||
#data-seeding_1 | Traceback (most recent call last):
|
||||
#data-seeding_1 | File "verify.py", line 157, in <module>
|
||||
#data-seeding_1 | cols = os.get_terminal_size().columns
|
||||
#data-seeding_1 | OSError: [Errno 25] Inappropriate ioctl for device
|
||||
#contract-migration_1 | DEBUG:create_account_script:register False
|
||||
#def to_terminalwidth(s):
|
||||
# ss = s.ljust(int(cols)-1)
|
||||
# ss += "\r"
|
||||
# return ss
|
||||
#
|
||||
#def default_outfunc(s):
|
||||
# ss = to_terminalwidth(s)
|
||||
# sys.stdout.write(ss)
|
||||
#outfunc = default_outfunc
|
||||
#if logg.isEnabledFor(logging.DEBUG):
|
||||
outfunc = logg.debug
|
||||
|
||||
def send_ussd_request(address, data_dir):
|
||||
upper_address = strip_0x(address).upper()
|
||||
|
||||
0
docker-compose.fontend.yml
Normal file
0
docker-compose.fontend.yml
Normal file
120
docker-compose.test.yml
Normal file
120
docker-compose.test.yml
Normal file
@ -0,0 +1,120 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
contract-migration:
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration
|
||||
build:
|
||||
context: apps/contract-migration
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
|
||||
pip_extra_args: $PIP_EXTRA_ARGS
|
||||
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
|
||||
environment:
|
||||
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
|
||||
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
|
||||
ETH_PROVIDER: http://eth:8545
|
||||
# And these two are for wait-for-it (could parse this)
|
||||
DEV_USE_DOCKER_WAIT_SCRIPT: 1
|
||||
ETH_PROVIDER_HOST: eth
|
||||
ETH_PROVIDER_PORT: 8545
|
||||
CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
|
||||
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
|
||||
DATABASE_HOST: ${DATABASE_HOST:-postgres}
|
||||
DATABASE_PORT: ${DATABASE_PORT:-5432}
|
||||
DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
|
||||
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
|
||||
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
|
||||
DATABASE_USER: ${DATABASE_USER:-postgres}
|
||||
REDIS_HOST: ${REDIS_HOST:-redis}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DB: ${REDIS_DB:-0}
|
||||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
|
||||
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
|
||||
RUN_MASK: ${RUN_MASK:-0}
|
||||
DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
|
||||
#DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
|
||||
DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
|
||||
CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
|
||||
TOKEN_NAME: $TOKEN_NAME
|
||||
TOKEN_DECIMALS: $TOKEN_DECIMALS
|
||||
TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
|
||||
TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
|
||||
TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
|
||||
TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
|
||||
TOKEN_TYPE: $TOKEN_TYPE
|
||||
#CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
|
||||
command: ["./run_job.sh"]
|
||||
#command: ["./reset.sh"]
|
||||
volumes:
|
||||
- contract-config:/tmp/cic/config
|
||||
|
||||
|
||||
data-seeding-tasker:
|
||||
build:
|
||||
context: apps/data-seeding
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
|
||||
pip_extra_args: $PIP_EXTRA_ARGS
|
||||
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
|
||||
environment:
|
||||
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
|
||||
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
|
||||
ETH_PROVIDER: http://eth:8545
|
||||
# And these two are for wait-for-it (could parse this)
|
||||
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
|
||||
DEV_USE_DOCKER_WAIT_SCRIPT: 1
|
||||
ETH_PROVIDER_HOST: eth
|
||||
ETH_PROVIDER_PORT: 8545
|
||||
REDIS_HOST: ${REDIS_HOST:-redis}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DB: ${REDIS_DB:-0}
|
||||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
|
||||
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
|
||||
RUN_MASK: ${RUN_MASK:-0} # 7
|
||||
TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
|
||||
USER_USSD_HOST: cic-user-ussd-server
|
||||
USER_USSD_PORT: 9000
|
||||
KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
|
||||
OUT_DIR: out
|
||||
NUMBER_OF_USERS: 10
|
||||
#command: "python cic_ussd/import_balance.py -v -c config -p $$ETH_PROVIDER -r $$CIC_REGISTRY_ADDRESS --token-symbol $$TOKEN_SYMBOL -y $$KEYSTORE_FILE_PATH $$OUT_DIR"
|
||||
command: "bash scripts/run_tasker.sh"
|
||||
volumes:
|
||||
- contract-config:/tmp/cic/config
|
||||
|
||||
data-seeding:
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding
|
||||
build:
|
||||
context: apps/data-seeding
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
|
||||
pip_extra_args: $PIP_EXTRA_ARGS
|
||||
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
|
||||
environment:
|
||||
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
|
||||
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
|
||||
ETH_PROVIDER: http://eth:8545
|
||||
# And these two are for wait-for-it (could parse this)
|
||||
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
|
||||
DEV_USE_DOCKER_WAIT_SCRIPT: 1
|
||||
ETH_PROVIDER_HOST: eth
|
||||
ETH_PROVIDER_PORT: 8545
|
||||
REDIS_HOST: ${REDIS_HOST:-redis}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DB: ${REDIS_DB:-0}
|
||||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
|
||||
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
|
||||
RUN_MASK: ${RUN_MASK:-0} # 7
|
||||
TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
|
||||
USER_USSD_HOST: cic-user-ussd-server
|
||||
USER_USSD_PORT: 9000
|
||||
KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
|
||||
OUT_DIR: out
|
||||
NUMBER_OF_USERS: 10
|
||||
command: ["bash", "./scripts/run_job.sh"]
|
||||
volumes:
|
||||
- contract-config:/tmp/cic/config
|
||||
|
||||
@ -55,93 +55,6 @@ services:
|
||||
volumes:
|
||||
- bee-data:/tmp/cic/bee
|
||||
|
||||
contract-migration:
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration
|
||||
build:
|
||||
context: apps/contract-migration
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
|
||||
pip_extra_args: $PIP_EXTRA_ARGS
|
||||
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
|
||||
environment:
|
||||
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
|
||||
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
|
||||
ETH_PROVIDER: http://eth:8545
|
||||
# And these two are for wait-for-it (could parse this)
|
||||
DEV_USE_DOCKER_WAIT_SCRIPT: 1
|
||||
ETH_PROVIDER_HOST: eth
|
||||
ETH_PROVIDER_PORT: 8545
|
||||
CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
|
||||
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
|
||||
DATABASE_HOST: ${DATABASE_HOST:-postgres}
|
||||
DATABASE_PORT: ${DATABASE_PORT:-5432}
|
||||
DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
|
||||
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
|
||||
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
|
||||
DATABASE_USER: ${DATABASE_USER:-postgres}
|
||||
REDIS_HOST: ${REDIS_HOST:-redis}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DB: ${REDIS_DB:-0}
|
||||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
|
||||
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
|
||||
RUN_MASK: ${RUN_MASK:-0}
|
||||
DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
|
||||
#DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
|
||||
DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
|
||||
CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
|
||||
TOKEN_NAME: $TOKEN_NAME
|
||||
TOKEN_DECIMALS: $TOKEN_DECIMALS
|
||||
TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
|
||||
TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
|
||||
TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
|
||||
TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
|
||||
TOKEN_TYPE: $TOKEN_TYPE
|
||||
#CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
|
||||
command: ["./run_job.sh"]
|
||||
#command: ["./reset.sh"]
|
||||
depends_on:
|
||||
- eth
|
||||
- postgres
|
||||
- redis
|
||||
- cic-eth-tasker
|
||||
volumes:
|
||||
- contract-config:/tmp/cic/config
|
||||
|
||||
data-seeding:
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding
|
||||
build:
|
||||
context: apps/data-seeding
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
|
||||
pip_extra_args: $PIP_EXTRA_ARGS
|
||||
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
|
||||
environment:
|
||||
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
|
||||
# ETH_PROVIDER should be broken out into host/port but cic-eth expects this
|
||||
ETH_PROVIDER: http://eth:8545
|
||||
# And these two are for wait-for-it (could parse this)
|
||||
CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
|
||||
DEV_USE_DOCKER_WAIT_SCRIPT: 1
|
||||
ETH_PROVIDER_HOST: eth
|
||||
ETH_PROVIDER_PORT: 8545
|
||||
REDIS_HOST: ${REDIS_HOST:-redis}
|
||||
REDIS_PORT: ${REDIS_PORT:-6379}
|
||||
REDIS_DB: ${REDIS_DB:-0}
|
||||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
|
||||
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
|
||||
RUN_MASK: ${RUN_MASK:-0} # 7
|
||||
TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
|
||||
USER_USSD_HOST: cic-user-ussd-server
|
||||
USER_USSD_PORT: 9000
|
||||
KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
|
||||
OUT_DIR: out
|
||||
NUMBER_OF_USERS: 10
|
||||
command: ["bash", "./scripts/run_job.sh"]
|
||||
volumes:
|
||||
- contract-config:/tmp/cic/config
|
||||
|
||||
cic-cache-tracker:
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
|
||||
build:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user