From 68f4ed383aab26ebbde5625cbefa7a5cb0285a8c Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Tue, 10 Aug 2021 12:43:11 -0700 Subject: [PATCH] adding changes --- apps/data-seeding/docker/.env | 4 + apps/data-seeding/docker/Dockerfile | 2 +- apps/data-seeding/scripts/run_job.sh | 26 +++- apps/data-seeding/scripts/run_tasker.sh | 3 + .../scripts/run_ussd_user_imports.sh | 40 ++---- apps/data-seeding/verify.py | 32 +++-- docker-compose.fontend.yml | 0 docker-compose.test.yml | 120 ++++++++++++++++++ docker-compose.yml | 87 ------------- 9 files changed, 184 insertions(+), 130 deletions(-) create mode 100644 apps/data-seeding/scripts/run_tasker.sh create mode 100644 docker-compose.fontend.yml create mode 100644 docker-compose.test.yml diff --git a/apps/data-seeding/docker/.env b/apps/data-seeding/docker/.env index 44c596b5..0b674d1f 100644 --- a/apps/data-seeding/docker/.env +++ b/apps/data-seeding/docker/.env @@ -8,3 +8,7 @@ REDIS_PORT=6379 USER_USSD_HOST=cic-user-ussd-server USER_USSD_PORT=9000 KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c +CIC_DATA_DIR=config/ +OUT_DIR=out +NUMBER_OF_USERS=10 +META_URL=http://cic-meta-server:63380 diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index c4f6ccf4..81dee79f 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -9,7 +9,7 @@ COPY package.json \ package-lock.json \ . -RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install +RUN npm ci --production COPY requirements.txt . diff --git a/apps/data-seeding/scripts/run_job.sh b/apps/data-seeding/scripts/run_job.sh index 0bba61a6..6d63824d 100644 --- a/apps/data-seeding/scripts/run_job.sh +++ b/apps/data-seeding/scripts/run_job.sh @@ -3,7 +3,31 @@ if [[ $((RUN_MASK & 3)) -eq 3 ]] then >&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 3 - data seeding" - ./scripts/run_ussd_user_imports.sh + + contract_migration_complete=0 + retry_count=0 + retry_sleep=30 #seconds + retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))" + while [[ $contract_migration_complete -ne 1 ]] + do + if [[ -f "$CIC_DATA_DIR/.env" ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env + then + echo "🤜💥🤛 data-seeding found the output of contract-migration!" + source /tmp/cic/config/.env + env + contract_migration_complete=1 + ./scripts/run_ussd_user_imports.sh + elif [[ $retry_count -ge $retry_limit ]] + then + echo "😢 data-seeding timeout waiting for contract migration to finish." >&2 + exit 1 + else + echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..." + ((retry_count= $retry_count + $retry_sleep)) + sleep $retry_sleep + fi + done + if [ $? -ne "0" ]; then >&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 3 - data seeding" exit 1; diff --git a/apps/data-seeding/scripts/run_tasker.sh b/apps/data-seeding/scripts/run_tasker.sh new file mode 100644 index 00000000..0ac3a55f --- /dev/null +++ b/apps/data-seeding/scripts/run_tasker.sh @@ -0,0 +1,3 @@ +#! /bin/bash + +python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR diff --git a/apps/data-seeding/scripts/run_ussd_user_imports.sh b/apps/data-seeding/scripts/run_ussd_user_imports.sh index 3ec2e70f..360673c5 100644 --- a/apps/data-seeding/scripts/run_ussd_user_imports.sh +++ b/apps/data-seeding/scripts/run_ussd_user_imports.sh @@ -3,30 +3,6 @@ set -u set -e -contract_migration_complete=0 -retry_count=0 -retry_sleep=30 #seconds -retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))" -while [[ $contract_migration_complete -ne 1 ]] -do - if [[ -f "$CIC_DATA_DIR/.env" ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env - then - echo "🤜💥🤛 data-seeding found the output of contract-migration!" - source /tmp/cic/config/.env - env - contract_migration_complete=1 - elif [[ $retry_count -ge $retry_limit ]] - then - echo "😢 data-seeding timeout waiting for contract migration to finish." >&2 - exit 1 - else - echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..." - ((retry_count= $retry_count + $retry_sleep)) - sleep $retry_sleep - fi -done - - while getopts ":n:o:g:" opt; do case $opt in @@ -66,14 +42,24 @@ echo "purging existing ussd tasks..." celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f - echo "running import_balance in the background..." -python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \ - -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 2>&1 & +#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \ +# -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR > nohup.out 2> nohup.err < /dev/null & + +#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \ +# -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR echo "import_balance pid: $!" echo "importing accounts" python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out + +sleep 3600 + +echo "data importing finished!" + +echo "run verify manually:" +echo "python verify.py -v -c config -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER --token-symbol $TOKEN_SYMBOL $OUT_DIR" + diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 1c227740..bb11c5e7 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -154,21 +154,25 @@ for t in custodial_tests: logg.info('activating custodial module'.format(t)) break -cols = os.get_terminal_size().columns - - -def to_terminalwidth(s): - ss = s.ljust(int(cols)-1) - ss += "\r" - return ss - -def default_outfunc(s): - ss = to_terminalwidth(s) - sys.stdout.write(ss) -outfunc = default_outfunc -if logg.isEnabledFor(logging.DEBUG): - outfunc = logg.debug +#cols = os.get_terminal_size().columns +##disabled because: +#data-seeding_1 | Traceback (most recent call last): +#data-seeding_1 | File "verify.py", line 157, in +#data-seeding_1 | cols = os.get_terminal_size().columns +#data-seeding_1 | OSError: [Errno 25] Inappropriate ioctl for device +#contract-migration_1 | DEBUG:create_account_script:register False +#def to_terminalwidth(s): +# ss = s.ljust(int(cols)-1) +# ss += "\r" +# return ss +# +#def default_outfunc(s): +# ss = to_terminalwidth(s) +# sys.stdout.write(ss) +#outfunc = default_outfunc +#if logg.isEnabledFor(logging.DEBUG): +outfunc = logg.debug def send_ussd_request(address, data_dir): upper_address = strip_0x(address).upper() diff --git a/docker-compose.fontend.yml b/docker-compose.fontend.yml new file mode 100644 index 00000000..e69de29b diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..93e70f37 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,120 @@ +version: "3" + +services: + contract-migration: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration + build: + context: apps/contract-migration + dockerfile: docker/Dockerfile + args: + pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} + pip_extra_args: $PIP_EXTRA_ARGS + # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + # ETH_PROVIDER should be broken out into host/port but cic-eth expects this + ETH_PROVIDER: http://eth:8545 + # And these two are for wait-for-it (could parse this) + DEV_USE_DOCKER_WAIT_SCRIPT: 1 + ETH_PROVIDER_HOST: eth + ETH_PROVIDER_PORT: 8545 + CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_USER: ${DATABASE_USER:-postgres} + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_DB: ${REDIS_DB:-0} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + RUN_MASK: ${RUN_MASK:-0} + DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0} + #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000} + DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1} + CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL + TOKEN_NAME: $TOKEN_NAME + TOKEN_DECIMALS: $TOKEN_DECIMALS + TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD + TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT + TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000} + TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS + TOKEN_TYPE: $TOKEN_TYPE + #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config} + command: ["./run_job.sh"] + #command: ["./reset.sh"] + volumes: + - contract-config:/tmp/cic/config + + + data-seeding-tasker: + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} + pip_extra_args: $PIP_EXTRA_ARGS + # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + # ETH_PROVIDER should be broken out into host/port but cic-eth expects this + ETH_PROVIDER: http://eth:8545 + # And these two are for wait-for-it (could parse this) + CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} + DEV_USE_DOCKER_WAIT_SCRIPT: 1 + ETH_PROVIDER_HOST: eth + ETH_PROVIDER_PORT: 8545 + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_DB: ${REDIS_DB:-0} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + RUN_MASK: ${RUN_MASK:-0} # 7 + TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} + USER_USSD_HOST: cic-user-ussd-server + USER_USSD_PORT: 9000 + KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c" + OUT_DIR: out + NUMBER_OF_USERS: 10 + #command: "python cic_ussd/import_balance.py -v -c config -p $$ETH_PROVIDER -r $$CIC_REGISTRY_ADDRESS --token-symbol $$TOKEN_SYMBOL -y $$KEYSTORE_FILE_PATH $$OUT_DIR" + command: "bash scripts/run_tasker.sh" + volumes: + - contract-config:/tmp/cic/config + + data-seeding: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} + pip_extra_args: $PIP_EXTRA_ARGS + # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + # ETH_PROVIDER should be broken out into host/port but cic-eth expects this + ETH_PROVIDER: http://eth:8545 + # And these two are for wait-for-it (could parse this) + CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} + DEV_USE_DOCKER_WAIT_SCRIPT: 1 + ETH_PROVIDER_HOST: eth + ETH_PROVIDER_PORT: 8545 + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_DB: ${REDIS_DB:-0} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + RUN_MASK: ${RUN_MASK:-0} # 7 + TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} + USER_USSD_HOST: cic-user-ussd-server + USER_USSD_PORT: 9000 + KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c" + OUT_DIR: out + NUMBER_OF_USERS: 10 + command: ["bash", "./scripts/run_job.sh"] + volumes: + - contract-config:/tmp/cic/config + diff --git a/docker-compose.yml b/docker-compose.yml index 5aec9fd8..a672a5cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,93 +55,6 @@ services: volumes: - bee-data:/tmp/cic/bee - contract-migration: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration - build: - context: apps/contract-migration - dockerfile: docker/Dockerfile - args: - pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} - pip_extra_args: $PIP_EXTRA_ARGS - # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest - environment: - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - # ETH_PROVIDER should be broken out into host/port but cic-eth expects this - ETH_PROVIDER: http://eth:8545 - # And these two are for wait-for-it (could parse this) - DEV_USE_DOCKER_WAIT_SCRIPT: 1 - ETH_PROVIDER_HOST: eth - ETH_PROVIDER_PORT: 8545 - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_USER: ${DATABASE_USER:-postgres} - REDIS_HOST: ${REDIS_HOST:-redis} - REDIS_PORT: ${REDIS_PORT:-6379} - REDIS_DB: ${REDIS_DB:-0} - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} - RUN_MASK: ${RUN_MASK:-0} - DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0} - #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000} - DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1} - CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL - TOKEN_NAME: $TOKEN_NAME - TOKEN_DECIMALS: $TOKEN_DECIMALS - TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD - TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT - TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000} - TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS - TOKEN_TYPE: $TOKEN_TYPE - #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config} - command: ["./run_job.sh"] - #command: ["./reset.sh"] - depends_on: - - eth - - postgres - - redis - - cic-eth-tasker - volumes: - - contract-config:/tmp/cic/config - - data-seeding: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding - build: - context: apps/data-seeding - dockerfile: docker/Dockerfile - args: - pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} - pip_extra_args: $PIP_EXTRA_ARGS - # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest - environment: - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - # ETH_PROVIDER should be broken out into host/port but cic-eth expects this - ETH_PROVIDER: http://eth:8545 - # And these two are for wait-for-it (could parse this) - CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} - DEV_USE_DOCKER_WAIT_SCRIPT: 1 - ETH_PROVIDER_HOST: eth - ETH_PROVIDER_PORT: 8545 - REDIS_HOST: ${REDIS_HOST:-redis} - REDIS_PORT: ${REDIS_PORT:-6379} - REDIS_DB: ${REDIS_DB:-0} - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} - RUN_MASK: ${RUN_MASK:-0} # 7 - TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} - USER_USSD_HOST: cic-user-ussd-server - USER_USSD_PORT: 9000 - KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c" - OUT_DIR: out - NUMBER_OF_USERS: 10 - command: ["bash", "./scripts/run_job.sh"] - volumes: - - contract-config:/tmp/cic/config - cic-cache-tracker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache build: