From 503934037f3a7748f725400885c3c50d16f5cb99 Mon Sep 17 00:00:00 2001 From: Blair V Date: Wed, 22 Sep 2021 11:42:32 +0300 Subject: [PATCH] chore: adding scripts to wait for contract migration output --- apps/contract-migration/.bash_history | 6 + apps/contract-migration/docker/init_readyz.sh | 8 +- apps/contract-migration/reset.sh | 7 +- apps/contract-migration/run_job.sh | 5 + apps/data-seeding/import_ussd.sh | 0 apps/data-seeding/scripts/get_readyz.sh | 10 + apps/data-seeding/scripts/run_job.sh | 15 ++ .../scripts/run_ussd_user_imports.sh | 0 apps/data-seeding/scripts/wait-for-it.sh | 182 ++++++++++++++++++ docker-compose.yml | 27 +-- scripts/dump.sh | 2 +- 11 files changed, 238 insertions(+), 24 deletions(-) create mode 100644 apps/contract-migration/.bash_history mode change 100644 => 100755 apps/data-seeding/import_ussd.sh create mode 100755 apps/data-seeding/scripts/get_readyz.sh create mode 100755 apps/data-seeding/scripts/run_job.sh mode change 100644 => 100755 apps/data-seeding/scripts/run_ussd_user_imports.sh create mode 100755 apps/data-seeding/scripts/wait-for-it.sh diff --git a/apps/contract-migration/.bash_history b/apps/contract-migration/.bash_history new file mode 100644 index 00000000..48e8e225 --- /dev/null +++ b/apps/contract-migration/.bash_history @@ -0,0 +1,6 @@ +ls /tmp/cic/config/ +cat /tmp/cic/config/env_reset +exit +cat /tmp/cic/config/ +cat /tmp/cic/config/env_reset +exit diff --git a/apps/contract-migration/docker/init_readyz.sh b/apps/contract-migration/docker/init_readyz.sh index 349dafed..dd423d66 100755 --- a/apps/contract-migration/docker/init_readyz.sh +++ b/apps/contract-migration/docker/init_readyz.sh @@ -9,8 +9,12 @@ source $DEV_DATA_DIR/env_reset jq --arg CIC_REGISTRY_ADDRESS "$CIC_REGISTRY_ADDRESS" \ --arg CIC_TRUST_ADDRESS "$CIC_TRUST_ADDRESS" \ - --arg RUN_MASK "$RUN_MASK" \ - -n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS, "CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS, "RUN_MASK": $RUN_MASK}' > $DEV_DATA_DIR/health/readyz + --arg CIC_DEFAULT_TOKEN_SYMBOL "$CIC_DEFAULT_TOKEN_SYMBOL"\ + --arg TOKEN_NAME "$TOKEN_NAME"\ + -n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS, + "CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS, + "CIC_DEFAULT_TOKEN_SYMBOL": $CIC_DEFAULT_TOKEN_SYMBOL, + "TOKEN_NAME": $TOKEN_NAME}' > $DEV_DATA_DIR/health/readyz cd $DEV_DATA_DIR/health diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index f4d45f7f..fe9fabcb 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -112,10 +112,13 @@ export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL echo "Writing env_reset file ..." -echo "export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS +# note these are also written in init_readyz.sh +cat < $DEV_DATA_DIR/env_reset +export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS +export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL export TOKEN_NAME=$TOKEN_NAME -" >> "${DEV_DATA_DIR}"/env_reset +EOF set +a set +e diff --git a/apps/contract-migration/run_job.sh b/apps/contract-migration/run_job.sh index fe4a0acd..50cd8154 100755 --- a/apps/contract-migration/run_job.sh +++ b/apps/contract-migration/run_job.sh @@ -22,6 +22,11 @@ fi if [[ $((RUN_MASK & 2)) -eq 2 ]] then >&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 2 - custodial service initialization" + if [ -f $DEV_DATA_DIR/env_reset ]; then + echo "found previous env_reset file" + cat $DEV_DATA_DIR/env_reset + source $DEV_DATA_DIR/env_reset; + fi ./seed_cic_eth.sh if [ $? -ne "0" ]; then >&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 2 - custodial service initialization" diff --git a/apps/data-seeding/import_ussd.sh b/apps/data-seeding/import_ussd.sh old mode 100644 new mode 100755 diff --git a/apps/data-seeding/scripts/get_readyz.sh b/apps/data-seeding/scripts/get_readyz.sh new file mode 100755 index 00000000..dd87a57f --- /dev/null +++ b/apps/data-seeding/scripts/get_readyz.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +set -e +set -u + +echo "fetching migration variables from $CONTRACT_MIGRATION_URL" +for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do + echo "exporting $s" + export $s +done diff --git a/apps/data-seeding/scripts/run_job.sh b/apps/data-seeding/scripts/run_job.sh new file mode 100755 index 00000000..698c503e --- /dev/null +++ b/apps/data-seeding/scripts/run_job.sh @@ -0,0 +1,15 @@ +#! /bin/bash +set -e +set -u +env +WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600} + +if [[ "$CONTRACT_MIGRATION_URL" ]]; then + echo "waiting for $CONTRACT_MIGRATION_URL/readyz" + ./scripts/wait-for-it.sh $CONTRACT_MIGRATION_URL -t $WAIT_FOR_TIMEOUT + source ./scripts/get_readyz.sh # set env vars form endpoint + ./import_ussd.sh +else + ./import_ussd.sh +fi + diff --git a/apps/data-seeding/scripts/run_ussd_user_imports.sh b/apps/data-seeding/scripts/run_ussd_user_imports.sh old mode 100644 new mode 100755 diff --git a/apps/data-seeding/scripts/wait-for-it.sh b/apps/data-seeding/scripts/wait-for-it.sh new file mode 100755 index 00000000..3974640b --- /dev/null +++ b/apps/data-seeding/scripts/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e7ba7e98..0e7526a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,8 +60,6 @@ services: contract-migration: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest} - profiles: - - migrations ports: - 8012:8000 build: @@ -72,10 +70,10 @@ services: pip_extra_args: $PIP_EXTRA_ARGS EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS - # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest environment: RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} DEV_USE_DOCKER_WAIT_SCRIPT: 1 CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} @@ -98,13 +96,16 @@ services: DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE TOKEN_NAME: ${TOKEN_NAME:-Giftable Token} TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} + TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token} + #TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token} + #TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET} + #TOKEN_TYPE: ${TOKEN_TYPE:-erc20_demurrage_token } TOKEN_DECIMALS: $TOKEN_DECIMALS TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS - TOKEN_TYPE: $TOKEN_TYPE SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} restart: on-failure command: ["./run_job.sh"] @@ -157,14 +158,14 @@ services: USSD_PROVIDER: http://cic-user-ussd-server:9000 CELERY_QUEUE: cic-import-ussd EXCLUSIONS: ussd - command: bash import_ussd.sh + CONTRACT_MIGRATION_URL: contract-migration:8000 + command: bash ./scripts/run_job.sh volumes: - contract-config:/tmp/cic/config/:ro + #- ./apps/data-seeding:/root cic-cache-tracker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - profiles: - - cache build: context: apps/cic-cache dockerfile: docker/Dockerfile @@ -203,8 +204,6 @@ services: cic-cache-tasker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - profiles: - - cache build: context: apps/cic-cache dockerfile: docker/Dockerfile @@ -242,8 +241,6 @@ services: cic-cache-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - profiles: - - cache build: context: apps/cic-cache dockerfile: docker/Dockerfile @@ -544,8 +541,6 @@ services: cic-meta-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest} - profiles: - - custodial-meta hostname: meta build: context: apps/cic-meta @@ -578,8 +573,6 @@ services: cic-user-ussd-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - profiles: - - custodial-ussd build: context: apps/cic-ussd dockerfile: docker/Dockerfile @@ -609,8 +602,6 @@ services: cic-user-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - profiles: - - custodial-ussd build: context: apps/cic-ussd dockerfile: docker/Dockerfile @@ -633,8 +624,6 @@ services: cic-user-tasker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - profiles: - - custodial-ussd build: context: apps/cic-ussd/ dockerfile: docker/Dockerfile diff --git a/scripts/dump.sh b/scripts/dump.sh index 814c451f..642285e0 100755 --- a/scripts/dump.sh +++ b/scripts/dump.sh @@ -1 +1 @@ -docker run -t -v --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset +docker run -v --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset