chore: adding scripts to wait for contract migration output

2021-09-22 11:42:32 +03:00 · 2021-09-22 11:42:32 +03:00 · 503934037f
commit 503934037f
parent b6b986e905
11 changed files with 238 additions and 24 deletions
--- a/apps/contract-migration/.bash_history
+++ b/apps/contract-migration/.bash_history
@ -0,0 +1,6 @@
+ls /tmp/cic/config/
+cat /tmp/cic/config/env_reset 
+exit
+cat /tmp/cic/config/
+cat /tmp/cic/config/env_reset 
+exit
--- a/apps/contract-migration/docker/init_readyz.sh
+++ b/apps/contract-migration/docker/init_readyz.sh
@ -9,8 +9,12 @@ source $DEV_DATA_DIR/env_reset

 jq --arg CIC_REGISTRY_ADDRESS "$CIC_REGISTRY_ADDRESS" \
   --arg CIC_TRUST_ADDRESS "$CIC_TRUST_ADDRESS" \
-   --arg RUN_MASK "$RUN_MASK" \
-   -n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS, "CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS, "RUN_MASK": $RUN_MASK}' > $DEV_DATA_DIR/health/readyz
+   --arg CIC_DEFAULT_TOKEN_SYMBOL "$CIC_DEFAULT_TOKEN_SYMBOL"\
+   --arg TOKEN_NAME "$TOKEN_NAME"\
+   -n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS, 
+        "CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS,
+        "CIC_DEFAULT_TOKEN_SYMBOL": $CIC_DEFAULT_TOKEN_SYMBOL,
+        "TOKEN_NAME": $TOKEN_NAME}' > $DEV_DATA_DIR/health/readyz

 cd $DEV_DATA_DIR/health

--- a/apps/contract-migration/reset.sh
+++ b/apps/contract-migration/reset.sh
@ -112,10 +112,13 @@ export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL

 echo "Writing env_reset file ..."

-echo "export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
+# note these are also written in init_readyz.sh
+cat <<EOF > $DEV_DATA_DIR/env_reset 
+export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
+export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS
 export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
 export TOKEN_NAME=$TOKEN_NAME
-" >> "${DEV_DATA_DIR}"/env_reset
+EOF

 set +a
 set +e
--- a/apps/contract-migration/run_job.sh
+++ b/apps/contract-migration/run_job.sh
@ -22,6 +22,11 @@ fi
 if [[ $((RUN_MASK & 2)) -eq 2 ]]
 then
 	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 2 - custodial service initialization"
+  if [ -f $DEV_DATA_DIR/env_reset ]; then
+    echo "found previous env_reset file"
+    cat $DEV_DATA_DIR/env_reset
+    source $DEV_DATA_DIR/env_reset;
+  fi
 	./seed_cic_eth.sh
 	if [ $? -ne "0" ]; then
 		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 2 - custodial service initialization"
--- a/apps/data-seeding/import_ussd.sh
+++ b/apps/data-seeding/import_ussd.sh
--- a/apps/data-seeding/scripts/get_readyz.sh
+++ b/apps/data-seeding/scripts/get_readyz.sh
@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done
--- a/apps/data-seeding/scripts/run_job.sh
+++ b/apps/data-seeding/scripts/run_job.sh
@ -0,0 +1,15 @@
+#! /bin/bash 
+set -e
+set -u
+env
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./scripts/wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./scripts/get_readyz.sh # set env vars form endpoint
+  ./import_ussd.sh
+else
+  ./import_ussd.sh
+fi
+
--- a/apps/data-seeding/scripts/run_ussd_user_imports.sh
+++ b/apps/data-seeding/scripts/run_ussd_user_imports.sh
--- a/apps/data-seeding/scripts/wait-for-it.sh
+++ b/apps/data-seeding/scripts/wait-for-it.sh
@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -60,8 +60,6 @@ services:

  contract-migration:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest}
-    profiles:
-      - migrations 
    ports:
      - 8012:8000
    build:
@ -72,10 +70,10 @@ services:
        pip_extra_args: $PIP_EXTRA_ARGS
        EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
        EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
    environment:
      RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
      ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
+      RPC_HTTP_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
      DEV_USE_DOCKER_WAIT_SCRIPT: 1
      CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
      CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
@ -98,13 +96,16 @@ services:
      DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE
      TOKEN_NAME: ${TOKEN_NAME:-Giftable Token}
      TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
+      TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token}
+      #TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token}
+      #TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET}
+      #TOKEN_TYPE: ${TOKEN_TYPE:-erc20_demurrage_token }
      TOKEN_DECIMALS: $TOKEN_DECIMALS
      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD
      TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
      TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
      TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL
      TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
-      TOKEN_TYPE: $TOKEN_TYPE
      SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
    restart: on-failure 
    command: ["./run_job.sh"]
@ -157,14 +158,14 @@ services:
      USSD_PROVIDER: http://cic-user-ussd-server:9000
      CELERY_QUEUE: cic-import-ussd
      EXCLUSIONS: ussd
-    command: bash import_ussd.sh
+      CONTRACT_MIGRATION_URL: contract-migration:8000
+    command: bash ./scripts/run_job.sh 
    volumes:
      - contract-config:/tmp/cic/config/:ro
+        #- ./apps/data-seeding:/root

  cic-cache-tracker:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
-    profiles:
-      - cache 
    build:
      context: apps/cic-cache
      dockerfile: docker/Dockerfile
@ -203,8 +204,6 @@ services:

  cic-cache-tasker:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
-    profiles:
-      - cache 
    build:
      context: apps/cic-cache
      dockerfile: docker/Dockerfile
@ -242,8 +241,6 @@ services:

  cic-cache-server:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
-    profiles:
-      - cache 
    build:
      context: apps/cic-cache
      dockerfile: docker/Dockerfile
@ -544,8 +541,6 @@ services:

  cic-meta-server:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest}
-    profiles:
-      - custodial-meta
    hostname: meta
    build:
      context: apps/cic-meta
@ -578,8 +573,6 @@ services:

  cic-user-ussd-server:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
-    profiles:
-      - custodial-ussd
    build:
      context: apps/cic-ussd
      dockerfile: docker/Dockerfile
@ -609,8 +602,6 @@ services:

  cic-user-server:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
-    profiles:
-      - custodial-ussd
    build:
      context: apps/cic-ussd
      dockerfile: docker/Dockerfile
@ -633,8 +624,6 @@ services:

  cic-user-tasker:
    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
-    profiles:
-      - custodial-ussd
    build:
      context: apps/cic-ussd/
      dockerfile: docker/Dockerfile
--- a/scripts/dump.sh
+++ b/scripts/dump.sh
@ -1 +1 @@
-docker run -t -v  --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset
+docker run -v --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset