kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore: adding scripts to wait for contract migration output

Browse Source
This commit is contained in:
Blair Vanderlugt 2021-09-22 11:42:32 +03:00
parent b6b986e905
commit 503934037f
11 changed files with 238 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/contract-migration/.bash_history Normal file
View File

@ -0,0 +1,6 @@
ls /tmp/cic/config/
cat /tmp/cic/config/env_reset
exit
cat /tmp/cic/config/
cat /tmp/cic/config/env_reset
exit

8
apps/contract-migration/docker/init_readyz.sh
View File

@ -9,8 +9,12 @@ source $DEV_DATA_DIR/env_reset
jq --arg CIC_REGISTRY_ADDRESS "$CIC_REGISTRY_ADDRESS" \
--arg CIC_TRUST_ADDRESS "$CIC_TRUST_ADDRESS" \
--arg RUN_MASK "$RUN_MASK" \
-n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS, "CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS, "RUN_MASK": $RUN_MASK}' > $DEV_DATA_DIR/health/readyz
--arg CIC_DEFAULT_TOKEN_SYMBOL "$CIC_DEFAULT_TOKEN_SYMBOL"\
--arg TOKEN_NAME "$TOKEN_NAME"\
-n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS,
"CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS,
"CIC_DEFAULT_TOKEN_SYMBOL": $CIC_DEFAULT_TOKEN_SYMBOL,
"TOKEN_NAME": $TOKEN_NAME}' > $DEV_DATA_DIR/health/readyz
cd $DEV_DATA_DIR/health

7
apps/contract-migration/reset.sh
View File

@ -112,10 +112,13 @@ export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
echo "Writing env_reset file ..."
echo "export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
# note these are also written in init_readyz.sh
cat <<EOF > $DEV_DATA_DIR/env_reset
export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS
export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
export TOKEN_NAME=$TOKEN_NAME
" >> "${DEV_DATA_DIR}"/env_reset
EOF
set +a
set +e

5
apps/contract-migration/run_job.sh
View File

@ -22,6 +22,11 @@ fi
if [[ $((RUN_MASK & 2)) -eq 2 ]]
then
>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 2 - custodial service initialization"
if [ -f $DEV_DATA_DIR/env_reset ]; then
echo "found previous env_reset file"
cat $DEV_DATA_DIR/env_reset
source $DEV_DATA_DIR/env_reset;
fi
./seed_cic_eth.sh
if [ $? -ne "0" ]; then
>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 2 - custodial service initialization"

0
apps/data-seeding/import_ussd.sh Normal file → Executable file
View File

10
apps/data-seeding/scripts/get_readyz.sh Executable file
View File

@ -0,0 +1,10 @@
#! /bin/bash
set -e
set -u
echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
echo "exporting $s"
export $s
done

15
apps/data-seeding/scripts/run_job.sh Executable file
View File

@ -0,0 +1,15 @@
#! /bin/bash
set -e
set -u
env
WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
if [[ "$CONTRACT_MIGRATION_URL" ]]; then
echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
./scripts/wait-for-it.sh $CONTRACT_MIGRATION_URL -t $WAIT_FOR_TIMEOUT
source ./scripts/get_readyz.sh # set env vars form endpoint
./import_ussd.sh
else
./import_ussd.sh
fi

0
apps/data-seeding/scripts/run_ussd_user_imports.sh Normal file → Executable file
View File

182
apps/data-seeding/scripts/wait-for-it.sh Executable file
View File

@ -0,0 +1,182 @@
#!/usr/bin/env bash
# Use this script to test if a given TCP host/port are available
WAITFORIT_cmdname=${0##*/}
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
usage()
{
cat << USAGE >&2
Usage:
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST Host or IP under test
-p PORT | --port=PORT TCP port under test
Alternatively, you specify the host and port as host:port
-s | --strict Only execute subcommand if the test succeeds
-q | --quiet Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
Timeout in seconds, zero for no timeout
-- COMMAND ARGS Execute command with args after the test finishes
USAGE
exit 1
}
wait_for()
{
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
else
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
fi
WAITFORIT_start_ts=$(date +%s)
while :
do
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
WAITFORIT_result=$?
else
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
WAITFORIT_result=$?
fi
if [[ $WAITFORIT_result -eq 0 ]]; then
WAITFORIT_end_ts=$(date +%s)
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
break
fi
sleep 1
done
return $WAITFORIT_result
}
wait_for_wrapper()
{
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
else
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
fi
WAITFORIT_PID=$!
trap "kill -INT -$WAITFORIT_PID" INT
wait $WAITFORIT_PID
WAITFORIT_RESULT=$?
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
fi
return $WAITFORIT_RESULT
}
# process arguments
while [[ $# -gt 0 ]]
do
case "$1" in
*:* )
WAITFORIT_hostport=(${1//:/ })
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
shift 1
;;
--child)
WAITFORIT_CHILD=1
shift 1
;;
-q | --quiet)
WAITFORIT_QUIET=1
shift 1
;;
-s | --strict)
WAITFORIT_STRICT=1
shift 1
;;
-h)
WAITFORIT_HOST="$2"
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
shift 2
;;
--host=*)
WAITFORIT_HOST="${1#*=}"
shift 1
;;
-p)
WAITFORIT_PORT="$2"
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
shift 2
;;
--port=*)
WAITFORIT_PORT="${1#*=}"
shift 1
;;
-t)
WAITFORIT_TIMEOUT="$2"
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
shift 2
;;
--timeout=*)
WAITFORIT_TIMEOUT="${1#*=}"
shift 1
;;
--)
shift
WAITFORIT_CLI=("$@")
break
;;
--help)
usage
;;
*)
echoerr "Unknown argument: $1"
usage
;;
esac
done
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
echoerr "Error: you need to provide a host and port to test."
usage
fi
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
# Check to see if timeout is from busybox?
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
WAITFORIT_BUSYTIMEFLAG=""
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
WAITFORIT_ISBUSY=1
# Check if busybox timeout uses -t flag
# (recent Alpine versions don't support -t anymore)
if timeout &>/dev/stdout | grep -q -e '-t '; then
WAITFORIT_BUSYTIMEFLAG="-t"
fi
else
WAITFORIT_ISBUSY=0
fi
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
wait_for
WAITFORIT_RESULT=$?
exit $WAITFORIT_RESULT
else
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
wait_for_wrapper
WAITFORIT_RESULT=$?
else
wait_for
WAITFORIT_RESULT=$?
fi
fi
if [[ $WAITFORIT_CLI != "" ]]; then
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
exit $WAITFORIT_RESULT
fi
exec "${WAITFORIT_CLI[@]}"
else
exit $WAITFORIT_RESULT
fi

27
docker-compose.yml
View File

@ -60,8 +60,6 @@ services:
contract-migration:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest}
profiles:
- migrations
ports:
- 8012:8000
build:
@ -72,10 +70,10 @@ services:
pip_extra_args: $PIP_EXTRA_ARGS
EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
# image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
environment:
RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
RPC_HTTP_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
DEV_USE_DOCKER_WAIT_SCRIPT: 1
CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
@ -98,13 +96,16 @@ services:
DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE
TOKEN_NAME: ${TOKEN_NAME:-Giftable Token}
TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token}
#TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token}
#TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET}
#TOKEN_TYPE: ${TOKEN_TYPE:-erc20_demurrage_token }
TOKEN_DECIMALS: $TOKEN_DECIMALS
TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD
TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL
TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
TOKEN_TYPE: $TOKEN_TYPE
SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
restart: on-failure
command: ["./run_job.sh"]
@ -157,14 +158,14 @@ services:
USSD_PROVIDER: http://cic-user-ussd-server:9000
CELERY_QUEUE: cic-import-ussd
EXCLUSIONS: ussd
command: bash import_ussd.sh
CONTRACT_MIGRATION_URL: contract-migration:8000
command: bash ./scripts/run_job.sh
volumes:
- contract-config:/tmp/cic/config/:ro
#- ./apps/data-seeding:/root
cic-cache-tracker:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
profiles:
- cache
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
@ -203,8 +204,6 @@ services:
cic-cache-tasker:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
profiles:
- cache
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
@ -242,8 +241,6 @@ services:
cic-cache-server:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
profiles:
- cache
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
@ -544,8 +541,6 @@ services:
cic-meta-server:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest}
profiles:
- custodial-meta
hostname: meta
build:
context: apps/cic-meta
@ -578,8 +573,6 @@ services:
cic-user-ussd-server:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
profiles:
- custodial-ussd
build:
context: apps/cic-ussd
dockerfile: docker/Dockerfile
@ -609,8 +602,6 @@ services:
cic-user-server:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
profiles:
- custodial-ussd
build:
context: apps/cic-ussd
dockerfile: docker/Dockerfile
@ -633,8 +624,6 @@ services:
cic-user-tasker:
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest}
profiles:
- custodial-ussd
build:
context: apps/cic-ussd/
dockerfile: docker/Dockerfile

2
scripts/dump.sh
View File

@ -1 +1 @@
docker run -t -v --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset
docker run -v --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset