kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add better output and minor fixes.

Browse Source
This commit is contained in:
PhilipWafula 2021-10-20 15:52:59 +03:00
parent 92ef656cab
commit f87554d0d0
Signed by untrusted user: mango-habanero
GPG Key ID: B00CE9034DA19FB7
1 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
apps/data-seeding/import_ussd.sh Normal file → Executable file
View File

@ -2,11 +2,11 @@
if [[ -d "$OUT_DIR" ]] if [[ -d "$OUT_DIR" ]]
then then
echo "found existing IMPORT DIR cleaning up..." echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m"
rm -rf "$OUT_DIR" rm -rf "$OUT_DIR"
mkdir -p "$OUT_DIR" mkdir -p "$OUT_DIR"
else else
echo "IMPORT DIR does not exist creating it." echo -e "\033[;96mIMPORT DIR does not exist creating it.\033[;96m"
mkdir -p "$OUT_DIR" mkdir -p "$OUT_DIR"
fi fi
@ -14,81 +14,81 @@ fi
timeout 5 celery inspect ping -b "$CELERY_BROKER_URL" timeout 5 celery inspect ping -b "$CELERY_BROKER_URL"
if [[ $? -eq 124 ]] if [[ $? -eq 124 ]]
then then
>&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?" >&2 echo -e "\033[;96mCelery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?\033[;96m"
exit 1 exit 1
fi fi
echo "Creating seed data..." echo -e "\033[;96mCreating seed data...\033[;96m"
python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS" python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
wait $! wait $!
echo "Check for running celery workers ..." echo -e "\033[;96mCheck for running celery workers ...\033[;96m"
if [ -f ./cic-ussd-import.pid ]; if [ -f ./cic-ussd-import.pid ];
then then
echo "Found a running worker. Killing ..." echo -e "\033[;96mFound a running worker. Killing ...\033[;96m"
kill -9 $(<cic-ussd-import.pid) kill -9 $(<cic-ussd-import.pid)
fi fi
echo "Purge tasks from celery worker" echo -e "\033[;96mPurge tasks from celery worker\033[;96m"
celery -A cic_ussd.import_task purge -Q "$CELERY_QUEUE" --broker redis://"$REDIS_HOST":"$REDIS_PORT" -f celery -A cic_ussd.import_task purge -Q "$CELERY_QUEUE" --broker redis://"$REDIS_HOST":"$REDIS_PORT" -f
echo "Start celery work and import balance job" echo -e "\033[;96mStart celery work and import balance job\033[;96m"
if [ "$INCLUDE_BALANCES" != "y" ] if [ "$INCLUDE_BALANCES" != "y" ]
then then
echo "Running worker without opening balance transactions" echo -e "\033[;96mRunning worker without opening balance transactions\033[;96m"
TARGET_TX_COUNT=$NUMBER_OF_USERS TARGET_TX_COUNT=$NUMBER_OF_USERS
nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null & nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
else else
echo "Running worker with opening balance transactions" echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m"
TARGET_TX_COUNT=$((NUMBER_OF_USERS*2)) TARGET_TX_COUNT=$((NUMBER_OF_USERS*2))
nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" & nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" &
fi fi
echo "Target count set to ${TARGET_TX_COUNT}" echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}"
until [ -f ./cic-import-ussd.pid ] until [ -f ./cic-import-ussd.pid ]
do do
echo "Polling for celery worker pid file..." echo -e "\033[;96mPolling for celery worker pid file...\033[;96m"
sleep 1 sleep 1
done done
IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid) IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid)
echo "Start import users job" echo -e "\033[;96mStart import users job\033[;96m"
if [ "$USSD_SSL" == "y" ] if [ "$USSD_SSL" == "y" ]
then then
echo "Targeting secure ussd-user server" echo -e "\033[;96mTargeting secure ussd-user server\033[;96m"
python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR" python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
else else
python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR" python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
fi fi
echo "Waiting for import balance job to complete ..." echo -e "\033[;96mWaiting for import balance job to complete ...\033[;96m"
tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null
set -e set -e
echo "Importing pins" echo -e "\033[;96mImporting pins\033[;96m"
python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR" python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR"
set +e set +e
wait $! wait $!
set -e set -e
echo "Importing ussd data" echo -e "\033[;96mImporting ussd data\033[;96m"
python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR" python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR"
set +e set +e
wait $! wait $!
echo "Importing person metadata" echo -e "\033[;96mImporting person metadata\033[;96m"
node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS" node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS"
echo "Import preferences metadata" echo -e "\033[;96mImport preferences metadata\033[;96m"
node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS" node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS"
CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME
NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL') NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
while [[ "$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ]] while (("$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ))
do do
NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL') NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
sleep 5 sleep 5
echo "Notification count is: ${NOTIFICATION_COUNT} of ${TARGET_TX_COUNT}. Checking after 5 ..." echo -e "\033[;96mNotification count is: ${NOTIFICATION_COUNT} of ${TARGET_TX_COUNT}. Checking after 5 ...\033[;96m"
done done
echo "Running verify script." echo -e "\033[;96mRunning verify script.\033[;96m"
python verify.py -c "$CONFIG" -v -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --exclude "$EXCLUSIONS" --meta-provider "$META_URL" --token-symbol "$TOKEN_SYMBOL" --ussd-provider "$USSD_PROVIDER" "$OUT_DIR" python verify.py -v -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --exclude "$EXCLUSIONS" --meta-provider "$META_URL" --token-symbol "$TOKEN_SYMBOL" --ussd-provider "$USSD_PROVIDER" "$OUT_DIR"