cic-internal-integration/apps/data-seeding/import_ussd.sh

95 lines
3.3 KiB
Bash
Raw Permalink Normal View History

2021-09-21 08:04:44 +02:00
#!/bin/bash
2021-08-30 20:09:47 +02:00
2021-09-21 08:04:44 +02:00
if [[ -d "$OUT_DIR" ]]
then
echo "found existing IMPORT DIR cleaning up..."
rm -rf "$OUT_DIR"
mkdir -p "$OUT_DIR"
else
echo "IMPORT DIR does not exist creating it."
mkdir -p "$OUT_DIR"
fi
# using timeout because the timeout flag for celery inspect does not work
timeout 5 celery inspect ping -b "$CELERY_BROKER_URL"
if [[ $? -eq 124 ]]
then
>&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?"
exit 1
fi
2021-08-30 20:09:47 +02:00
2021-08-29 11:55:47 +02:00
echo "Creating seed data..."
2021-09-21 08:04:44 +02:00
python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
2021-08-29 11:55:47 +02:00
wait $!
2021-09-21 08:04:44 +02:00
echo "Check for running celery workers ..."
if [ -f ./cic-ussd-import.pid ];
then
echo "Found a running worker. Killing ..."
kill -9 $(<cic-ussd-import.pid)
fi
2021-08-29 11:55:47 +02:00
echo "Purge tasks from celery worker"
celery -A cic_ussd.import_task purge -Q "$CELERY_QUEUE" --broker redis://"$REDIS_HOST":"$REDIS_PORT" -f
2021-09-21 08:04:44 +02:00
2021-08-29 11:55:47 +02:00
echo "Start celery work and import balance job"
if [ "$INCLUDE_BALANCES" != "y" ]
then
echo "Running worker without opening balance transactions"
2021-09-21 08:04:44 +02:00
TARGET_TX_COUNT=$NUMBER_OF_USERS
nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
2021-08-29 11:55:47 +02:00
else
echo "Running worker with opening balance transactions"
2021-09-21 08:04:44 +02:00
TARGET_TX_COUNT=$((NUMBER_OF_USERS*2))
nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" &
2021-08-29 11:55:47 +02:00
fi
2021-08-30 20:09:47 +02:00
2021-09-21 08:04:44 +02:00
echo "Target count set to ${TARGET_TX_COUNT}"
2021-08-29 11:55:47 +02:00
until [ -f ./cic-import-ussd.pid ]
do
echo "Polling for celery worker pid file..."
2021-08-30 20:09:47 +02:00
sleep 1
2021-08-29 11:55:47 +02:00
done
IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid)
2021-09-21 08:04:44 +02:00
2021-08-29 11:55:47 +02:00
echo "Start import users job"
if [ "$USSD_SSL" == "y" ]
then
echo "Targeting secure ussd-user server"
2021-09-21 08:04:44 +02:00
python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
2021-08-29 11:55:47 +02:00
else
2021-09-21 08:04:44 +02:00
python cic_ussd/import_users.py -vv -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
2021-08-29 11:55:47 +02:00
fi
2021-09-21 08:04:44 +02:00
2021-08-29 11:55:47 +02:00
echo "Waiting for import balance job to complete ..."
tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null
set -e
2021-09-21 08:04:44 +02:00
2021-08-29 11:55:47 +02:00
echo "Importing pins"
2021-09-21 08:04:44 +02:00
python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR"
2021-08-29 11:55:47 +02:00
set +e
wait $!
set -e
2021-09-21 08:04:44 +02:00
2021-08-29 11:55:47 +02:00
echo "Importing ussd data"
2021-09-21 08:04:44 +02:00
python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR"
2021-08-29 11:55:47 +02:00
set +e
wait $!
2021-09-21 08:04:44 +02:00
2021-08-29 11:55:47 +02:00
echo "Importing person metadata"
2021-09-21 08:04:44 +02:00
node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS"
2021-08-29 11:55:47 +02:00
echo "Import preferences metadata"
2021-09-21 08:04:44 +02:00
node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS"
2021-08-29 11:55:47 +02:00
CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME
NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
while [[ "$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ]]
do
NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
sleep 5
2021-09-21 08:04:44 +02:00
echo "Notification count is: ${NOTIFICATION_COUNT} of ${TARGET_TX_COUNT}. Checking after 5 ..."
2021-08-29 11:55:47 +02:00
done
2021-09-21 08:04:44 +02:00
echo "Running verify script."
python verify.py -c "$CONFIG" -v -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --exclude "$EXCLUSIONS" --meta-provider "$META_URL" --token-symbol "$TOKEN_SYMBOL" --ussd-provider "$USSD_PROVIDER" "$OUT_DIR"