if user dir not ready just retry instead of bailing

apps/cic-notify/docker/Dockerfile

@@ -16,7 +16,7 @@ COPY . .
 RUN python setup.py install
 
 COPY docker/*.sh .
-RUN chmod +x *.sh 
+RUN chmod +x /root/*.sh
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
@@ -24,5 +24,4 @@ RUN chmod +x *.sh
 COPY .config/ /usr/local/etc/cic-notify/
 COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
 
-
 ENTRYPOINT []

apps/cic-notify/docker/Dockerfile_ci

@@ -11,12 +11,14 @@ RUN pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt
 
-COPY . .
+COPY . . 
 
 RUN python setup.py install
 
+# TODO please review..can this go into requirements?
+RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
+
 COPY docker/*.sh .
-RUN chmod +x *.sh 
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables

apps/cic-notify/docker/db.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-set -e
+
-python scripts/migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv
+migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv

apps/data-seeding/.dockerignore

@@ -2,7 +2,7 @@
 .cache
 .dot
 **/doc
-node_modules/
+**/node_modules
 **/venv
 **/.venv
 

apps/data-seeding/cic_ussd/import_balance.py

@@ -2,6 +2,7 @@
 import argparse
 import logging
 import sys
+import time
 import os
 
 # external imports
@@ -124,7 +125,16 @@ def main():
 
     # TODO get decimals from token
     balances = {}
-    f = open('{}/balances.csv'.format(user_dir, 'r'))
+    f = None
+    while f is None:
+        try:
+            f = open('{}/balances.csv'.format(user_dir, 'r'))
+        except FileNotFoundError:
+            print('{} not found...', user_dir)
+            time.sleep(30)
+
+
+
     remove_zeros = 10 ** 6
     i = 0
     while True:

apps/data-seeding/docker/.env

@@ -8,5 +8,3 @@ REDIS_PORT=6379
 USER_USSD_HOST=cic-user-ussd-server
 USER_USSD_PORT=9000
 KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
-OUT_DIR=out
-NUMBER_OF_USERS=100

apps/data-seeding/docker/Dockerfile

@@ -8,9 +8,11 @@ RUN mkdir -vp /usr/local/etc/cic
 COPY package.json \
      package-lock.json \
      .
-RUN npm ci --production 
+
+RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
 
 COPY requirements.txt . 
+
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
@@ -19,5 +21,4 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
 
 COPY . . 
 
-
 ENTRYPOINT [ ]

apps/data-seeding/scripts/run_ussd_user_imports.sh

@@ -41,27 +41,14 @@ echo "purging existing ussd tasks..."
 
 celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f
 
+
 echo "running import_balance in the background..."
 
 python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
-  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR > import_task_log.log & 
+  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 2>&1 & 
 
-import_pid=$!
+echo "import_balance pid: $!" 
-echo "import_balance pid: $import_pid" 
 
 echo "importing accounts"
 
 python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out
-
-echo "importing user meta data"
-node cic_meta/import_meta.js $OUT_DIR $NUMBER_OF_USERS 
-
-echo "import meta prefereneces"
-node cic_meta/import_meta_preferences.js $OUT_DIR $NUMBER_OF_USERS 
-
-echo "Running validation!"
-python verify.py -v -c config -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER \
-  --token-symbol $TOKEN_SYMBOL $OUT_DIR 
-
-kill $import_pid
-exit 0