init

Bvander/fix notify migrations

See merge request grassrootseconomics/cic-internal-integration!238

apps/cic-notify/docker/Dockerfile

@@ -16,7 +16,7 @@ COPY . .
 RUN python setup.py install
 
 COPY docker/*.sh .
-RUN chmod +x /root/*.sh
+RUN chmod +x *.sh 
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
@@ -24,4 +24,5 @@ RUN chmod +x /root/*.sh
 COPY .config/ /usr/local/etc/cic-notify/
 COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
 
+
 ENTRYPOINT []

apps/cic-notify/docker/Dockerfile_ci

@@ -11,14 +11,12 @@ RUN pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt
 
-COPY . . 
+COPY . .
 
 RUN python setup.py install
 
-# TODO please review..can this go into requirements?
-RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
-
 COPY docker/*.sh .
+RUN chmod +x *.sh 
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables

apps/cic-notify/docker/db.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-
-migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv
+set -e
+python scripts/migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv

apps/data-seeding/.dockerignore

@@ -2,7 +2,7 @@
 .cache
 .dot
 **/doc
-**/node_modules
+node_modules/
 **/venv
 **/.venv
 

apps/data-seeding/cic_ussd/import_balance.py

@@ -2,7 +2,6 @@
 import argparse
 import logging
 import sys
-import time
 import os
 
 # external imports
@@ -125,16 +124,7 @@ def main():
 
     # TODO get decimals from token
     balances = {}
-    f = None
-    while f is None:
-        try:
-            f = open('{}/balances.csv'.format(user_dir, 'r'))
-        except FileNotFoundError:
-            print('{} not found...', user_dir)
-            time.sleep(30)
-
-
-
+    f = open('{}/balances.csv'.format(user_dir, 'r'))
     remove_zeros = 10 ** 6
     i = 0
     while True:

apps/data-seeding/docker/.env

@@ -8,3 +8,5 @@ REDIS_PORT=6379
 USER_USSD_HOST=cic-user-ussd-server
 USER_USSD_PORT=9000
 KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+OUT_DIR=out
+NUMBER_OF_USERS=100

apps/data-seeding/docker/Dockerfile

@@ -8,11 +8,9 @@ RUN mkdir -vp /usr/local/etc/cic
 COPY package.json \
      package-lock.json \
      .
-
-RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
+RUN npm ci --production 
 
 COPY requirements.txt . 
-
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
@@ -21,4 +19,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \
 
 COPY . . 
 
+
 ENTRYPOINT [ ]

apps/data-seeding/scripts/run_ussd_user_imports.sh

@@ -41,14 +41,27 @@ echo "purging existing ussd tasks..."
 
 celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f
 
-
 echo "running import_balance in the background..."
 
 python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
-  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 2>&1 & 
+  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR > import_task_log.log & 
 
-echo "import_balance pid: $!" 
+import_pid=$!
+echo "import_balance pid: $import_pid" 
 
 echo "importing accounts"
 
 python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out
+
+echo "importing user meta data"
+node cic_meta/import_meta.js $OUT_DIR $NUMBER_OF_USERS 
+
+echo "import meta prefereneces"
+node cic_meta/import_meta_preferences.js $OUT_DIR $NUMBER_OF_USERS 
+
+echo "Running validation!"
+python verify.py -v -c config -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER \
+  --token-symbol $TOKEN_SYMBOL $OUT_DIR 
+
+kill $import_pid
+exit 0