Merge branch 'bvander/data-seeding-profiles' into 'master'

e2e ussd import user scripts

See merge request grassrootseconomics/cic-internal-integration!230

apps/contract-migration/.dockerignore

@@ -1,4 +1,6 @@
 .git
 .cache
 .dot
-**/doc
+**/doc
+**/.venv
+**/venv

apps/contract-migration/reset.sh

@@ -158,6 +158,7 @@ export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
 EOF
 
 cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
+cat ./envlist
 # popd
 
 set +a

apps/data-seeding/docker/.env

@@ -0,0 +1,10 @@
+ETH_PROVIDER=http://eth:8545
+CELERY_BROKER_URL=redis://redis:6379
+CELERY_RESULT_URL=redis://redis:6379
+CIC_REGISTRY_ADDRESS=0xea6225212005e86a4490018ded4bf37f3e772161
+TOKEN_SYMBOL=GFT
+REDIS_HOST=redis
+REDIS_PORT=6379
+USER_USSD_HOST=cic-user-ussd-server
+USER_USSD_PORT=9000
+KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c

apps/data-seeding/scripts/run_ussd_user_imports.sh

@@ -0,0 +1,54 @@
+#! /bin/sh
+
+set -u
+set -e
+
+while getopts ":n:o:g:" opt; do
+  case $opt in
+    n) NUMBER_OF_USERS="$OPTARG"
+    ;;
+    o) OUT_DIR="$OPTARG"
+    ;;
+    \?) echo "Invalid option -$OPTARG" >&2
+    ;;
+  esac
+done
+
+# using timeout because the timeout flag for celery inspect does not work
+timeout 5 celery inspect ping -b $CELERY_BROKER_URL
+if [[ $? -eq 124 ]] 
+then
+  >&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?"
+  exit 1
+fi
+
+if [[ -d $OUT_DIR ]]
+then
+  echo "found existing OUT_DIR cleaning up..."
+  rm -rf $OUT_DIR
+  mkdir -p $OUT_DIR
+else
+  echo "OUT_DIR does not exist creating it."
+  mkdir -p $OUT_DIR
+fi
+
+
+echo "creating accounts"
+
+python create_import_users.py --dir $OUT_DIR $NUMBER_OF_USERS
+
+echo "purging existing ussd tasks..."
+
+celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f
+
+
+echo "running import_balance in the background..."
+
+python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
+  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 2>&1 & 
+
+echo "import_balance pid: $!" 
+
+echo "importing accounts"
+
+python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out

docker-compose.yml

@@ -71,6 +71,8 @@ services:
       - bee-data:/tmp/cic/bee
 
   contract-migration:
+    profiles:
+      - migrations 
     build:
       context: apps/contract-migration
       dockerfile: docker/Dockerfile
@@ -79,6 +81,7 @@ services:
         pip_extra_args: $PIP_EXTRA_ARGS
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
       # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
       ETH_PROVIDER: http://eth:8545
       # And these two are for wait-for-it (could parse this)
@@ -123,6 +126,8 @@ services:
       - contract-config:/tmp/cic/config
 
   cic-cache-tracker:
+    profiles:
+      - cache 
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -159,6 +164,8 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-tasker:
+    profiles:
+      - cache 
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -196,6 +203,8 @@ services:
       - contract-config:/tmp/cic/config/:ro
 
   cic-cache-server:
+    profiles:
+      - cache 
     build:
       context: apps/cic-cache
       dockerfile: docker/Dockerfile
@@ -227,7 +236,6 @@ services:
 
 
   cic-eth-tasker:
-    # image: grassrootseconomics:cic-eth-service
     build: 
         context: apps/cic-eth
         dockerfile: docker/Dockerfile
@@ -338,7 +346,6 @@ services:
       TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
       DATABASE_DEBUG: ${DATABASE_DEBUG:-false}
         #DATABASE_DEBUG: 1
-
     depends_on:
       - eth
       - postgres
@@ -429,6 +436,8 @@ services:
 
 
   cic-meta-server:
+    profiles:
+      - custodial-meta
     hostname: meta
     build:
       context: apps/cic-meta
@@ -462,6 +471,8 @@ services:
     # command: "/root/start_server.sh -vv"
 
   cic-user-ussd-server:
+    profiles:
+      - custodial-ussd
     build:
       context: apps/cic-ussd
       dockerfile: docker/Dockerfile
@@ -491,6 +502,8 @@ services:
     command: "/root/start_cic_user_ussd_server.sh -vv"
 
   cic-user-server:
+    profiles:
+      - custodial-ussd
     build:
       context: apps/cic-ussd
       dockerfile: docker/Dockerfile
@@ -513,6 +526,8 @@ services:
     command: "/root/start_cic_user_server.sh -vv"
 
   cic-user-tasker:
+    profiles:
+      - custodial-ussd
     build:
       context: apps/cic-ussd/
       dockerfile: docker/Dockerfile