WIP Replace env vars in data-seeding with well-known

apps/data-seeding/cic_ussd/import_balance.py

@@ -65,7 +65,8 @@ args_override = {
     'REDIS_DB': getattr(args, 'redis_db'),
     'META_HOST': getattr(args, 'meta_host'),
     'META_PORT': getattr(args, 'meta_port'),
-    'WALLET_KEY_FILE': getattr(args, 'y')
+    'WALLET_KEY_FILE': getattr(args, 'y'),
+    'TOKEN_SYMBOL': getattr(args, 'token_symbol'),
 }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -110,7 +111,7 @@ def main():
                                                     config.get('CIC_REGISTRY_ADDRESS'),
                                                     signer_address,
                                                     signer)
-    ImportTask.balance_processor.init(args.token_symbol)
+    ImportTask.balance_processor.init(config.get('TOKEN_SYMBOL'))
     balances = {}
     accuracy = 10 ** 6
     count = 0

apps/data-seeding/docker/Dockerfile

@@ -1,30 +1,46 @@
-# syntax = docker/dockerfile:1.2
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 
 WORKDIR /root
 
 RUN mkdir -vp /usr/local/etc/cic
 
-COPY package.json \
+ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org}
-     package-lock.json \
+RUN npm config set snyk=false
-     ./
+#RUN npm config set registry={NPM_REPOSITORY}
+RUN npm config set registry=${NPM_REPOSITORY}
+
+# copy the dependencies
+COPY package.json package-lock.json ./
+RUN --mount=type=cache,mode=0755,target=/root/.npm \
+    npm set cache /root/.npm && \
+    npm cache verify && \
+    npm ci --verbose
 
 
-RUN npm ci --production 
+#RUN npm ci --production --verbose
 #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
 
 COPY common/ cic_ussd/common/
 COPY requirements.txt .
 COPY config/ config
 
+
 ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
 ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL=https://pypi.org/simple
 
-RUN  pip install --index-url $PIP_INDEX_URL  \
+RUN pip install --index-url $PIP_INDEX_URL \
-	  --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt 
 
+
+#RUN  pip install --index-url $PIP_INDEX_URL  \
+#	  --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+#    -r requirements.txt
+
 COPY . . 
 
 ENTRYPOINT [ ]

apps/data-seeding/import_ussd.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+. /tmp/cic/config/env_reset
+
+OUT_DIR=out
+CONFIG_DIR=config
+confini-dump --schema-dir $CONFIG_DIR
+
 if [[ -d "$OUT_DIR" ]]
 then
   echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m"
@@ -19,14 +25,14 @@ then
 fi
 
 echo -e "\033[;96mCreating seed data...\033[;96m"
-python create_import_users.py -vv  -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
+python create_import_users.py -vv  -c "$CONFIG_DIR" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
 wait $!
 
 echo -e "\033[;96mCheck for running celery workers ...\033[;96m"
 if [ -f ./cic-ussd-import.pid ];
 then
   echo -e "\033[;96mFound a running worker. Killing ...\033[;96m"
-  kill -9 $(<cic-ussd-import.pid)
+  kill -TERM $(<cic-ussd-import.pid)
 fi
 
 echo -e "\033[;96mPurge tasks from celery worker\033[;96m"
@@ -37,11 +43,11 @@ if [ "$INCLUDE_BALANCES" != "y" ]
 then
   echo -e "\033[;96mRunning worker without opening balance transactions\033[;96m"
   TARGET_TX_COUNT=$NUMBER_OF_USERS
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
+  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
 else
   echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m"
   TARGET_TX_COUNT=$((NUMBER_OF_USERS*2))
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" &
+  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" &
 fi
 
 echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}"
@@ -53,12 +59,12 @@ done
 IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid)
 
 echo -e "\033[;96mStart import users job\033[;96m"
-if [ "$USSD_SSL" == "y" ]
+if [ ! -z "$USSD_SSL" ]
 then
   echo -e "\033[;96mTargeting secure ussd-user server\033[;96m"
-  python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
 else
-  python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
 fi
 
 echo -e "\033[;96mWaiting for import balance job to complete ...\033[;96m"
@@ -66,13 +72,13 @@ tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null
 set -e
 
 echo -e "\033[;96mImporting pins\033[;96m"
-python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_pins.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
 set +e
 wait $!
 set -e
 
 echo -e "\033[;96mImporting ussd data\033[;96m"
-python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_ussd_data.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
 set +e
 wait $!
 
@@ -82,8 +88,9 @@ node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS"
 echo -e "\033[;96mImport preferences metadata\033[;96m"
 node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS"
 
-CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME
+CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME_NOTIFY
 NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
+# TODO: add max wait
 while (("$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ))
 do
   NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')

apps/data-seeding/requirements.txt

@@ -1,6 +1,6 @@
 sarafu-faucet~=0.0.7a2
 cic-eth[tools]~=0.12.4a13
-cic-types~=0.2.0a6
+cic-types~=0.2.1a2
 funga>=0.5.1a1,<=0.5.15
 faker==4.17.1
 chainsyncer~=0.0.7a3

apps/data-seeding/verify.py

@@ -60,7 +60,7 @@ eth_tests = [
         ]
 
 phone_tests = [
-        'ussd',
+#        'ussd',
         'ussd_pins'
         ]
 

docker-compose.yml

@@ -493,6 +493,9 @@ services:
       dockerfile: docker/Dockerfile
       args:
         DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
+        PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
+        EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
+        EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
     environment:
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
@@ -515,8 +518,8 @@ services:
       - postgres
       - redis
         #- cic-meta-server
-      - cic-eth-tasker
+        #- cic-eth-tasker
-      - cic-cache-tasker
+        #- cic-cache-tasker
     volumes:
       - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/
     command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv"
@@ -627,6 +630,7 @@ services:
         dockerfile: docker/Dockerfile
         args:
           DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
+          NPM_REPOSITORY: ${DEV_NPM_REPOSITORY:-https://registry.npmjs.org}
           PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
           EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
           EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
@@ -644,27 +648,28 @@ services:
         CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
         CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
-        OUT_DIR: out
+          #OUT_DIR: out
-        NUMBER_OF_USERS: 10
+        NUMBER_OF_USERS: ${NUMBER_OF_USERS:-10}
-        CONFIG: config
+          #CONFIG_DIR: config
         CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg}
-        TOKEN_SYMBOL: GFT
+        TOKEN_SYMBOL: $TOKEN_SYMBOL
           #KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
         WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c}
         USSD_HOST: cic-user-ussd-server
         USSD_PORT: 9000
         INCLUDE_BALANCES: y
-        USSD_SSL: n
+        GIFT_THRESHOLD: ${GIFT_THRESHOLD:-0}
-        NOTIFY_DATABASE_NAME: cic_notify
+        USSD_SSL: $USSD_SSL
+        DATABASE_NAME_NOTIFY: cic_notify
         REDIS_HOST: redis
         REDIS_PORT: 6379
         REDIS_DB: 0
         META_HOST: meta
         META_PORT: 8000
         META_URL: http://meta:8000
+          # TODO: this should be generated from host/port/ssl
         USSD_PROVIDER: http://cic-user-ussd-server:9000
         CELERY_QUEUE: cic-import-ussd
-        EXCLUSIONS: ussd
       command:
         - /bin/bash
         - -c