WIP Replace env vars in data-seeding with well-known

2021-10-27 16:56:03 +02:00 · 2021-10-27 16:56:03 +02:00 · 1f37632f0f
parent 03d7518f8c
commit 1f37632f0f
6 changed files with 61 additions and 32 deletions
--- a/apps/data-seeding/cic_ussd/import_balance.py
+++ b/apps/data-seeding/cic_ussd/import_balance.py
@ -65,7 +65,8 @@ args_override = {
    'REDIS_DB': getattr(args, 'redis_db'),
    'META_HOST': getattr(args, 'meta_host'),
    'META_PORT': getattr(args, 'meta_port'),
-    'WALLET_KEY_FILE': getattr(args, 'y')
+    'WALLET_KEY_FILE': getattr(args, 'y'),
+    'TOKEN_SYMBOL': getattr(args, 'token_symbol'),
 }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@ -110,7 +111,7 @@ def main():
                                                    config.get('CIC_REGISTRY_ADDRESS'),
                                                    signer_address,
                                                    signer)
-    ImportTask.balance_processor.init(args.token_symbol)
+    ImportTask.balance_processor.init(config.get('TOKEN_SYMBOL'))
    balances = {}
    accuracy = 10 ** 6
    count = 0
--- a/apps/data-seeding/docker/Dockerfile
+++ b/apps/data-seeding/docker/Dockerfile
@ -1,29 +1,45 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2

 WORKDIR /root

 RUN mkdir -vp /usr/local/etc/cic

-COPY package.json \
-     package-lock.json \
-     ./
+ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org}
+RUN npm config set snyk=false
+#RUN npm config set registry={NPM_REPOSITORY}
+RUN npm config set registry=${NPM_REPOSITORY}
+
+# copy the dependencies
+COPY package.json package-lock.json ./
+RUN --mount=type=cache,mode=0755,target=/root/.npm \
+    npm set cache /root/.npm && \
+    npm cache verify && \
+    npm ci --verbose


-RUN npm ci --production 
+#RUN npm ci --production --verbose
 #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install

 COPY common/ cic_ussd/common/
 COPY requirements.txt .
 COPY config/ config

+
 ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
 ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL=https://pypi.org/simple

-RUN  pip install --index-url $PIP_INDEX_URL  \
-	  --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-    -r requirements.txt
+RUN pip install --index-url $PIP_INDEX_URL \
+    --pre \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+    -r requirements.txt 
+
+
+#RUN  pip install --index-url $PIP_INDEX_URL  \
+#	  --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+#    -r requirements.txt

 COPY . . 

--- a/apps/data-seeding/import_ussd.sh
+++ b/apps/data-seeding/import_ussd.sh
@ -1,5 +1,11 @@
 #!/bin/bash

+. /tmp/cic/config/env_reset
+
+OUT_DIR=out
+CONFIG_DIR=config
+confini-dump --schema-dir $CONFIG_DIR
+
 if [[ -d "$OUT_DIR" ]]
 then
  echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m"
@ -19,14 +25,14 @@ then
 fi

 echo -e "\033[;96mCreating seed data...\033[;96m"
-python create_import_users.py -vv  -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
+python create_import_users.py -vv  -c "$CONFIG_DIR" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
 wait $!

 echo -e "\033[;96mCheck for running celery workers ...\033[;96m"
 if [ -f ./cic-ussd-import.pid ];
 then
  echo -e "\033[;96mFound a running worker. Killing ...\033[;96m"
-  kill -9 $(<cic-ussd-import.pid)
+  kill -TERM $(<cic-ussd-import.pid)
 fi

 echo -e "\033[;96mPurge tasks from celery worker\033[;96m"
@ -37,11 +43,11 @@ if [ "$INCLUDE_BALANCES" != "y" ]
 then
  echo -e "\033[;96mRunning worker without opening balance transactions\033[;96m"
  TARGET_TX_COUNT=$NUMBER_OF_USERS
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
+  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
 else
  echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m"
  TARGET_TX_COUNT=$((NUMBER_OF_USERS*2))
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" &
+  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" &
 fi

 echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}"
@ -53,12 +59,12 @@ done
 IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid)

 echo -e "\033[;96mStart import users job\033[;96m"
-if [ "$USSD_SSL" == "y" ]
+if [ ! -z "$USSD_SSL" ]
 then
  echo -e "\033[;96mTargeting secure ussd-user server\033[;96m"
-  python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
 else
-  python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
 fi

 echo -e "\033[;96mWaiting for import balance job to complete ...\033[;96m"
@ -66,13 +72,13 @@ tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null
 set -e

 echo -e "\033[;96mImporting pins\033[;96m"
-python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_pins.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
 set +e
 wait $!
 set -e

 echo -e "\033[;96mImporting ussd data\033[;96m"
-python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_ussd_data.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
 set +e
 wait $!

@ -82,8 +88,9 @@ node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS"
 echo -e "\033[;96mImport preferences metadata\033[;96m"
 node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS"

-CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME
+CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME_NOTIFY
 NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
+# TODO: add max wait
 while (("$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ))
 do
  NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
--- a/apps/data-seeding/requirements.txt
+++ b/apps/data-seeding/requirements.txt
@ -1,6 +1,6 @@
 sarafu-faucet~=0.0.7a2
 cic-eth[tools]~=0.12.4a13
-cic-types~=0.2.0a6
+cic-types~=0.2.1a2
 funga>=0.5.1a1,<=0.5.15
 faker==4.17.1
 chainsyncer~=0.0.7a3
--- a/apps/data-seeding/verify.py
+++ b/apps/data-seeding/verify.py
@ -60,7 +60,7 @@ eth_tests = [
        ]

 phone_tests = [
-        'ussd',
+#        'ussd',
        'ussd_pins'
        ]

--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -493,6 +493,9 @@ services:
      dockerfile: docker/Dockerfile
      args:
        DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
+        PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
+        EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
+        EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
    environment:
      DATABASE_HOST: ${DATABASE_HOST:-postgres}
      DATABASE_PORT: ${DATABASE_PORT:-5432}
@ -515,8 +518,8 @@ services:
      - postgres
      - redis
        #- cic-meta-server
-      - cic-eth-tasker
-      - cic-cache-tasker
+        #- cic-eth-tasker
+        #- cic-cache-tasker
    volumes:
      - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/
    command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv"
@ -627,6 +630,7 @@ services:
        dockerfile: docker/Dockerfile
        args:
          DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
+          NPM_REPOSITORY: ${DEV_NPM_REPOSITORY:-https://registry.npmjs.org}
          PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
          EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
          EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
@ -644,27 +648,28 @@ services:
        CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
        CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
        RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
-        OUT_DIR: out
-        NUMBER_OF_USERS: 10
-        CONFIG: config
+          #OUT_DIR: out
+        NUMBER_OF_USERS: ${NUMBER_OF_USERS:-10}
+          #CONFIG_DIR: config
        CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg}
-        TOKEN_SYMBOL: GFT
+        TOKEN_SYMBOL: $TOKEN_SYMBOL
          #KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
        WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c}
        USSD_HOST: cic-user-ussd-server
        USSD_PORT: 9000
        INCLUDE_BALANCES: y
-        USSD_SSL: n
-        NOTIFY_DATABASE_NAME: cic_notify
+        GIFT_THRESHOLD: ${GIFT_THRESHOLD:-0}
+        USSD_SSL: $USSD_SSL
+        DATABASE_NAME_NOTIFY: cic_notify
        REDIS_HOST: redis
        REDIS_PORT: 6379
        REDIS_DB: 0
        META_HOST: meta
        META_PORT: 8000
        META_URL: http://meta:8000
+          # TODO: this should be generated from host/port/ssl
        USSD_PROVIDER: http://cic-user-ussd-server:9000
        CELERY_QUEUE: cic-import-ussd
-        EXCLUSIONS: ussd
      command:
        - /bin/bash
        - -c