From 1f37632f0f1819adc67d1837ca797270039160b9 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 27 Oct 2021 16:56:03 +0200 Subject: [PATCH] WIP Replace env vars in data-seeding with well-known --- apps/data-seeding/cic_ussd/import_balance.py | 5 +-- apps/data-seeding/docker/Dockerfile | 34 ++++++++++++++------ apps/data-seeding/import_ussd.sh | 27 ++++++++++------ apps/data-seeding/requirements.txt | 2 +- apps/data-seeding/verify.py | 2 +- docker-compose.yml | 23 +++++++------ 6 files changed, 61 insertions(+), 32 deletions(-) diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 6d8b9a2d..bff5ed89 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -65,7 +65,8 @@ args_override = { 'REDIS_DB': getattr(args, 'redis_db'), 'META_HOST': getattr(args, 'meta_host'), 'META_PORT': getattr(args, 'meta_port'), - 'WALLET_KEY_FILE': getattr(args, 'y') + 'WALLET_KEY_FILE': getattr(args, 'y'), + 'TOKEN_SYMBOL': getattr(args, 'token_symbol'), } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') @@ -110,7 +111,7 @@ def main(): config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer) - ImportTask.balance_processor.init(args.token_symbol) + ImportTask.balance_processor.init(config.get('TOKEN_SYMBOL')) balances = {} accuracy = 10 ** 6 count = 0 diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index ec6064d1..f2d09df6 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -1,29 +1,45 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45 +ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" + +FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2 WORKDIR /root RUN mkdir -vp /usr/local/etc/cic -COPY package.json \ - package-lock.json \ - ./ +ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org} +RUN npm config set snyk=false +#RUN npm config set registry={NPM_REPOSITORY} +RUN npm config set registry=${NPM_REPOSITORY} + +# copy the dependencies +COPY package.json package-lock.json ./ +RUN --mount=type=cache,mode=0755,target=/root/.npm \ + npm set cache /root/.npm && \ + npm cache verify && \ + npm ci --verbose -RUN npm ci --production +#RUN npm ci --production --verbose #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install COPY common/ cic_ussd/common/ COPY requirements.txt . COPY config/ config + ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433 ARG EXTRA_PIP_ARGS="" ARG PIP_INDEX_URL=https://pypi.org/simple -RUN pip install --index-url $PIP_INDEX_URL \ - --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ - -r requirements.txt +RUN pip install --index-url $PIP_INDEX_URL \ + --pre \ + --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ + -r requirements.txt + + +#RUN pip install --index-url $PIP_INDEX_URL \ +# --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ +# -r requirements.txt COPY . . diff --git a/apps/data-seeding/import_ussd.sh b/apps/data-seeding/import_ussd.sh index 4b9e9550..8dd4c3e9 100755 --- a/apps/data-seeding/import_ussd.sh +++ b/apps/data-seeding/import_ussd.sh @@ -1,5 +1,11 @@ #!/bin/bash +. /tmp/cic/config/env_reset + +OUT_DIR=out +CONFIG_DIR=config +confini-dump --schema-dir $CONFIG_DIR + if [[ -d "$OUT_DIR" ]] then echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m" @@ -19,14 +25,14 @@ then fi echo -e "\033[;96mCreating seed data...\033[;96m" -python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS" +python create_import_users.py -vv -c "$CONFIG_DIR" --dir "$OUT_DIR" "$NUMBER_OF_USERS" wait $! echo -e "\033[;96mCheck for running celery workers ...\033[;96m" if [ -f ./cic-ussd-import.pid ]; then echo -e "\033[;96mFound a running worker. Killing ...\033[;96m" - kill -9 $( nohup.out 2> nohup.err < /dev/null & + nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null & else echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m" TARGET_TX_COUNT=$((NUMBER_OF_USERS*2)) - nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" & + nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" & fi echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}" @@ -53,12 +59,12 @@ done IMPORT_BALANCE_JOB=$(=0.5.1a1,<=0.5.15 faker==4.17.1 chainsyncer~=0.0.7a3 diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index fd59df45..b01a5e8a 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -60,7 +60,7 @@ eth_tests = [ ] phone_tests = [ - 'ussd', +# 'ussd', 'ussd_pins' ] diff --git a/docker-compose.yml b/docker-compose.yml index dd23fce4..9ee5eef8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -493,6 +493,9 @@ services: dockerfile: docker/Dockerfile args: DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PORT: ${DATABASE_PORT:-5432} @@ -515,8 +518,8 @@ services: - postgres - redis #- cic-meta-server - - cic-eth-tasker - - cic-cache-tasker + #- cic-eth-tasker + #- cic-cache-tasker volumes: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" @@ -627,6 +630,7 @@ services: dockerfile: docker/Dockerfile args: DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + NPM_REPOSITORY: ${DEV_NPM_REPOSITORY:-https://registry.npmjs.org} PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS @@ -644,27 +648,28 @@ services: CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} - OUT_DIR: out - NUMBER_OF_USERS: 10 - CONFIG: config + #OUT_DIR: out + NUMBER_OF_USERS: ${NUMBER_OF_USERS:-10} + #CONFIG_DIR: config CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} - TOKEN_SYMBOL: GFT + TOKEN_SYMBOL: $TOKEN_SYMBOL #KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c} USSD_HOST: cic-user-ussd-server USSD_PORT: 9000 INCLUDE_BALANCES: y - USSD_SSL: n - NOTIFY_DATABASE_NAME: cic_notify + GIFT_THRESHOLD: ${GIFT_THRESHOLD:-0} + USSD_SSL: $USSD_SSL + DATABASE_NAME_NOTIFY: cic_notify REDIS_HOST: redis REDIS_PORT: 6379 REDIS_DB: 0 META_HOST: meta META_PORT: 8000 META_URL: http://meta:8000 + # TODO: this should be generated from host/port/ssl USSD_PROVIDER: http://cic-user-ussd-server:9000 CELERY_QUEUE: cic-import-ussd - EXCLUSIONS: ussd command: - /bin/bash - -c