WIP Replace env vars in data-seeding with well-known

This commit is contained in:
nolash 2021-10-27 16:56:03 +02:00
parent 03d7518f8c
commit 1f37632f0f
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
6 changed files with 61 additions and 32 deletions

View File

@ -65,7 +65,8 @@ args_override = {
'REDIS_DB': getattr(args, 'redis_db'), 'REDIS_DB': getattr(args, 'redis_db'),
'META_HOST': getattr(args, 'meta_host'), 'META_HOST': getattr(args, 'meta_host'),
'META_PORT': getattr(args, 'meta_port'), 'META_PORT': getattr(args, 'meta_port'),
'WALLET_KEY_FILE': getattr(args, 'y') 'WALLET_KEY_FILE': getattr(args, 'y'),
'TOKEN_SYMBOL': getattr(args, 'token_symbol'),
} }
config.dict_override(args_override, 'cli flag') config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'DATABASE')
@ -110,7 +111,7 @@ def main():
config.get('CIC_REGISTRY_ADDRESS'), config.get('CIC_REGISTRY_ADDRESS'),
signer_address, signer_address,
signer) signer)
ImportTask.balance_processor.init(args.token_symbol) ImportTask.balance_processor.init(config.get('TOKEN_SYMBOL'))
balances = {} balances = {}
accuracy = 10 ** 6 accuracy = 10 ** 6
count = 0 count = 0

View File

@ -1,30 +1,46 @@
# syntax = docker/dockerfile:1.2 ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
WORKDIR /root WORKDIR /root
RUN mkdir -vp /usr/local/etc/cic RUN mkdir -vp /usr/local/etc/cic
COPY package.json \ ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org}
package-lock.json \ RUN npm config set snyk=false
./ #RUN npm config set registry={NPM_REPOSITORY}
RUN npm config set registry=${NPM_REPOSITORY}
# copy the dependencies
COPY package.json package-lock.json ./
RUN --mount=type=cache,mode=0755,target=/root/.npm \
npm set cache /root/.npm && \
npm cache verify && \
npm ci --verbose
RUN npm ci --production #RUN npm ci --production --verbose
#RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
COPY common/ cic_ussd/common/ COPY common/ cic_ussd/common/
COPY requirements.txt . COPY requirements.txt .
COPY config/ config COPY config/ config
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433 ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
ARG EXTRA_PIP_ARGS="" ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple ARG PIP_INDEX_URL=https://pypi.org/simple
RUN pip install --index-url $PIP_INDEX_URL \ RUN pip install --index-url $PIP_INDEX_URL \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ --pre \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt -r requirements.txt
#RUN pip install --index-url $PIP_INDEX_URL \
# --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
# -r requirements.txt
COPY . . COPY . .
ENTRYPOINT [ ] ENTRYPOINT [ ]

View File

@ -1,5 +1,11 @@
#!/bin/bash #!/bin/bash
. /tmp/cic/config/env_reset
OUT_DIR=out
CONFIG_DIR=config
confini-dump --schema-dir $CONFIG_DIR
if [[ -d "$OUT_DIR" ]] if [[ -d "$OUT_DIR" ]]
then then
echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m" echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m"
@ -19,14 +25,14 @@ then
fi fi
echo -e "\033[;96mCreating seed data...\033[;96m" echo -e "\033[;96mCreating seed data...\033[;96m"
python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS" python create_import_users.py -vv -c "$CONFIG_DIR" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
wait $! wait $!
echo -e "\033[;96mCheck for running celery workers ...\033[;96m" echo -e "\033[;96mCheck for running celery workers ...\033[;96m"
if [ -f ./cic-ussd-import.pid ]; if [ -f ./cic-ussd-import.pid ];
then then
echo -e "\033[;96mFound a running worker. Killing ...\033[;96m" echo -e "\033[;96mFound a running worker. Killing ...\033[;96m"
kill -9 $(<cic-ussd-import.pid) kill -TERM $(<cic-ussd-import.pid)
fi fi
echo -e "\033[;96mPurge tasks from celery worker\033[;96m" echo -e "\033[;96mPurge tasks from celery worker\033[;96m"
@ -37,11 +43,11 @@ if [ "$INCLUDE_BALANCES" != "y" ]
then then
echo -e "\033[;96mRunning worker without opening balance transactions\033[;96m" echo -e "\033[;96mRunning worker without opening balance transactions\033[;96m"
TARGET_TX_COUNT=$NUMBER_OF_USERS TARGET_TX_COUNT=$NUMBER_OF_USERS
nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null & nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
else else
echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m" echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m"
TARGET_TX_COUNT=$((NUMBER_OF_USERS*2)) TARGET_TX_COUNT=$((NUMBER_OF_USERS*2))
nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" & nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" &
fi fi
echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}" echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}"
@ -53,12 +59,12 @@ done
IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid) IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid)
echo -e "\033[;96mStart import users job\033[;96m" echo -e "\033[;96mStart import users job\033[;96m"
if [ "$USSD_SSL" == "y" ] if [ ! -z "$USSD_SSL" ]
then then
echo -e "\033[;96mTargeting secure ussd-user server\033[;96m" echo -e "\033[;96mTargeting secure ussd-user server\033[;96m"
python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR" python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
else else
python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR" python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
fi fi
echo -e "\033[;96mWaiting for import balance job to complete ...\033[;96m" echo -e "\033[;96mWaiting for import balance job to complete ...\033[;96m"
@ -66,13 +72,13 @@ tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null
set -e set -e
echo -e "\033[;96mImporting pins\033[;96m" echo -e "\033[;96mImporting pins\033[;96m"
python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR" python cic_ussd/import_pins.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
set +e set +e
wait $! wait $!
set -e set -e
echo -e "\033[;96mImporting ussd data\033[;96m" echo -e "\033[;96mImporting ussd data\033[;96m"
python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR" python cic_ussd/import_ussd_data.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
set +e set +e
wait $! wait $!
@ -82,8 +88,9 @@ node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS"
echo -e "\033[;96mImport preferences metadata\033[;96m" echo -e "\033[;96mImport preferences metadata\033[;96m"
node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS" node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS"
CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME_NOTIFY
NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL') NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
# TODO: add max wait
while (("$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" )) while (("$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ))
do do
NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL') NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')

View File

@ -1,6 +1,6 @@
sarafu-faucet~=0.0.7a2 sarafu-faucet~=0.0.7a2
cic-eth[tools]~=0.12.4a13 cic-eth[tools]~=0.12.4a13
cic-types~=0.2.0a6 cic-types~=0.2.1a2
funga>=0.5.1a1,<=0.5.15 funga>=0.5.1a1,<=0.5.15
faker==4.17.1 faker==4.17.1
chainsyncer~=0.0.7a3 chainsyncer~=0.0.7a3

View File

@ -60,7 +60,7 @@ eth_tests = [
] ]
phone_tests = [ phone_tests = [
'ussd', # 'ussd',
'ussd_pins' 'ussd_pins'
] ]

View File

@ -493,6 +493,9 @@ services:
dockerfile: docker/Dockerfile dockerfile: docker/Dockerfile
args: args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment: environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432} DATABASE_PORT: ${DATABASE_PORT:-5432}
@ -515,8 +518,8 @@ services:
- postgres - postgres
- redis - redis
#- cic-meta-server #- cic-meta-server
- cic-eth-tasker #- cic-eth-tasker
- cic-cache-tasker #- cic-cache-tasker
volumes: volumes:
- ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/
command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv"
@ -627,6 +630,7 @@ services:
dockerfile: docker/Dockerfile dockerfile: docker/Dockerfile
args: args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
NPM_REPOSITORY: ${DEV_NPM_REPOSITORY:-https://registry.npmjs.org}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
@ -644,27 +648,28 @@ services:
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
OUT_DIR: out #OUT_DIR: out
NUMBER_OF_USERS: 10 NUMBER_OF_USERS: ${NUMBER_OF_USERS:-10}
CONFIG: config #CONFIG_DIR: config
CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg}
TOKEN_SYMBOL: GFT TOKEN_SYMBOL: $TOKEN_SYMBOL
#KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c #KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c} WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c}
USSD_HOST: cic-user-ussd-server USSD_HOST: cic-user-ussd-server
USSD_PORT: 9000 USSD_PORT: 9000
INCLUDE_BALANCES: y INCLUDE_BALANCES: y
USSD_SSL: n GIFT_THRESHOLD: ${GIFT_THRESHOLD:-0}
NOTIFY_DATABASE_NAME: cic_notify USSD_SSL: $USSD_SSL
DATABASE_NAME_NOTIFY: cic_notify
REDIS_HOST: redis REDIS_HOST: redis
REDIS_PORT: 6379 REDIS_PORT: 6379
REDIS_DB: 0 REDIS_DB: 0
META_HOST: meta META_HOST: meta
META_PORT: 8000 META_PORT: 8000
META_URL: http://meta:8000 META_URL: http://meta:8000
# TODO: this should be generated from host/port/ssl
USSD_PROVIDER: http://cic-user-ussd-server:9000 USSD_PROVIDER: http://cic-user-ussd-server:9000
CELERY_QUEUE: cic-import-ussd CELERY_QUEUE: cic-import-ussd
EXCLUSIONS: ussd
command: command:
- /bin/bash - /bin/bash
- -c - -c