From 7b57f1b4c2a7decae34bea20f9351f44fb416fc1 Mon Sep 17 00:00:00 2001 From: Philip Wafula Date: Tue, 21 Sep 2021 06:04:44 +0000 Subject: [PATCH] Dataseeding container --- .gitignore | 1 + .gitlab-ci.yml | 6 +- apps/data-seeding/cic_ussd/import_balance.py | 2 +- apps/data-seeding/config/cic.ini | 7 +-- apps/data-seeding/create_import_users.py | 9 ++- apps/data-seeding/docker/Dockerfile | 3 +- apps/data-seeding/import_ussd.sh | 64 +++++++++++++++----- apps/data-seeding/requirements.txt | 9 +-- apps/data-seeding/verify.py | 14 +---- ci_templates/.cic-template.yml | 12 +++- docker-compose.yml | 47 +++++++++++++- 11 files changed, 123 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index 3fb25a37..bd35ec75 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ build/ **/.venv .idea **/.vim +**/*secret.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 584d01c1..67ccdc9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,13 +12,13 @@ include: stages: - build - test - - deploy + - deploy -image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest +image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest variables: DOCKER_BUILDKIT: "1" - COMPOSE_DOCKER_CLI_BUILD: "1" + COMPOSE_DOCKER_CLI_BUILD: "1" CI_DEBUG_TRACE: "true" before_script: diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 9f974e70..8fb487a3 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -16,7 +16,7 @@ from crypto_dev_signer.keystore.dict import DictKeystore from import_util import BalanceProcessor, get_celery_worker_status from import_task import ImportTask, MetadataTask -default_config_dir = './config' +default_config_dir = '/usr/local/etc/data-seeding/' logg = logging.getLogger() arg_parser = argparse.ArgumentParser(description='Daemon worker that handles data seeding tasks.') diff --git a/apps/data-seeding/config/cic.ini b/apps/data-seeding/config/cic.ini index 5064df15..2e346dae 100644 --- a/apps/data-seeding/config/cic.ini +++ b/apps/data-seeding/config/cic.ini @@ -1,11 +1,10 @@ [cic] -registry_address = 0x32E860c2A0645d1B7B005273696905F5D6DC5D05 +registry_address = token_index_address = accounts_index_address = declarator_address = approval_escrow_address = -chain_spec = evm:bloxberg:8996 +chain_spec = tx_retry_delay = -trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C +trust_address = user_ussd_svc_service_port = - diff --git a/apps/data-seeding/create_import_users.py b/apps/data-seeding/create_import_users.py index aa8e4e59..01d58c09 100644 --- a/apps/data-seeding/create_import_users.py +++ b/apps/data-seeding/create_import_users.py @@ -28,12 +28,11 @@ logg = logging.getLogger() fake = Faker(['sl', 'en_US', 'no', 'de', 'ro']) -script_dir = os.path.realpath(os.path.dirname(__file__)) -# config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') -config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config')) +default_config_dir = './config' + argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=config_dir, help='Config dir') +argparser.add_argument('-c', type=str, default=default_config_dir, help='Config dir') argparser.add_argument('--tag', type=str, action='append', help='Tags to add to record') argparser.add_argument('--gift-threshold', type=int, @@ -53,7 +52,7 @@ elif args.vv: config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX')) config.process() -logg.info('loaded config\n{}'.format(config)) +logg.debug('loaded config\n{}'.format(config)) dt_now = datetime.datetime.utcnow() diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index 4113c2d1..f993e33c 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -13,7 +13,8 @@ COPY package.json \ RUN npm ci --production #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install -COPY requirements.txt . +COPY requirements.txt . +COPY config/ /usr/local/etc/data-seeding ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" diff --git a/apps/data-seeding/import_ussd.sh b/apps/data-seeding/import_ussd.sh index 02773481..1eb7e3f6 100644 --- a/apps/data-seeding/import_ussd.sh +++ b/apps/data-seeding/import_ussd.sh @@ -1,60 +1,94 @@ -#!/usr/bin/env bash +#!/bin/bash -set -e +if [[ -d "$OUT_DIR" ]] +then + echo "found existing IMPORT DIR cleaning up..." + rm -rf "$OUT_DIR" + mkdir -p "$OUT_DIR" +else + echo "IMPORT DIR does not exist creating it." + mkdir -p "$OUT_DIR" +fi + +# using timeout because the timeout flag for celery inspect does not work +timeout 5 celery inspect ping -b "$CELERY_BROKER_URL" +if [[ $? -eq 124 ]] +then + >&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?" + exit 1 +fi echo "Creating seed data..." -python create_import_users.py -vv --dir "$IMPORT_DIR" "$ACCOUNT_COUNT" +python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS" wait $! + +echo "Check for running celery workers ..." +if [ -f ./cic-ussd-import.pid ]; +then + echo "Found a running worker. Killing ..." + kill -9 $( nohup.out 2> nohup.err < /dev/null & else echo "Running worker with opening balance transactions" - TARGET_TX_COUNT=$((ACCOUNT_COUNT*2)) - python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$IMPORT_DIR" & + TARGET_TX_COUNT=$((NUMBER_OF_USERS*2)) + nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" & fi +echo "Target count set to ${TARGET_TX_COUNT}" until [ -f ./cic-import-ussd.pid ] do echo "Polling for celery worker pid file..." sleep 1 done IMPORT_BALANCE_JOB=$(=0.4.15a4,<=0.4.15 +crypto-dev-signer~=0.4.15a7 faker==4.17.1 chainsyncer~=0.0.6a3 -chainlib-eth~=0.0.9a13 +chainlib-eth~=0.0.9a14 eth-address-index~=0.2.3a4 eth-contract-registry~=0.6.3a3 eth-accounts-index~=0.1.2a3 -eth-erc20~=0.1.2a2 +eth-erc20~=0.1.2a3 erc20-faucet~=0.3.2a2 psycopg2==2.8.6 +liveness~=0.0.1a7 diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 3db3e919..216b26ef 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -164,20 +164,8 @@ for t in custodial_tests: logg.info('activating custodial module'.format(t)) break -cols = os.get_terminal_size().columns - -def to_terminalwidth(s): - ss = s.ljust(int(cols)-1) - ss += "\r" - return ss - -def default_outfunc(s): - ss = to_terminalwidth(s) - sys.stdout.write(ss) -outfunc = default_outfunc -if logg.isEnabledFor(logging.DEBUG): - outfunc = logg.debug +outfunc = logg.debug def send_ussd_request(address, data_dir): diff --git a/ci_templates/.cic-template.yml b/ci_templates/.cic-template.yml index 5ba47394..2ecba76b 100644 --- a/ci_templates/.cic-template.yml +++ b/ci_templates/.cic-template.yml @@ -1,12 +1,12 @@ -image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] variables: KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h" MR_IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA .py_build_merge_request: + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] stage: build script: - mkdir -p /kaniko/.docker @@ -16,6 +16,9 @@ variables: --cache-repo $CI_REGISTRY_IMAGE --destination $MR_IMAGE_TAG .py_build_target_dev: + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] stage: build variables: IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA @@ -28,6 +31,9 @@ variables: --destination $MR_IMAGE_TAG .py_build_push: + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] stage: build variables: IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA diff --git a/docker-compose.yml b/docker-compose.yml index 4c5b8f5f..143f9818 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,8 +94,8 @@ services: RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0} DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE - TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token} - TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET} + TOKEN_NAME: ${TOKEN_NAME:-Giftable Token} + TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} TOKEN_DECIMALS: $TOKEN_DECIMALS TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} @@ -115,6 +115,49 @@ services: volumes: - contract-config:/tmp/cic/config + + data-seeding: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:${TAG:-latest} + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} + pip_extra_args: $PIP_EXTRA_ARGS + environment: + CIC_REGISTRY_ADDRESS: ${CIC_REGISTRY_ADDRESS:-0xea6225212005e86a4490018ded4bf37f3e772161} + OUT_DIR: out + NUMBER_OF_USERS: 10 + CONFIG: /usr/local/etc/data-seeding + CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + ETH_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} + TOKEN_SYMBOL: GFT + KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c + USSD_HOST: cic-user-ussd-server + USSD_PORT: 9000 + INCLUDE_BALANCES: y + USSD_SSL: n + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + NOTIFY_DATABASE_NAME: cic_notify + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + META_HOST: meta + META_PORT: 8000 + META_URL: http://meta:8000 + USSD_PROVIDER: http://cic-user-ussd-server:9000 + CELERY_QUEUE: cic-import-ussd + EXCLUSIONS: ussd + command: bash import_ussd.sh + volumes: + - contract-config:/tmp/cic/config/:ro + cic-cache-tracker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} profiles: