migrations: Enable deployment and data seeding to Bloxberg

2021-12-22 18:24:05 +00:00
parent b15cfee1c9
commit d7c4cb71eb
65 changed files with 940 additions and 224 deletions
--- a/apps/data-seeding/cic_eth/import_users.py
+++ b/apps/data-seeding/cic_eth/import_users.py
@@ -40,7 +40,7 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, help='config override directory')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
 argparser.add_argument('-f', action='store_true', help='force clear previous state')
-argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec')
 argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
 argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
 argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@@ -227,9 +227,10 @@ if __name__ == '__main__':
                    )
            os.makedirs(os.path.dirname(filepath), exist_ok=True)
           
-            sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
+            sub_old_chain_str = '{}:{}'.format(old_chain_spec.network_id(), old_chain_spec.common_name())
+            logg.debug('u id {}'.format(u.identities))
            f = open(filepath, 'w')
-            k = u.identities['evm'][sub_old_chain_str][0]
+            k = u.identities['evm'][old_chain_spec.fork()][sub_old_chain_str][0]
            tag_data = {'tags': user_tags[strip_0x(k)]}
            f.write(json.dumps(tag_data))
            f.close()
--- a/apps/data-seeding/cic_ussd/import_balance.py
+++ b/apps/data-seeding/cic_ussd/import_balance.py
@@ -65,7 +65,8 @@ args_override = {
    'REDIS_DB': getattr(args, 'redis_db'),
    'META_HOST': getattr(args, 'meta_host'),
    'META_PORT': getattr(args, 'meta_port'),
-    'WALLET_KEY_FILE': getattr(args, 'y')
+    'WALLET_KEY_FILE': getattr(args, 'y'),
+    'TOKEN_SYMBOL': getattr(args, 'token_symbol'),
 }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -110,7 +111,7 @@ def main():
                                                    config.get('CIC_REGISTRY_ADDRESS'),
                                                    signer_address,
                                                    signer)
-    ImportTask.balance_processor.init(args.token_symbol)
+    ImportTask.balance_processor.init(config.get('TOKEN_SYMBOL'))
    balances = {}
    accuracy = 10 ** 6
    count = 0
--- a/apps/data-seeding/config/token.ini
+++ b/apps/data-seeding/config/token.ini
@@ -0,0 +1,2 @@
+[token]
+symbol =
--- a/apps/data-seeding/config/traffic.ini
+++ b/apps/data-seeding/config/traffic.ini
@@ -0,0 +1,4 @@
+[traffic]
+#local.noop_traffic = 2
+#local.account = 2
+local.transfer = 2
--- a/apps/data-seeding/docker/Dockerfile
+++ b/apps/data-seeding/docker/Dockerfile
@@ -1,16 +1,25 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2

 WORKDIR /root

 RUN mkdir -vp /usr/local/etc/cic

-COPY package.json \
-     package-lock.json \
-     ./
+ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org}
+RUN npm config set snyk=false
+#RUN npm config set registry={NPM_REPOSITORY}
+RUN npm config set registry=${NPM_REPOSITORY}
+
+# copy the dependencies
+COPY package.json package-lock.json ./
+RUN --mount=type=cache,mode=0755,target=/root/.npm \
+    npm set cache /root/.npm && \
+    npm cache verify && \
+    npm ci --verbose


-RUN npm ci --production 
+#RUN npm ci --production --verbose
 #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install

 COPY common/ cic_ussd/common/
@@ -22,6 +31,7 @@ ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL=https://pypi.org/simple

 RUN  pip install --index-url $PIP_INDEX_URL  \
+     --pre \
     --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt

--- a/apps/data-seeding/import_ussd.sh
+++ b/apps/data-seeding/import_ussd.sh
@@ -1,5 +1,11 @@
 #!/bin/bash

+. /tmp/cic/config/env_reset
+
+OUT_DIR=out
+CONFIG_DIR=config
+confini-dump --schema-dir $CONFIG_DIR
+
 if [[ -d "$OUT_DIR" ]]
 then
  echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m"
@@ -19,14 +25,14 @@ then
 fi

 echo -e "\033[;96mCreating seed data...\033[;96m"
-python create_import_users.py -vv  -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
+python create_import_users.py -vv  -c "$CONFIG_DIR" --dir "$OUT_DIR" "$NUMBER_OF_USERS"
 wait $!

 echo -e "\033[;96mCheck for running celery workers ...\033[;96m"
 if [ -f ./cic-ussd-import.pid ];
 then
  echo -e "\033[;96mFound a running worker. Killing ...\033[;96m"
-  kill -9 $(<cic-ussd-import.pid)
+  kill -TERM $(<cic-ussd-import.pid)
 fi

 echo -e "\033[;96mPurge tasks from celery worker\033[;96m"
@@ -37,11 +43,11 @@ if [ "$INCLUDE_BALANCES" != "y" ]
 then
  echo -e "\033[;96mRunning worker without opening balance transactions\033[;96m"
  TARGET_TX_COUNT=$NUMBER_OF_USERS
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
+  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null &
 else
  echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m"
  TARGET_TX_COUNT=$((NUMBER_OF_USERS*2))
-  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" &
+  nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" &
 fi

 echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}"
@@ -53,12 +59,12 @@ done
 IMPORT_BALANCE_JOB=$(<cic-import-ussd.pid)

 echo -e "\033[;96mStart import users job\033[;96m"
-if [ "$USSD_SSL" == "y" ]
+if [ ! -z "$USSD_SSL" ]
 then
  echo -e "\033[;96mTargeting secure ussd-user server\033[;96m"
-  python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" "$OUT_DIR"
 else
-  python cic_ussd/import_users.py -vv -f -c "$CONFIG" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
+  python cic_ussd/import_users.py -vv -f -c "$CONFIG_DIR" --ussd-host "$USSD_HOST" --ussd-port "$USSD_PORT" --ussd-no-ssl "$OUT_DIR"
 fi

 echo -e "\033[;96mWaiting for import balance job to complete ...\033[;96m"
@@ -66,13 +72,13 @@ tail --pid="$IMPORT_BALANCE_JOB" -f /dev/null
 set -e

 echo -e "\033[;96mImporting pins\033[;96m"
-python cic_ussd/import_pins.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_pins.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
 set +e
 wait $!
 set -e

 echo -e "\033[;96mImporting ussd data\033[;96m"
-python cic_ussd/import_ussd_data.py -c "$CONFIG" -vv "$OUT_DIR"
+python cic_ussd/import_ussd_data.py -c "$CONFIG_DIR" -vv "$OUT_DIR"
 set +e
 wait $!

@@ -82,8 +88,9 @@ node cic_meta/import_meta.js "$OUT_DIR" "$NUMBER_OF_USERS"
 echo -e "\033[;96mImport preferences metadata\033[;96m"
 node cic_meta/import_meta_preferences.js "$OUT_DIR" "$NUMBER_OF_USERS"

-CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$NOTIFY_DATABASE_NAME
+CIC_NOTIFY_DATABASE=postgres://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME_NOTIFY
 NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
+# TODO: add max wait
 while (("$NOTIFICATION_COUNT" < "$TARGET_TX_COUNT" ))
 do
  NOTIFICATION_COUNT=$(psql -qtA "$CIC_NOTIFY_DATABASE" -c 'SELECT COUNT(message) FROM notification WHERE message IS NOT NULL')
--- a/apps/data-seeding/requirements.txt
+++ b/apps/data-seeding/requirements.txt
@@ -1,10 +1,10 @@
 sarafu-faucet~=0.0.7a2
 cic-eth[tools]~=0.12.4a13
-cic-types~=0.2.0a6
+cic-types~=0.2.1a2
 funga>=0.5.1a1,<=0.5.15
 faker==4.17.1
 chainsyncer~=0.0.7a3
-chainlib-eth~=0.0.10a10
+chainlib-eth~=0.0.10a18
 eth-address-index~=0.2.4a1
 eth-contract-registry~=0.6.3a3
 eth-accounts-index~=0.1.2a3
--- a/apps/data-seeding/verify.py
+++ b/apps/data-seeding/verify.py
@@ -60,7 +60,7 @@ eth_tests = [
        ]

 phone_tests = [
-        'ussd',
+#        'ussd',
        'ussd_pins'
        ]