adding changes

apps/data-seeding/docker/.env

@@ -8,3 +8,7 @@ REDIS_PORT=6379
 USER_USSD_HOST=cic-user-ussd-server
 USER_USSD_PORT=9000
 KEYSTORE_FILE_PATH=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+CIC_DATA_DIR=config/
+OUT_DIR=out
+NUMBER_OF_USERS=10
+META_URL=http://cic-meta-server:63380

apps/data-seeding/docker/Dockerfile

@@ -9,7 +9,7 @@ COPY package.json \
      package-lock.json \
      .
 
-RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
+RUN npm ci --production 
 
 COPY requirements.txt . 
 

apps/data-seeding/scripts/run_job.sh

@@ -3,7 +3,31 @@
 if [[ $((RUN_MASK & 3)) -eq 3 ]]
 then
 	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 3 - data seeding"
-	./scripts/run_ussd_user_imports.sh
+
+    contract_migration_complete=0
+    retry_count=0
+    retry_sleep=30 #seconds
+    retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))"
+    while [[ $contract_migration_complete -ne 1 ]]
+    do
+      if [[ -f "$CIC_DATA_DIR/.env"  ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env
+      then
+        echo "🤜💥🤛 data-seeding found the output of contract-migration!"
+        source /tmp/cic/config/.env
+        env
+        contract_migration_complete=1
+        ./scripts/run_ussd_user_imports.sh
+      elif [[ $retry_count -ge $retry_limit ]] 
+      then
+        echo "😢 data-seeding timeout waiting for contract migration to finish." >&2
+        exit 1
+      else
+        echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..."
+        ((retry_count= $retry_count + $retry_sleep))
+        sleep $retry_sleep 
+      fi
+    done
+  
 	if [ $? -ne "0" ]; then
 		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 3 - data seeding"
 		exit 1;

apps/data-seeding/scripts/run_tasker.sh

@@ -0,0 +1,3 @@
+#! /bin/bash 
+
+python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR

apps/data-seeding/scripts/run_ussd_user_imports.sh

@@ -3,30 +3,6 @@
 set -u
 set -e
 
-contract_migration_complete=0
-retry_count=0
-retry_sleep=30 #seconds
-retry_limit="$((${TIMEOUT_MINUTES:-10}*60/2))"
-while [[ $contract_migration_complete -ne 1 ]]
-do
-  if [[ -f "$CIC_DATA_DIR/.env"  ]] && grep -q CIC_DECLARATOR_ADDRESS $CIC_DATA_DIR/.env
-  then
-    echo "🤜💥🤛 data-seeding found the output of contract-migration!"
-    source /tmp/cic/config/.env
-    env
-    contract_migration_complete=1
-  elif [[ $retry_count -ge $retry_limit ]] 
-  then
-    echo "😢 data-seeding timeout waiting for contract migration to finish." >&2
-    exit 1
-  else
-    echo "⏳ data-seeding waiting for contract-migration output $retry_count:$retry_limit ..."
-    ((retry_count= $retry_count + $retry_sleep))
-    sleep $retry_sleep 
-  fi
-done
-  
-
 
 while getopts ":n:o:g:" opt; do
   case $opt in
@@ -66,14 +42,24 @@ echo "purging existing ussd tasks..."
 
 celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker $CELERY_BROKER_URL -f
 
-
 echo "running import_balance in the background..."
 
-python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
+#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
-  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 2>&1 & 
+#  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR > nohup.out 2> nohup.err < /dev/null & 
+
+#nohup python cic_ussd/import_balance.py -v -c config -p $ETH_PROVIDER \
+#  -r $CIC_REGISTRY_ADDRESS --token-symbol $TOKEN_SYMBOL -y $KEYSTORE_FILE_PATH $OUT_DIR 
 
 echo "import_balance pid: $!" 
 
 echo "importing accounts"
 
 python cic_ussd/import_users.py -vv -c config --ussd-host $USER_USSD_HOST --ussd-port $USER_USSD_PORT --ussd-no-ssl out
+
+sleep 3600
+
+echo "data importing finished!"
+
+echo "run verify manually:"
+echo "python verify.py -v -c config -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER --token-symbol $TOKEN_SYMBOL $OUT_DIR"
+

apps/data-seeding/verify.py

@@ -154,21 +154,25 @@ for t in custodial_tests:
         logg.info('activating custodial module'.format(t))
         break
 
-cols = os.get_terminal_size().columns
+#cols = os.get_terminal_size().columns
-
-
-def to_terminalwidth(s):
-    ss = s.ljust(int(cols)-1)
-    ss += "\r"
-    return ss
-
-def default_outfunc(s):
-    ss = to_terminalwidth(s)
-    sys.stdout.write(ss)
-outfunc = default_outfunc
-if logg.isEnabledFor(logging.DEBUG):
-    outfunc = logg.debug
 
+##disabled because:
+#data-seeding_1          | Traceback (most recent call last):
+#data-seeding_1          |   File "verify.py", line 157, in <module>
+#data-seeding_1          |     cols = os.get_terminal_size().columns
+#data-seeding_1          | OSError: [Errno 25] Inappropriate ioctl for device
+#contract-migration_1    | DEBUG:create_account_script:register False
+#def to_terminalwidth(s):
+#    ss = s.ljust(int(cols)-1)
+#    ss += "\r"
+#    return ss
+#
+#def default_outfunc(s):
+#    ss = to_terminalwidth(s)
+#    sys.stdout.write(ss)
+#outfunc = default_outfunc
+#if logg.isEnabledFor(logging.DEBUG):
+outfunc = logg.debug
 
 def send_ussd_request(address, data_dir):
     upper_address = strip_0x(address).upper()

docker-compose.test.yml

@@ -0,0 +1,120 @@
+version: "3"
+
+services:
+  contract-migration:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration
+    build:
+      context: apps/contract-migration
+      dockerfile: docker/Dockerfile
+      args:
+        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
+        pip_extra_args: $PIP_EXTRA_ARGS
+    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
+    environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
+      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
+      ETH_PROVIDER: http://eth:8545
+      # And these two are for wait-for-it (could parse this)
+      DEV_USE_DOCKER_WAIT_SCRIPT: 1
+      ETH_PROVIDER_HOST: eth 
+      ETH_PROVIDER_PORT: 8545
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
+      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
+      DATABASE_HOST: ${DATABASE_HOST:-postgres}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+      DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
+      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
+      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
+      DATABASE_USER: ${DATABASE_USER:-postgres}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_DB: ${REDIS_DB:-0}
+      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
+      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
+      RUN_MASK: ${RUN_MASK:-0} 
+      DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
+      #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
+      DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
+      CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
+      TOKEN_NAME: $TOKEN_NAME
+      TOKEN_DECIMALS: $TOKEN_DECIMALS
+      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
+      TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
+      TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
+      TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
+      TOKEN_TYPE: $TOKEN_TYPE
+        #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
+    command: ["./run_job.sh"]
+        #command: ["./reset.sh"]
+    volumes:
+      - contract-config:/tmp/cic/config
+
+
+  data-seeding-tasker:
+    build:
+      context: apps/data-seeding
+      dockerfile: docker/Dockerfile
+      args:
+        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
+        pip_extra_args: $PIP_EXTRA_ARGS
+    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
+    environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
+      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
+      ETH_PROVIDER: http://eth:8545
+      # And these two are for wait-for-it (could parse this)
+      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
+      DEV_USE_DOCKER_WAIT_SCRIPT: 1
+      ETH_PROVIDER_HOST: eth 
+      ETH_PROVIDER_PORT: 8545
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_DB: ${REDIS_DB:-0}
+      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
+      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
+      RUN_MASK: ${RUN_MASK:-0} # 7
+      TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
+      USER_USSD_HOST: cic-user-ussd-server
+      USER_USSD_PORT: 9000
+      KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
+      OUT_DIR: out
+      NUMBER_OF_USERS: 10
+      #command: "python cic_ussd/import_balance.py -v -c config -p $$ETH_PROVIDER -r $$CIC_REGISTRY_ADDRESS --token-symbol $$TOKEN_SYMBOL -y $$KEYSTORE_FILE_PATH $$OUT_DIR"
+    command: "bash scripts/run_tasker.sh"
+    volumes:
+      - contract-config:/tmp/cic/config
+
+  data-seeding:
+    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding
+    build:
+      context: apps/data-seeding
+      dockerfile: docker/Dockerfile
+      args:
+        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
+        pip_extra_args: $PIP_EXTRA_ARGS
+    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
+    environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
+      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
+      ETH_PROVIDER: http://eth:8545
+      # And these two are for wait-for-it (could parse this)
+      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
+      DEV_USE_DOCKER_WAIT_SCRIPT: 1
+      ETH_PROVIDER_HOST: eth 
+      ETH_PROVIDER_PORT: 8545
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_DB: ${REDIS_DB:-0}
+      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
+      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
+      RUN_MASK: ${RUN_MASK:-0} # 7
+      TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
+      USER_USSD_HOST: cic-user-ussd-server
+      USER_USSD_PORT: 9000
+      KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
+      OUT_DIR: out
+      NUMBER_OF_USERS: 10
+    command: ["bash", "./scripts/run_job.sh"]
+    volumes:
+      - contract-config:/tmp/cic/config
+

docker-compose.yml

@@ -55,93 +55,6 @@ services:
     volumes:
       - bee-data:/tmp/cic/bee
 
-  contract-migration:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration
-    build:
-      context: apps/contract-migration
-      dockerfile: docker/Dockerfile
-      args:
-        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
-        pip_extra_args: $PIP_EXTRA_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
-      ETH_PROVIDER: http://eth:8545
-      # And these two are for wait-for-it (could parse this)
-      DEV_USE_DOCKER_WAIT_SCRIPT: 1
-      ETH_PROVIDER_HOST: eth 
-      ETH_PROVIDER_PORT: 8545
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
-      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
-      DATABASE_HOST: ${DATABASE_HOST:-postgres}
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
-      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
-      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_USER: ${DATABASE_USER:-postgres}
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_DB: ${REDIS_DB:-0}
-      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
-      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
-      RUN_MASK: ${RUN_MASK:-0} 
-      DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
-      #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
-      DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
-      CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
-      TOKEN_NAME: $TOKEN_NAME
-      TOKEN_DECIMALS: $TOKEN_DECIMALS
-      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
-      TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
-      TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
-      TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
-      TOKEN_TYPE: $TOKEN_TYPE
-        #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
-    command: ["./run_job.sh"]
-        #command: ["./reset.sh"]
-    depends_on:
-      - eth
-      - postgres
-      - redis
-      - cic-eth-tasker
-    volumes:
-      - contract-config:/tmp/cic/config
-
-  data-seeding:
-    image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding
-    build:
-      context: apps/data-seeding
-      dockerfile: docker/Dockerfile
-      args:
-        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
-        pip_extra_args: $PIP_EXTRA_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-      # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
-      ETH_PROVIDER: http://eth:8545
-      # And these two are for wait-for-it (could parse this)
-      CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
-      DEV_USE_DOCKER_WAIT_SCRIPT: 1
-      ETH_PROVIDER_HOST: eth 
-      ETH_PROVIDER_PORT: 8545
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_DB: ${REDIS_DB:-0}
-      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
-      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
-      RUN_MASK: ${RUN_MASK:-0} # 7
-      TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
-      USER_USSD_HOST: cic-user-ussd-server
-      USER_USSD_PORT: 9000
-      KEYSTORE_FILE_PATH: "/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c"
-      OUT_DIR: out
-      NUMBER_OF_USERS: 10
-    command: ["bash", "./scripts/run_job.sh"]
-    volumes:
-      - contract-config:/tmp/cic/config
-
   cic-cache-tracker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache
     build: