refactor docker-compose cic-eth-tasker, bootstrap (aka contract migration)

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         4,
-        'alpha.8',
+        'alpha.12',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -1,45 +1,53 @@
-# syntax = docker/dockerfile:1.2
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
 # Copy just the requirements and install....this _might_ give docker a hint on caching but we 
 # do load these all into setup.py later
 # TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
 #COPY cic-eth/requirements.txt .
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG pip_index_url=https://pypi.org/simple
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG EXTRA_PIP_ARGS=""
+ARG PIP_INDEX_URL="https://pypi.org/simple"
+
+#ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+#ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+#ARG EXTRA_PIP_ARGS=""
 #RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
 #    pip install --index-url https://pypi.org/simple \
 #    --force-reinstall \
 #	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
 #    -r requirements.txt
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
+RUN    pip install --index-url $PIP_INDEX_URL \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --pre \
-    --extra-index-url $EXTRA_INDEX_URL \
+    --force-reinstall \
-    $EXTRA_PIP_ARGS \
+    --no-cache \
-    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+        cic-eth-aux-erc20-demurrage-token~=0.0.2a6
 
 
 COPY *requirements.txt ./
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
+RUN    pip install --index-url $PIP_INDEX_URL \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --pre \
-    --extra-index-url $EXTRA_INDEX_URL \
+    --force-reinstall \
-    $EXTRA_PIP_ARGS \
+    --no-cache \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt 
    
 # always install the latest signer 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
+#    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+#	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
+#    --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
+#    $EXTRA_PIP_ARGS \
-    crypto-dev-signer
+#    crypto-dev-signer
 
 COPY . .
 RUN python setup.py install

apps/cic-eth/setup.cfg

@@ -1,6 +1,7 @@
 [metadata]
 name = cic-eth
-version = attr: cic_eth.version.__version_string__
+#version = attr: cic_eth.version.__version_string__
+version = 0.12.4a12
 description = CIC Network Ethereum interaction 
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth/tools_requirements.txt

@@ -1,5 +1,5 @@
 crypto-dev-signer>=0.4.15a7,<=0.4.15
-chainqueue>=0.0.5a1,<0.1.0
+chainqueue>=0.0.5a3,<0.1.0
 cic-eth-registry>=0.6.1a6,<0.7.0
 redis==3.5.3
 hexathon~=0.0.1a8

apps/contract-migration/1_deploy_contract_root.sh

@@ -17,7 +17,6 @@ must_eth_rpc
 
 # Deploy address declarator registry
 >&2 echo -e "\033[;96mDeploy address declarator contract\033[;39m"
-echo "DEV_ADDRESS_DECLARATOR=eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w $DEV_DEBUG_FLAG $DEV_DECLARATOR_DESCRIPTION"
 DEV_ADDRESS_DECLARATOR=`eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w $DEV_DEBUG_FLAG $DEV_DECLARATOR_DESCRIPTION`
 
 echo -e "\033[;96mWriting env_reset file\033[;39m"

apps/contract-migration/config.sh

@@ -40,9 +40,4 @@ export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOY
 # Migration variable processing
 confini-dump --schema-dir ./config > ${DEV_DATA_DIR}/env_reset
 
-#echo "export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS
-#export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
-#export WALLET_KEY_FILE=$WALLET_KEY_FILE
-#" >> ${DEV_DATA_DIR}/env_reset
-
 set +a

apps/contract-migration/docker/Dockerfile

@@ -1,5 +1,6 @@
-# syntax = docker/dockerfile:1.2
+ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e
+
+FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
 
 WORKDIR /root
 
@@ -9,8 +10,6 @@ RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main'
 RUN cat /etc/apt/sources.list.d/ethereum.list
 RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
 
-#RUN apt-get install solc
-
 RUN mkdir -vp /usr/local/etc/cic
 
 ENV CONFINI_DIR /usr/local/etc/cic/
@@ -18,29 +17,18 @@ ENV CONFINI_DIR /usr/local/etc/cic/
 
 COPY config_template/ /usr/local/etc/cic/
 COPY requirements.txt . 
-COPY override_requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG EXTRA_PIP_ARGS=""
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG PIP_INDEX_URL="https://pypi.org/simple"
 ARG pip_trusted_host=pypi.org
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+RUN    pip install --index-url $PIP_INDEX_URL \
-    pip install --index-url https://pypi.org/simple \
     --pre \
     --force-reinstall \
-    --trusted-host $pip_trusted_host \
+    --no-cache \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt 
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple \
-    --force-reinstall \
-    --pre \
-    --trusted-host $pip_trusted_host \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-    -r override_requirements.txt 
-
-
 COPY . .
 RUN chmod +x *.sh

apps/contract-migration/override_requirements.txt

@@ -1,4 +0,0 @@
-#eth-contract-registry==0.6.3a2
-#erc20-demurrage-token==0.0.2a3
-#eth-address-index==0.1.1a12
-

apps/contract-migration/requirements.txt

@@ -3,9 +3,8 @@ chainlib-eth>=0.0.9rc4,<0.0.10
 chainlib==0.0.9rc1,<0.0.10
 eth-erc20>=0.1.2a3,<0.2.0
 erc20-demurrage-token>=0.0.5a2,<0.1.0
-#eth-accounts-index>=0.1.2a2,<0.2.0
 eth-address-index>=0.2.4a1,<0.3.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a6,<0.7.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 erc20-faucet>=0.3.2a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0

docker-compose_local.yml

@@ -46,21 +46,16 @@ services:
       context: apps/contract-migration
       dockerfile: docker/Dockerfile
       args:
+          DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
           PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
-          EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
+          EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
           EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
     environment:
       DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}
       DEV_CONFIG_RESET: $DEV_CONFIG_RESET
-      RPC_PROVIDER: $RPC_PROVIDER
+      RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
-      CHAIN_SPEC: $CHAIN_SPEC
+      CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"}
       DEV_USE_DOCKER_WAIT_SCRIPT: 1
-      DATABASE_HOST: ${DATABASE_HOST:-postgres}
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth}
-      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql}
-      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_USER: ${DATABASE_USER:-postgres}
       REDIS_HOST: ${REDIS_HOST:-redis}
       REDIS_PORT: ${REDIS_PORT:-6379}
       REDIS_DB: ${REDIS_DB:-0}
@@ -78,8 +73,65 @@ services:
     command: ["./run_job.sh"]
     depends_on:
       - evm
-        #- postgres
+      - postgres
-        #- redis
+      - redis
         #- cic-eth-tasker
     volumes:
       - contract-config:/tmp/cic/config
+
+
+  cic-eth-tasker:
+    image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest}
+    build: 
+      context: apps/cic-eth
+      dockerfile: docker/Dockerfile
+      target: dev
+      args:
+          DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
+          PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
+          EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
+          EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
+    environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
+      DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}
+      DEV_CONFIG_RESET: $DEV_CONFIG_RESET
+      RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
+      CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"}
+      DEV_USE_DOCKER_WAIT_SCRIPT: 1
+      DATABASE_HOST: ${DATABASE_HOST:-postgres}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+      DATABASE_NAME: ${DATABASE_NAME:-cic_eth}
+      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
+      DATABASE_USER: ${DATABASE_USER:-grassroots}
+      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
+      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
+      DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
+      DATABASE_POOL_SIZE: 0
+      REDIS_PORT: 6379
+      REDIS_HOST: redis
+      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
+      CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
+      CELERY_DEBUG: ${CELERY_DEBUG:-1}
+      SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-eth-signer:8000}
+      SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
+      TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
+    restart: unless-stopped 
+    depends_on:
+      - evm
+      - postgres
+      - redis
+        #   - cic-eth-signer
+    volumes:
+      - signer-data:/run/crypto-dev-signer
+      - contract-config:/tmp/cic/config/:ro
+        #command: ["/usr/local/bin/cic-eth-taskerd"]
+        #command: ["sleep", "3600"]
+    command:
+      - /bin/bash
+      - -c
+      - |
+        set -a
+        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        set +a
+        ./start_tasker.sh --aux-all -q cic-eth  -vv
+