From 97f4fe8ca747d39842c6803d620bf64d54dec903 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 15 Oct 2021 22:16:45 +0200 Subject: [PATCH] refactor docker-compose cic-eth-tasker, bootstrap (aka contract migration) --- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/docker/Dockerfile | 50 +++++++------ apps/cic-eth/setup.cfg | 3 +- apps/cic-eth/tools_requirements.txt | 2 +- .../1_deploy_contract_root.sh | 1 - apps/contract-migration/config.sh | 5 -- apps/contract-migration/docker/Dockerfile | 28 ++----- .../override_requirements.txt | 4 - apps/contract-migration/requirements.txt | 3 +- docker-compose_local.yml | 74 ++++++++++++++++--- 10 files changed, 105 insertions(+), 67 deletions(-) delete mode 100644 apps/contract-migration/override_requirements.txt diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 9f366d63..f6baa79c 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 12, 4, - 'alpha.8', + 'alpha.12', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 579eeb9f..3c27babd 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -1,45 +1,53 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev +ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" + +FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e as dev # Copy just the requirements and install....this _might_ give docker a hint on caching but we # do load these all into setup.py later # TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py #COPY cic-eth/requirements.txt . -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +ARG pip_index_url=https://pypi.org/simple +ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG EXTRA_PIP_ARGS="" +ARG PIP_INDEX_URL="https://pypi.org/simple" + +#ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +#ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +#ARG EXTRA_PIP_ARGS="" #RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ # pip install --index-url https://pypi.org/simple \ # --force-reinstall \ # --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ # -r requirements.txt -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --index-url https://pypi.org/simple \ - --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL \ - $EXTRA_PIP_ARGS \ - cic-eth-aux-erc20-demurrage-token~=0.0.2a6 +#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ +RUN pip install --index-url $PIP_INDEX_URL \ + --pre \ + --force-reinstall \ + --no-cache \ + --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ + cic-eth-aux-erc20-demurrage-token~=0.0.2a6 COPY *requirements.txt ./ -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --index-url https://pypi.org/simple \ - --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL \ - $EXTRA_PIP_ARGS \ +#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ +RUN pip install --index-url $PIP_INDEX_URL \ + --pre \ + --force-reinstall \ + --no-cache \ + --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt \ -r services_requirements.txt \ -r admin_requirements.txt # always install the latest signer -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --index-url https://pypi.org/simple \ - --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL \ - $EXTRA_PIP_ARGS \ - crypto-dev-signer +#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ +# pip install --index-url https://pypi.org/simple \ +# --extra-index-url $GITLAB_PYTHON_REGISTRY \ +# --extra-index-url $EXTRA_INDEX_URL \ +# $EXTRA_PIP_ARGS \ +# crypto-dev-signer COPY . . RUN python setup.py install diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index 2b798095..27829f23 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -1,6 +1,7 @@ [metadata] name = cic-eth -version = attr: cic_eth.version.__version_string__ +#version = attr: cic_eth.version.__version_string__ +version = 0.12.4a12 description = CIC Network Ethereum interaction author = Louis Holbrook author_email = dev@holbrook.no diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index e4085ff9..c4a31eb0 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,5 +1,5 @@ crypto-dev-signer>=0.4.15a7,<=0.4.15 -chainqueue>=0.0.5a1,<0.1.0 +chainqueue>=0.0.5a3,<0.1.0 cic-eth-registry>=0.6.1a6,<0.7.0 redis==3.5.3 hexathon~=0.0.1a8 diff --git a/apps/contract-migration/1_deploy_contract_root.sh b/apps/contract-migration/1_deploy_contract_root.sh index ba89dd59..91dbe27d 100644 --- a/apps/contract-migration/1_deploy_contract_root.sh +++ b/apps/contract-migration/1_deploy_contract_root.sh @@ -17,7 +17,6 @@ must_eth_rpc # Deploy address declarator registry >&2 echo -e "\033[;96mDeploy address declarator contract\033[;39m" -echo "DEV_ADDRESS_DECLARATOR=eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w $DEV_DEBUG_FLAG $DEV_DECLARATOR_DESCRIPTION" DEV_ADDRESS_DECLARATOR=`eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w $DEV_DEBUG_FLAG $DEV_DECLARATOR_DESCRIPTION` echo -e "\033[;96mWriting env_reset file\033[;39m" diff --git a/apps/contract-migration/config.sh b/apps/contract-migration/config.sh index 074513b2..60c50eb0 100644 --- a/apps/contract-migration/config.sh +++ b/apps/contract-migration/config.sh @@ -40,9 +40,4 @@ export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOY # Migration variable processing confini-dump --schema-dir ./config > ${DEV_DATA_DIR}/env_reset -#echo "export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS -#export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL -#export WALLET_KEY_FILE=$WALLET_KEY_FILE -#" >> ${DEV_DATA_DIR}/env_reset - set +a diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 3aa0bfc2..65f522f2 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -1,5 +1,6 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e +ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" + +FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e WORKDIR /root @@ -9,8 +10,6 @@ RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' RUN cat /etc/apt/sources.list.d/ethereum.list RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9 -#RUN apt-get install solc - RUN mkdir -vp /usr/local/etc/cic ENV CONFINI_DIR /usr/local/etc/cic/ @@ -18,29 +17,18 @@ ENV CONFINI_DIR /usr/local/etc/cic/ COPY config_template/ /usr/local/etc/cic/ COPY requirements.txt . -COPY override_requirements.txt . ARG pip_index_url=https://pypi.org/simple -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG EXTRA_PIP_ARGS="" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +ARG PIP_INDEX_URL="https://pypi.org/simple" ARG pip_trusted_host=pypi.org -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --index-url https://pypi.org/simple \ +RUN pip install --index-url $PIP_INDEX_URL \ --pre \ --force-reinstall \ - --trusted-host $pip_trusted_host \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \ + --no-cache \ + --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --index-url https://pypi.org/simple \ - --force-reinstall \ - --pre \ - --trusted-host $pip_trusted_host \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \ - -r override_requirements.txt - - COPY . . RUN chmod +x *.sh diff --git a/apps/contract-migration/override_requirements.txt b/apps/contract-migration/override_requirements.txt deleted file mode 100644 index 27979c8d..00000000 --- a/apps/contract-migration/override_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -#eth-contract-registry==0.6.3a2 -#erc20-demurrage-token==0.0.2a3 -#eth-address-index==0.1.1a12 - diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index a92cf0b4..56f7da67 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -3,9 +3,8 @@ chainlib-eth>=0.0.9rc4,<0.0.10 chainlib==0.0.9rc1,<0.0.10 eth-erc20>=0.1.2a3,<0.2.0 erc20-demurrage-token>=0.0.5a2,<0.1.0 -#eth-accounts-index>=0.1.2a2,<0.2.0 eth-address-index>=0.2.4a1,<0.3.0 -cic-eth-registry>=0.6.1a2,<0.7.0 +cic-eth-registry>=0.6.1a6,<0.7.0 erc20-transfer-authorization>=0.3.5a2,<0.4.0 erc20-faucet>=0.3.2a2,<0.4.0 sarafu-faucet>=0.0.7a2,<0.1.0 diff --git a/docker-compose_local.yml b/docker-compose_local.yml index 7d5b2b10..1e60b65f 100644 --- a/docker-compose_local.yml +++ b/docker-compose_local.yml @@ -46,21 +46,16 @@ services: context: apps/contract-migration dockerfile: docker/Dockerfile args: + DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config} DEV_CONFIG_RESET: $DEV_CONFIG_RESET - RPC_PROVIDER: $RPC_PROVIDER - CHAIN_SPEC: $CHAIN_SPEC + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"} DEV_USE_DOCKER_WAIT_SCRIPT: 1 - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_USER: ${DATABASE_USER:-postgres} REDIS_HOST: ${REDIS_HOST:-redis} REDIS_PORT: ${REDIS_PORT:-6379} REDIS_DB: ${REDIS_DB:-0} @@ -78,8 +73,65 @@ services: command: ["./run_job.sh"] depends_on: - evm - #- postgres - #- redis + - postgres + - redis #- cic-eth-tasker volumes: - contract-config:/tmp/cic/config + + + cic-eth-tasker: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} + build: + context: apps/cic-eth + dockerfile: docker/Dockerfile + target: dev + args: + DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config} + DEV_CONFIG_RESET: $DEV_CONFIG_RESET + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"} + DEV_USE_DOCKER_WAIT_SCRIPT: 1 + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_eth} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + REDIS_PORT: 6379 + REDIS_HOST: redis + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + CELERY_DEBUG: ${CELERY_DEBUG:-1} + SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-eth-signer:8000} + SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} + TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} + restart: unless-stopped + depends_on: + - evm + - postgres + - redis + # - cic-eth-signer + volumes: + - signer-data:/run/crypto-dev-signer + - contract-config:/tmp/cic/config/:ro + #command: ["/usr/local/bin/cic-eth-taskerd"] + #command: ["sleep", "3600"] + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./start_tasker.sh --aux-all -q cic-eth -vv +