diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c81b3e0..584d01c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,3 +36,8 @@ build-push: rules: - if: $CI_COMMIT_REF_PROTECTED == "true" when: always + +deploy-dev: + stage: deploy + trigger: grassrootseconomics/devops + when: manual diff --git a/apps/cic-eth/.gitlab-ci.yml b/apps/cic-eth/.gitlab-ci.yml index 2fcfde1e..fd609ed9 100644 --- a/apps/cic-eth/.gitlab-ci.yml +++ b/apps/cic-eth/.gitlab-ci.yml @@ -9,8 +9,8 @@ build-test-cic-eth: - cd apps/cic-eth - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . - docker run $MR_IMAGE_TAG sh docker/run_tests.sh - #rules: - #- if: $CI_PIPELINE_SOURCE == "merge_request_event" - # changes: - # - apps/$APP_NAME/**/* - # when: always + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/$APP_NAME/**/* + when: always diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index bbb0fbbe..3331458a 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -13,7 +13,16 @@ ARG EXTRA_PIP_ARGS="" # pip install --index-url https://pypi.org/simple \ # --force-reinstall \ # --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ -# -r requirements.txt +# -r requirements.txt + +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY \ + --extra-index-url $EXTRA_INDEX_URL \ + $EXTRA_PIP_ARGS \ + cic-eth-aux-erc20-demurrage-token~=0.0.2a6 + + COPY *requirements.txt ./ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install --index-url https://pypi.org/simple \ @@ -22,19 +31,21 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ $EXTRA_PIP_ARGS \ -r requirements.txt \ -r services_requirements.txt \ - -r admin_requirements.txt - -COPY . . -RUN python setup.py install - -ENV PYTHONPATH . - + -r admin_requirements.txt + +# always install the latest signer RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install --index-url https://pypi.org/simple \ --extra-index-url $GITLAB_PYTHON_REGISTRY \ --extra-index-url $EXTRA_INDEX_URL \ $EXTRA_PIP_ARGS \ - cic-eth-aux-erc20-demurrage-token~=0.0.2a6 + crypto-dev-signer + +COPY . . +RUN python setup.py install + +ENV PYTHONPATH . + COPY docker/entrypoints/* ./ RUN chmod 755 *.sh diff --git a/apps/cic-eth/docker/entrypoints/start_tasker.sh b/apps/cic-eth/docker/entrypoints/start_tasker.sh index d0984322..14d82b90 100644 --- a/apps/cic-eth/docker/entrypoints/start_tasker.sh +++ b/apps/cic-eth/docker/entrypoints/start_tasker.sh @@ -5,27 +5,27 @@ set -e # set CONFINI_ENV_PREFIX to override the env prefix to override env vars -echo "!!! starting signer" -python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log & +#echo "!!! starting signer" +#python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log & echo "!!! starting taskerd" /usr/local/bin/cic-eth-taskerd $@ # thanks! https://docs.docker.com/config/containers/multi-service_container/ -sleep 1; -echo "!!! entering monitor loop" -while true; do - ps aux | grep crypto-dev-daemon | grep -q -v grep - PROCESS_1_STATUS=$? - ps aux | grep cic-eth-tasker |grep -q -v grep - PROCESS_2_STATUS=$? - # If the greps above find anything, they exit with 0 status - # If they are not both 0, then something is wrong - if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then - echo "One of the processes has already exited." - exit 1 - fi - sleep 15; -done - +#sleep 1; +#echo "!!! entering monitor loop" +#while true; do +# ps aux | grep crypto-dev-daemon | grep -q -v grep +# PROCESS_1_STATUS=$? +# ps aux | grep cic-eth-tasker |grep -q -v grep +# PROCESS_2_STATUS=$? +# # If the greps above find anything, they exit with 0 status +# # If they are not both 0, then something is wrong +# if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then +# echo "One of the processes has already exited." +# exit 1 +# fi +# sleep 15; +#done +# set +e diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 2e7fb0bb..a4b62f1a 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,4 @@ celery==4.4.7 chainlib-eth>=0.0.9a9,<0.1.0 +chainlib-eth>=0.0.9a13,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index 1ecb54a6..4a6a7fee 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,4 +1,4 @@ -crypto-dev-signer>=0.4.15a1,<=0.4.15 +crypto-dev-signer>=0.4.15a4,<=0.4.15 chainqueue>=0.0.5a1,<0.1.0 cic-eth-registry>=0.6.1a2,<0.7.0 redis==3.5.3 diff --git a/apps/cic-meta/.gitlab-ci.yml b/apps/cic-meta/.gitlab-ci.yml index c77d18d7..577efd7f 100644 --- a/apps/cic-meta/.gitlab-ci.yml +++ b/apps/cic-meta/.gitlab-ci.yml @@ -9,8 +9,8 @@ build-test-cic-meta: - cd apps/cic-meta - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . - docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh - #rules: - #- if: $CI_PIPELINE_SOURCE == "merge_request_event" - # changes: - # - apps/$APP_NAME/**/* - # when: always + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - apps/$APP_NAME/**/* + when: always diff --git a/apps/cic-meta/README.md b/apps/cic-meta/README.md new file mode 100644 index 00000000..d1893a3e --- /dev/null +++ b/apps/cic-meta/README.md @@ -0,0 +1 @@ +# CIC-Meta diff --git a/apps/contract-migration/docker/Dockerfile_ci b/apps/contract-migration/docker/Dockerfile_ci deleted file mode 100644 index 4a983449..00000000 --- a/apps/contract-migration/docker/Dockerfile_ci +++ /dev/null @@ -1,44 +0,0 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e - -WORKDIR /root - -RUN touch /etc/apt/sources.list.d/ethereum.list -RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list -RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list -RUN cat /etc/apt/sources.list.d/ethereum.list -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9 - -#RUN apt-get install solc - -RUN mkdir -vp /usr/local/etc/cic - -ENV CONFINI_DIR /usr/local/etc/cic/ - - -COPY config_template/ /usr/local/etc/cic/ -COPY requirements.txt . -COPY override_requirements.txt . - -ARG pip_index_url=https://pypi.org/simple -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG EXTRA_PIP_ARGS="" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -ARG pip_trusted_host=pypi.org -RUN pip install --index-url https://pypi.org/simple \ - pip install --index-url https://pypi.org/simple \ - --pre \ - --force-reinstall \ - --trusted-host $pip_trusted_host \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \ - -r requirements.txt - -RUN pip install --index-url https://pypi.org/simple \ - --force-reinstall \ - --pre \ - --trusted-host $pip_trusted_host \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \ - -r override_requirements.txt - -COPY . . -RUN chmod +x *.sh diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 37bd17c6..b04c6d90 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -9,3 +9,4 @@ erc20-transfer-authorization>=0.3.5a2,<0.4.0 erc20-faucet>=0.3.2a2,<0.4.0 sarafu-faucet>=0.0.7a2,<0.1.0 confini>=0.4.2rc3,<1.0.0 +crypto-dev-signer>=0.4.15a4,<=0.4.15 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index 1c221021..b9cd99d5 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -3,6 +3,8 @@ set -a . ${DEV_DATA_DIR}/env_reset +WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60} + set -e @@ -24,7 +26,7 @@ if [ ! -z "$DEV_USE_DOCKER_WAIT_SCRIPT" ]; then read -i "/" rpc_provider_port <<< "${p[2]}" rpc_provider_host=${p[1]:2} echo "waiting for provider host $rpc_provider_host port $rpc_provider_port..." - ./wait-for-it.sh "$rpc_provider_host:$rpc_provider_port" + ./wait-for-it.sh "$rpc_provider_host:$rpc_provider_port" -t $WAIT_FOR_TIMEOUT fi if [ "$TOKEN_TYPE" == "giftable_erc20_token" ]; then @@ -96,7 +98,7 @@ eth-token-index-add $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_S DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s` >&2 echo "set token faucet amount" -sarafu-faucet-set $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT +sarafu-faucet-set $fee_price_arg -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT >&2 echo "register faucet in registry" eth-contract-registry-set -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS @@ -105,7 +107,10 @@ eth-contract-registry-set -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $CIC_RE giftable-token-minter -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv $DEV_FAUCET_ADDRESS +#echo "export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL" >> ${DEV_DATA_DIR}/env_reset +export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset +confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config set +a set +e diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index 36731720..eb1bfd59 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -82,5 +82,7 @@ cic-eth-ctl -i $CHAIN_SPEC unlock INIT cic-eth-ctl -i $CHAIN_SPEC unlock SEND cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE +confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config + set +a set +e diff --git a/docker-compose.yml b/docker-compose.yml index 47880a32..563e8621 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,7 @@ services: # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest environment: RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} DEV_USE_DOCKER_WAIT_SCRIPT: 1 CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} @@ -101,6 +102,7 @@ services: TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS TOKEN_TYPE: $TOKEN_TYPE + SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} command: ["./run_job.sh"] #command: ["./reset.sh"] depends_on: @@ -148,7 +150,7 @@ services: - /bin/bash - -c - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi ./start_tracker.sh -c /usr/local/etc/cic-cache -vv volumes: - contract-config:/tmp/cic/config/:ro @@ -189,7 +191,7 @@ services: - /bin/bash - -c - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi /usr/local/bin/cic-cache-taskerd -vv volumes: - contract-config:/tmp/cic/config/:ro @@ -223,7 +225,7 @@ services: - /bin/bash - -c - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi "/usr/local/bin/uwsgi" \ --wsgi-file /root/cic_cache/runnable/daemons/server.py \ --http :8000 \ @@ -260,8 +262,64 @@ services: CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} - SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} - SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} + SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} + SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} + SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} + ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} + TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} + CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} + depends_on: + - eth + - postgres + - redis + - cic-eth-signer + deploy: + restart_policy: + condition: on-failure + volumes: + - signer-data:/run/crypto-dev-signer + - contract-config:/tmp/cic/config/:ro + #command: ["/usr/local/bin/cic-eth-taskerd"] + #command: ["sleep", "3600"] + command: + - /bin/bash + - -c + - | + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + ./start_tasker.sh --aux-all -q cic-eth -vv + + cic-eth-signer: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} + build: + context: apps/cic-eth + dockerfile: docker/Dockerfile + target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER + ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_NAME: ${DATABASE_NAME_CIC_ETH:-cic_eth} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + REDIS_PORT: 6379 + REDIS_HOST: redis + PGPASSWORD: ${DATABASE_PASSWORD:-tralala} + CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + CELERY_DEBUG: ${CELERY_DEBUG:-1} + SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000} + SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} @@ -274,14 +332,15 @@ services: restart_policy: condition: on-failure volumes: - - signer-data:/tmp/cic/signer + - signer-data:/run/crypto-dev-signer - contract-config:/tmp/cic/config/:ro - command: - - /bin/bash - - -c - - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tasker.sh --aux-all -q cic-eth -vv + command: ["python", "/usr/local/bin/crypto-dev-daemon", "-c", "/usr/local/etc/crypto-dev-signer", "-vv"] + #command: + # - /bin/bash + # - -c + # - | + # if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + # ./start_tasker.sh --aux-all -q cic-eth -vv # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ] cic-eth-tracker: @@ -323,7 +382,7 @@ services: - /bin/bash - -c - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi #./start_tracker.sh -vv -c /usr/local/etc/cic-eth ./start_tracker.sh -vv # command: "/root/start_manager.sh head -vv" @@ -369,9 +428,10 @@ services: - /bin/bash - -c - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi ./start_dispatcher.sh -q cic-eth -vv # command: "/root/start_dispatcher.sh -q cic-eth -vv" + cic-eth-retrier: @@ -416,7 +476,7 @@ services: - /bin/bash - -c - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi ./start_retry.sh -vv # command: "/root/start_retry.sh -q cic-eth -vv" diff --git a/dump.sh b/dump.sh new file mode 100644 index 00000000..dd02ea32 --- /dev/null +++ b/dump.sh @@ -0,0 +1 @@ +docker run -t -v cic-internal-integration_contract-config:/tmp/cic/config cic-internal-integration_contract-migration cat /tmp/cic/config/env_reset