From 93bcbd7d512da2ddaddfaebc130c09384f176b7e Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Fri, 29 Oct 2021 23:33:46 +0000 Subject: [PATCH 1/8] feat: (automation) add semver --- .gitlab-ci.yml | 44 ++++++++++++++++++++++++++++++++++++++++---- .semverbot.toml | 16 ++++++++++++++++ README.md | 1 + scripts/build.sh | 5 ++++- 4 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 .semverbot.toml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67ccdc9d..f01a4f54 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,7 @@ include: #- local: 'apps/data-seeding/.gitlab-ci.yml' stages: + - version - build - test - deploy @@ -20,9 +21,39 @@ variables: DOCKER_BUILDKIT: "1" COMPOSE_DOCKER_CLI_BUILD: "1" CI_DEBUG_TRACE: "true" + SEMVERBOT_VERSION: "0.2.0" -before_script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + #before_script: + # - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + +version: + #image: python:3.7-stretch + image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae + stage: version + script: + - mkdir -p ~/.ssh && chmod 700 ~/.ssh + - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts + - eval $(ssh-agent -s) + - ssh-add <(echo "$SSH_PRIVATE_KEY") + - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git + - export TAG=$(sbot predict version -m auto) + - | + if [[ -z $TAG ]] + then + echo "tag could not be set $@" + exit 1 + fi + - echo $TAG > version + - git tag -a v$TAG -m "ci tagged" + - git push origin v$TAG + artifacts: + paths: + - version + rules: + - if: $CI_COMMIT_REF_PROTECTED == "true" + when: always + - if: $CI_COMMIT_REF_NAME == "master" + when: always # runs on protected branches and pushes to repo build-push: @@ -30,12 +61,17 @@ build-push: tags: - integration #script: - # - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh + # - TAG=$CI_Cbefore_script: + before_script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY script: - - TAG=latest sh ./scripts/build-push.sh + - TAG=latest ./scripts/build-push.sh + - TAG=$(cat ./version) ./scripts/build-push.sh rules: - if: $CI_COMMIT_REF_PROTECTED == "true" when: always + - if: $CI_COMMIT_REF_NAME == "master" + when: always deploy-dev: stage: deploy diff --git a/.semverbot.toml b/.semverbot.toml new file mode 100644 index 00000000..f24eb80b --- /dev/null +++ b/.semverbot.toml @@ -0,0 +1,16 @@ +[git] + +[git.config] +email = "semverbot@grassroots.org" +name = "semvervot" + +[git.tags] +prefix = "v" + +[semver] +mode = "git-commit" + +[semver.detection] +patch = ["fix", "[fix]", "patch", "[patch]"] +minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"] +major = ["BREAKING CHANGE"] diff --git a/README.md b/README.md index 9e0d62d0..a7966dc9 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ This repo uses docker-compose and docker buildkit. Set the following environment variables to get started: + ``` export COMPOSE_DOCKER_CLI_BUILD=1 export DOCKER_BUILDKIT=1 diff --git a/scripts/build.sh b/scripts/build.sh index b3da3a6a..744e0eb4 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,4 +6,7 @@ set -e TAG=${TAG?Variable not set} \ docker-compose \ -f docker-compose.yml \ -build +build \ +--no-cache \ +--parallel \ +--progress plain From c85692ab9e2d179292ab331e8b5211718fde1d06 Mon Sep 17 00:00:00 2001 From: William Luke Date: Mon, 1 Nov 2021 10:48:05 +0300 Subject: [PATCH 2/8] graphql server --- .envrc | 6 +- apps/cic-eth/cic_eth/graphql/app.py | 24 +++++++ apps/cic-eth/cic_eth/graphql/helpers.py | 87 +++++++++++++++++++++++++ apps/cic-eth/cic_eth/graphql/schema.py | 78 ---------------------- apps/cic-eth/requirements.txt | 4 +- docker-compose.yml | 4 +- 6 files changed, 121 insertions(+), 82 deletions(-) create mode 100644 apps/cic-eth/cic_eth/graphql/app.py create mode 100644 apps/cic-eth/cic_eth/graphql/helpers.py diff --git a/.envrc b/.envrc index 6ad52af5..9daa87cb 100644 --- a/.envrc +++ b/.envrc @@ -1,2 +1,6 @@ export COMPOSE_DOCKER_CLI_BUILD=1 -export DOCKER_BUILDKIT=1 \ No newline at end of file +export DOCKER_BUILDKIT=1 +export FAUCET_AMOUNT=100000000000 +export TOKEN_TYPE=giftable_erc20_token +export TOKEN_NAME=Coffee +export TOKEN_SYMBOL=COFE \ No newline at end of file diff --git a/apps/cic-eth/cic_eth/graphql/app.py b/apps/cic-eth/cic_eth/graphql/app.py new file mode 100644 index 00000000..d669df77 --- /dev/null +++ b/apps/cic-eth/cic_eth/graphql/app.py @@ -0,0 +1,24 @@ +from cic_eth.graphql.schema import schema +from flask import Flask +from flask_graphql import GraphQLView + + +def create_app(): + app = Flask(__name__) + app.add_url_rule( + '/graphql', + view_func=GraphQLView.as_view( + 'graphql', + schema=schema, + graphiql=True + ) + ) + @app.route("/") + def test(): + return "Test ok!" + + return app + +if __name__ == "__main__": + app = create_app() + app.run(host='0.0.0.0', port=5000) diff --git a/apps/cic-eth/cic_eth/graphql/helpers.py b/apps/cic-eth/cic_eth/graphql/helpers.py new file mode 100644 index 00000000..f35f9513 --- /dev/null +++ b/apps/cic-eth/cic_eth/graphql/helpers.py @@ -0,0 +1,87 @@ + +query_with_argument = """ +{ + defaultToken{ + symbol + } +} + +""" +# result = schema.execute(query_with_argument) +# print(result) + + +mutation_with_argument = """ +mutation { + createAccount(password:"test"){ + address + } +} + +""" +m_result = schema.execute(mutation_with_argument) +print(m_result) +mutation_with_argument = """ +mutation { + createAccount(password:"test"){ + address + } +} + +""" +m_result = schema.execute(mutation_with_argument) +print(m_result) + +balance_query = """ +query { + balance(address:"0xcf41bd087b71c72d748fe2b8b4c88b2367c37df3", tokenSymbols:["GFT", "COFE"]){ + balanceNetwork + balanceIncoming + balanceOutgoing + balanceAvailable + } +} + +""" +# balance_query_result = schema.execute(balance_query) +# print(balance_query_result) + + +transfer_mutation = """ +mutation { + transfer(fromAddress :"0xcf41bd087b71c72d748fe2b8b4c88b2367c37df3", +toAddress: "0x63474da1a315f9abe0999b6adb61ae8a9ea19a76" +value: 20 +tokenSymbol: "COFE" ){ + test + } +} + +""" +transfer_mutation_result = schema.execute(transfer_mutation) +print(transfer_mutation_result) + + +balance_query = """ +query { + balance(address:"0x63474da1a315f9abe0999b6adb61ae8a9ea19a76", tokenSymbols:["GFT"]){ + balanceNetwork + balanceIncoming + balanceOutgoing + balanceAvailable + } +} + +""" +# balance_query_result = schema.execute(balance_query) +# print(balance_query_result) + + +transactions_query = """ +query { + transactions(address:"0xcf41bd087b71c72d748fe2b8b4c88b2367c37df3") +} + +""" +# transactions_query_result = schema.execute(transactions_query) +# print(transactions_query_result) diff --git a/apps/cic-eth/cic_eth/graphql/schema.py b/apps/cic-eth/cic_eth/graphql/schema.py index 6ef70270..cec83593 100644 --- a/apps/cic-eth/cic_eth/graphql/schema.py +++ b/apps/cic-eth/cic_eth/graphql/schema.py @@ -186,81 +186,3 @@ class MyMutations(ObjectType): schema = Schema(query=Query, mutation=MyMutations) - -query_with_argument = """ -{ - defaultToken{ - symbol - } -} - -""" -# result = schema.execute(query_with_argument) -# print(result) - - -mutation_with_argument = """ -mutation { - createAccount(password:"test"){ - address - } -} - -""" -# m_result = schema.execute(mutation_with_argument) -# print(m_result) - - -balance_query = """ -query { - balance(address:"0x82e66cf2766bf20672a605bbf5a6faaa12d5b907", tokenSymbols:["GFT"]){ - balanceNetwork - balanceIncoming - balanceOutgoing - balanceAvailable - } -} - -""" -# balance_query_result = schema.execute(balance_query) -# print(balance_query_result) - - -transfer_mutation = """ -mutation { - transfer(fromAddress :"0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C", -toAddress: "0x82e66cf2766bf20672a605bbf5a6faaa12d5b907" -value: 20 -tokenSymbol: "GFT" ){ - - } -} - -""" -transfer_mutation_result = schema.execute(transfer_mutation) -print(transfer_mutation_result) - - -balance_query = """ -query { - balance(address:"0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C", tokenSymbols:["GFT"]){ - balanceNetwork - balanceIncoming - balanceOutgoing - balanceAvailable - } -} - -""" -# balance_query_result = schema.execute(balance_query) -# print(balance_query_result) - - -transactions_query = """ -query { - transactions(address:"0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C") -} - -""" -# transactions_query_result = schema.execute(transactions_query) -# print(transactions_query_result) diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 74239a6c..e3a6ab5e 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -3,4 +3,6 @@ chainlib-eth>=0.0.10a4,<0.1.0 semver==2.13.0 crypto-dev-signer>=0.4.15rc2,<0.5.0 uwsgi==2.0.19.1 -graphene>=2.0 \ No newline at end of file +graphene>=2.0 +flask>=2.0.2 +Flask-GraphQL>=2.0.1 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 92141f12..409562da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -142,7 +142,7 @@ services: TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} restart: unless-stopped ports: - - 8080:8000 + - 5000:5000 depends_on: - evm - postgres @@ -158,7 +158,7 @@ services: set -a if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi set +a - ./start_tasker.sh --aux-all -q cic-eth -vv + ./start_tasker.sh --aux-all -q cic-eth -vv & python /root/cic_eth/graphql/app.py cic-eth-tracker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} From f28ba1098faa69c1eb9776ae9109fe5cd15d45e7 Mon Sep 17 00:00:00 2001 From: William Luke Date: Mon, 1 Nov 2021 13:56:12 +0300 Subject: [PATCH 3/8] revert api_task:balance changes --- apps/cic-eth/cic_eth/api/api_task.py | 9 +- apps/cic-eth/cic_eth/graphql/helpers.py | 11 +- apps/cic-eth/cic_eth/graphql/schema.py | 11 +- apps/cic-eth/requirements.txt | 1 - docker-compose.yml | 205 +++++++++++++----------- 5 files changed, 127 insertions(+), 110 deletions(-) diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index cea7fd57..db2e537a 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -16,7 +16,6 @@ from cic_eth.api.base import ApiBase from cic_eth.enum import LockEnum app = celery.current_app - #logg = logging.getLogger(__name__) logg = logging.getLogger() @@ -430,13 +429,13 @@ class Api(ApiBase): return t - def balance(self, address, token_symbols, include_pending=True): + def balance(self, address, token_symbol, include_pending=True): """Calls the provided callback with the current token balance of the given address. :param address: Ethereum address of holder :type address: str, 0x-hex - :param token_symbols: ERC20 token symbols of tokens to send - :type token_symbols: list[str] + :param token_symbol: ERC20 token symbol of tokens to send + :type token_symbol: str :param include_pending: If set, will include transactions that have not yet been fully processed :type include_pending: bool :returns: uuid of root task @@ -448,7 +447,7 @@ class Api(ApiBase): s_tokens = celery.signature( 'cic_eth.eth.erc20.resolve_tokens_by_symbol', [ - token_symbols, + [token_symbol], self.chain_spec.asdict(), ], queue=self.queue, diff --git a/apps/cic-eth/cic_eth/graphql/helpers.py b/apps/cic-eth/cic_eth/graphql/helpers.py index f35f9513..3c2e62ef 100644 --- a/apps/cic-eth/cic_eth/graphql/helpers.py +++ b/apps/cic-eth/cic_eth/graphql/helpers.py @@ -1,3 +1,4 @@ +from cic_eth.graphql.schema import schema query_with_argument = """ { @@ -34,7 +35,7 @@ print(m_result) balance_query = """ query { - balance(address:"0xcf41bd087b71c72d748fe2b8b4c88b2367c37df3", tokenSymbols:["GFT", "COFE"]){ + balance(address:"0x643c99d5ab51b5e06d55a68e6b24fd36e9d6b1c9", tokenSymbols:["GFT", "COFE"]){ balanceNetwork balanceIncoming balanceOutgoing @@ -43,8 +44,8 @@ query { } """ -# balance_query_result = schema.execute(balance_query) -# print(balance_query_result) +balance_query_result = schema.execute(balance_query) +print(balance_query_result) transfer_mutation = """ @@ -58,8 +59,8 @@ tokenSymbol: "COFE" ){ } """ -transfer_mutation_result = schema.execute(transfer_mutation) -print(transfer_mutation_result) +# transfer_mutation_result = schema.execute(transfer_mutation) +# print(transfer_mutation_result) balance_query = """ diff --git a/apps/cic-eth/cic_eth/graphql/schema.py b/apps/cic-eth/cic_eth/graphql/schema.py index cec83593..652d9055 100644 --- a/apps/cic-eth/cic_eth/graphql/schema.py +++ b/apps/cic-eth/cic_eth/graphql/schema.py @@ -3,6 +3,7 @@ import json import sys import uuid +import celery import cic_eth.cli import redis from cic_eth.api.api_task import Api @@ -17,7 +18,7 @@ argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -celery_app = cic_eth.cli.CeleryApp.from_config(config) +celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) chain_spec = config.get('CHAIN_SPEC') redis_host = config.get('REDIS_HOST') redis_port = config.get('REDIS_PORT') @@ -43,7 +44,7 @@ def call(method, *args): ps = r.pubsub() ps.subscribe(redis_channel) ps.get_message() # Subscription Object - + print(f"Channel init {redis_channel}") api = init_api(redis_channel) print(args) @@ -91,10 +92,10 @@ class Query(ObjectType): return Token(**data) balance = Field(List(TokenBalance), address=String(), - token_symbols=List(String), include_pending=Boolean()) + token_symbol=String(), include_pending=Boolean()) - def resolve_balance(root, info, address, token_symbols, include_pending=True): - data = call('balance', address, token_symbols, include_pending) + def resolve_balance(root, info, address, token_symbol, include_pending=True): + data = call('balance', address, token_symbol, include_pending) print(data) #[{'address': '3ff776b6f888980def9d4220858803f9dc5e341e', 'converters': [], 'balance_network': 0}] return map(lambda token: TokenBalance( diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 370e9bd7..c9a796d2 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -2,7 +2,6 @@ celery==4.4.7 chainlib-eth>=0.0.10a16,<0.1.0 semver==2.13.0 crypto-dev-signer>=0.4.15rc2,<0.5.0 -uwsgi==2.0.19.1 graphene>=2.0 flask>=2.0.2 Flask-GraphQL>=2.0.1 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d76a27f2..bfc1072f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,8 @@ volumes: postgres-db: {} signer-data: {} bloxberg-data: {} - contract-config: {} + contract-config: {} + services: evm: @@ -23,13 +24,13 @@ services: postgres: image: postgres:12.5-alpine environment: - POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging. + POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging. # PGDATA: /tmp/cic/postgres ports: - ${DEV_POSTGRES_PORT:-63432}:5432 - command: ["-c", "max_connections=200"] + command: [ "-c", "max_connections=200" ] volumes: - - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql + - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql - postgres-db:/var/lib/postgresql/data redis: @@ -38,16 +39,17 @@ services: - ${DEV_REDIS_PORT:-63379}:6379 command: "--loglevel verbose" + bootstrap: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/contract-migration:${TAG:-latest} build: context: apps/contract-migration dockerfile: docker/Dockerfile args: - DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY - PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config} DEV_CONFIG_RESET: $DEV_CONFIG_RESET @@ -85,9 +87,11 @@ services: volumes: - contract-config:/tmp/cic/config + + cic-signer: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/funga-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/funga-eth:${TAG:-latest} + build: context: apps/cic-signer dockerfile: Dockerfile args: @@ -116,8 +120,10 @@ services: # queue handling for outgoing transactions and incoming transactions cic-eth-tasker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + ports: + - 5000:5000 + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -146,9 +152,7 @@ services: SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-signer:8000} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} - restart: unless-stopped - ports: - - 5000:5000 + restart: unless-stopped depends_on: - evm - postgres @@ -164,11 +168,12 @@ services: set -a if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi set +a - ./start_tasker.sh --aux-all -q cic-eth -vv & python /root/cic_eth/graphql/app.py + ./start_tasker.sh --aux-all -q cic-eth -vv + cic-eth-tracker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -197,7 +202,7 @@ services: SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} SYNCER_OFFSET: ${SYNCER_OFFSET:-0} TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -213,9 +218,10 @@ services: set +a ./start_tracker.sh -vv + cic-eth-dispatcher: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -240,7 +246,7 @@ services: CELERY_DEBUG: ${CELERY_DEBUG:-1} CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} DISPATCHER_LOOP_INTERVAL: ${DISPATCHER_LOOP_INTERVAL:-1} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -256,9 +262,10 @@ services: set +a ./start_dispatcher.sh -vv + cic-eth-retrier: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -285,7 +292,7 @@ services: SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-20} RETRY_DELAY: ${RETRY_DELAY:-60} RETRY_BATCH_SIZE: ${RETRY_BATCH_SIZE:-100} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -301,9 +308,11 @@ services: set +a ./start_retry.sh -vv + + cic-cache-tracker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-cache:${TAG:-latest} + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -332,7 +341,7 @@ services: SYNCER_OFFSET: ${SYNCER_OFFSET:-0} SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -348,9 +357,10 @@ services: set +a ./start_tracker.sh -vv + cic-cache-tasker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-cache:${TAG:-latest} + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -375,7 +385,7 @@ services: CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -390,14 +400,15 @@ services: if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi set +a /usr/local/bin/cic-cache-taskerd -vv - # "/usr/local/bin/uwsgi" \ - # --wsgi-file /root/cic_cache/runnable/daemons/server.py \ - # --http :8000 \ - # --pyargv "-vv" +# "/usr/local/bin/uwsgi" \ +# --wsgi-file /root/cic_cache/runnable/daemons/server.py \ +# --http :8000 \ +# --pyargv "-vv" + cic-cache-server: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-cache:${TAG:-latest} + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -416,7 +427,7 @@ services: DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 SERVER_PORT: 8000 - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_CACHE_SERVER_PORT:-63313}:8000 depends_on: @@ -436,6 +447,7 @@ services: volumes: - contract-config:/tmp/cic/config/:ro + # metadata replacement server for swarm cic-meta-server: @@ -467,7 +479,7 @@ services: PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_META_SERVER_PORT:-63380}:8000 depends_on: @@ -475,6 +487,7 @@ services: volumes: - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp + cic-user-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-user:${TAG:-latest} build: @@ -499,7 +512,7 @@ services: REDIS_HOST: redis PGP_PASSPHRASE: merman CIC_META_URL: ${CIC_META_URL:-http://meta:8000} - restart: unless-stopped + restart: unless-stopped depends_on: - postgres - redis @@ -510,6 +523,7 @@ services: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" + cic-user-server: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-user:${TAG:-latest} build: @@ -530,7 +544,7 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_USER_SERVER_PORT:-63415}:9500 depends_on: @@ -538,6 +552,7 @@ services: - redis command: "/root/start_cic_user_server.sh -vv" + cic-user-ussd-server: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-user:${TAG:-latest} build: @@ -575,6 +590,7 @@ services: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_ussd_server.sh -vv" + cic-notify-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-notify:${TAG:-latest} build: @@ -607,56 +623,57 @@ services: command: "/root/start_tasker.sh -q cic-notify -vv" data-seeding: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/data-seeding:${TAG:-latest} - build: - context: apps/data-seeding - dockerfile: docker/Dockerfile - args: - DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} - PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS - environment: - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - DATABASE_POOL_SIZE: 0 - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} - OUT_DIR: out - NUMBER_OF_USERS: 10 - CONFIG: config - CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} - TOKEN_SYMBOL: GFT - KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c - USSD_HOST: cic-user-ussd-server - USSD_PORT: 9000 - INCLUDE_BALANCES: y - USSD_SSL: n - NOTIFY_DATABASE_NAME: cic_notify - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_DB: 0 - META_HOST: meta - META_PORT: 8000 - META_URL: http://meta:8000 - USSD_PROVIDER: http://cic-user-ussd-server:9000 - CELERY_QUEUE: cic-import-ussd - EXCLUSIONS: ussd - command: - - /bin/bash - - -c - - | - set -a - if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - set +a - ./import_ussd.sh - volumes: - - contract-config:/tmp/cic/config/:ro + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/data-seeding:${TAG:-latest} + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + OUT_DIR: out + NUMBER_OF_USERS: 10 + CONFIG: config + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + TOKEN_SYMBOL: GFT + #KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c + WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c} + USSD_HOST: cic-user-ussd-server + USSD_PORT: 9000 + INCLUDE_BALANCES: y + USSD_SSL: n + NOTIFY_DATABASE_NAME: cic_notify + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + META_HOST: meta + META_PORT: 8000 + META_URL: http://meta:8000 + USSD_PROVIDER: http://cic-user-ussd-server:9000 + CELERY_QUEUE: cic-import-ussd + EXCLUSIONS: ussd + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./import_ussd.sh + volumes: + - contract-config:/tmp/cic/config/:ro From f26510848501fdaedb5e4ed922c672a5d52cdde4 Mon Sep 17 00:00:00 2001 From: William Luke Date: Tue, 2 Nov 2021 18:50:14 +0300 Subject: [PATCH 4/8] add eth-server --- apps/cic-eth/cic_eth/api/api_task.py | 13 +-- apps/cic-eth/cic_eth/graphql/app.py | 23 +++-- apps/cic-eth/cic_eth/graphql/config.py | 10 ++ apps/cic-eth/cic_eth/graphql/schema.py | 96 +++++++++++-------- .../cic_eth/runnable/daemons/server.py | 2 +- apps/cic-eth/requirements.txt | 1 + docker-compose.yml | 49 +++++++++- test.sh | 49 ++++++++++ 8 files changed, 187 insertions(+), 56 deletions(-) create mode 100644 apps/cic-eth/cic_eth/graphql/config.py create mode 100644 test.sh diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index db2e537a..fa12ecc6 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -9,13 +9,14 @@ import logging # external imports import celery from chainlib.chain import ChainSpec -from hexathon import strip_0x - # local imports from cic_eth.api.base import ApiBase from cic_eth.enum import LockEnum +from hexathon import strip_0x app = celery.current_app +print(app.backend) + #logg = logging.getLogger(__name__) logg = logging.getLogger() @@ -488,9 +489,9 @@ class Api(ApiBase): s_balance_incoming.link(s_balance_outgoing) last_in_chain = s_balance_outgoing - one = celery.chain(s_tokens, s_balance) - two = celery.chain(s_tokens, s_balance_incoming) - three = celery.chain(s_tokens, s_balance_outgoing) + one = s_tokens | s_balance + two = s_tokens | s_balance_incoming + three = s_tokens | s_balance_outgoing t = None if self.callback_param != None: @@ -500,7 +501,7 @@ class Api(ApiBase): t = celery.chord([one, two, three])(s_result) else: # TODO: Chord is inefficient with only one chain, but assemble_balances must be able to handle different structures in order to avoid chord - one = celery.chain(s_tokens, s_balance) + one = s_tokens | s_balance if self.callback_param != None: s_result.link(self.callback_success).on_error(self.callback_error) t = celery.chord([one])(s_result) diff --git a/apps/cic-eth/cic_eth/graphql/app.py b/apps/cic-eth/cic_eth/graphql/app.py index d669df77..6b5f9a04 100644 --- a/apps/cic-eth/cic_eth/graphql/app.py +++ b/apps/cic-eth/cic_eth/graphql/app.py @@ -1,11 +1,18 @@ -from cic_eth.graphql.schema import schema +import os + +import cic_eth.cli +from cic_eth.graphql.config import config from flask import Flask from flask_graphql import GraphQLView +from cic_eth.graphql.schema import schema - +app = cic_eth.cli.CeleryApp.from_config(config) +app.set_default() +print(app) def create_app(): - app = Flask(__name__) - app.add_url_rule( + flask_app = Flask(__name__) + + flask_app.add_url_rule( '/graphql', view_func=GraphQLView.as_view( 'graphql', @@ -13,12 +20,12 @@ def create_app(): graphiql=True ) ) - @app.route("/") + @flask_app.route("/") def test(): return "Test ok!" - return app + return flask_app if __name__ == "__main__": - app = create_app() - app.run(host='0.0.0.0', port=5000) + flask_app = create_app() + flask_app.run(host='0.0.0.0', port=5000, debug=True) diff --git a/apps/cic-eth/cic_eth/graphql/config.py b/apps/cic-eth/cic_eth/graphql/config.py new file mode 100644 index 00000000..ace3630d --- /dev/null +++ b/apps/cic-eth/cic_eth/graphql/config.py @@ -0,0 +1,10 @@ +import cic_eth.cli + +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) + +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) + diff --git a/apps/cic-eth/cic_eth/graphql/schema.py b/apps/cic-eth/cic_eth/graphql/schema.py index 652d9055..dc25a5ff 100644 --- a/apps/cic-eth/cic_eth/graphql/schema.py +++ b/apps/cic-eth/cic_eth/graphql/schema.py @@ -1,41 +1,26 @@ # standard imports import json +import logging import sys import uuid -import celery -import cic_eth.cli import redis from cic_eth.api.api_task import Api +from cic_eth.graphql.config import config from graphene import (Boolean, Field, Float, Int, List, Mutation, ObjectType, Schema, String) -arg_flags = cic_eth.cli.argflag_std_base -local_arg_flags = cic_eth.cli.argflag_local_taskcallback -argparser = cic_eth.cli.ArgumentParser(arg_flags) - -argparser.process_local_flags(local_arg_flags) -args = argparser.parse_args() -config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) - -celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) chain_spec = config.get('CHAIN_SPEC') redis_host = config.get('REDIS_HOST') redis_port = config.get('REDIS_PORT') redis_db = config.get('REDIS_DB') celery_queue = config.get('CELERY_QUEUE') +logging.basicConfig(level=logging.DEBUG) + +log = logging.getLogger(__name__) + -def init_api(redis_channel: str): - api = Api( - chain_spec, - queue=celery_queue, - callback_param='{}:{}:{}:{}'.format( - redis_host, redis_port, redis_db, redis_channel), - callback_task='cic_eth.callbacks.redis.redis', - callback_queue=celery_queue, - ) - return api def call(method, *args): @@ -45,9 +30,15 @@ def call(method, *args): ps.subscribe(redis_channel) ps.get_message() # Subscription Object print(f"Channel init {redis_channel}") - api = init_api(redis_channel) print(args) - + api = Api( + chain_spec, + queue=celery_queue, + callback_param='{}:{}:{}:{}'.format( + redis_host, redis_port, redis_db, redis_channel), + callback_task='cic_eth.callbacks.redis.redis', + callback_queue=celery_queue, + ) getattr(api, method)(*args) ps.get_message() # returns None !? @@ -58,6 +49,7 @@ def call(method, *args): 'got no new address from cic-eth before timeout: {}\n'.format(e)) sys.exit(1) ps.unsubscribe() + print(o) m = json.loads(o['data']) return m["result"] @@ -82,12 +74,19 @@ class TokenBalance(ObjectType): return str(int(parent.balance_network) + int(parent.balance_incoming) - int(parent.balance_outgoing)) + class Query(ObjectType): default_token = Field(Token) def resolve_default_token(parent, info): + # always pass an object for `me` field - data = call('default_token') + api = Api( + chain_spec, + queue=celery_queue, + ) + task = api.default_token() + data = task.get() print(data) return Token(**data) @@ -95,8 +94,14 @@ class Query(ObjectType): token_symbol=String(), include_pending=Boolean()) def resolve_balance(root, info, address, token_symbol, include_pending=True): - data = call('balance', address, token_symbol, include_pending) - print(data) + api = Api( + chain_spec, + queue=celery_queue, + ) + task = api.balance(address=address, token_symbol=token_symbol, include_pending=include_pending) + data = task.get() #api call('balance', address, token_symbol, include_pending) + log.debug(data) + print(data, file=sys.stdout) #[{'address': '3ff776b6f888980def9d4220858803f9dc5e341e', 'converters': [], 'balance_network': 0}] return map(lambda token: TokenBalance( address=token.get("address"), @@ -111,7 +116,13 @@ class Query(ObjectType): required=True), limit=Int(default_value=10)) def resolve_transactions(root, info, address, limit): - data = call('list', address, limit) + + api = Api( + chain_spec, + queue=celery_queue, + ) + task = api.list(address=address, limit=limit) + data = task.get() print(data) #[{'address': '3ff776b6f888980def9d4220858803f9dc5e341e', 'converters': [], 'balance_network': 0}] return "test" @@ -126,8 +137,12 @@ class CreateAccount(Mutation): def mutate(root, info, password, register=True): print(password, register) - - address = call('create_account', password, register) + api = Api( + chain_spec, + queue=celery_queue, + ) + + address = call('create_account',password, register) return CreateAccount(address=f"0x{address}") @@ -145,10 +160,10 @@ class Transfer(Mutation): value, token_symbol): print(from_address, to_address, value, token_symbol) - - redis_channel = str(uuid.uuid4()) - - api = init_api(redis_channel) + api = Api( + chain_spec, + queue=celery_queue, + ) t = api.transfer(from_address, to_address, int(value * (10**6)), token_symbol) print(f"t {t}") @@ -172,12 +187,15 @@ class TransferFrom(Mutation): to_address, value, token_symbol, spender_address): - - data = call('transfer_from', from_address, - to_address, - value, - token_symbol, spender_address) - return data + api = Api( + chain_spec, + queue=celery_queue, + ) + task = api.transfer_from(from_address=from_address, + to_address=to_address, + value=value, + token_symbol=token_symbol, spender_address=spender_address) + return task.get() class MyMutations(ObjectType): diff --git a/apps/cic-eth/cic_eth/runnable/daemons/server.py b/apps/cic-eth/cic_eth/runnable/daemons/server.py index 52494882..e8d6ccfe 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/server.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/server.py @@ -8,7 +8,6 @@ from urllib.parse import parse_qsl, urlparse import cic_eth.cli import redis -from cic_eth.api.api_task import Api logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -22,6 +21,7 @@ args = argparser.parse_args() config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) celery_app = cic_eth.cli.CeleryApp.from_config(config) +from cic_eth.api.api_task import Api # uwsgi application diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index c9a796d2..370e9bd7 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -2,6 +2,7 @@ celery==4.4.7 chainlib-eth>=0.0.10a16,<0.1.0 semver==2.13.0 crypto-dev-signer>=0.4.15rc2,<0.5.0 +uwsgi==2.0.19.1 graphene>=2.0 flask>=2.0.2 Flask-GraphQL>=2.0.1 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index bfc1072f..a18f95c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,8 +121,6 @@ services: # queue handling for outgoing transactions and incoming transactions cic-eth-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} - ports: - - 5000:5000 build: context: apps/cic-eth dockerfile: docker/Dockerfile @@ -170,6 +168,53 @@ services: set +a ./start_tasker.sh --aux-all -q cic-eth -vv + cic-eth-server: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + ports: + - 5000:5000 + build: + context: apps/cic-eth + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_eth} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + REDIS_PORT: 6379 + REDIS_HOST: redis + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + CELERY_DEBUG: ${CELERY_DEBUG:-1} + SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-signer:8000} + SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} + TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} + restart: unless-stopped + depends_on: + - cic-eth-tasker + volumes: + - signer-data:/run/crypto-dev-signer + - contract-config:/tmp/cic/config/:ro + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + python /root/cic_eth/graphql/app.py -vv cic-eth-tracker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} diff --git a/test.sh b/test.sh new file mode 100644 index 00000000..5035f28a --- /dev/null +++ b/test.sh @@ -0,0 +1,49 @@ +# 0000 +# 0001 +# 0010 +# 0011 +# 0100 +# 0101 +# 0111 +# 1000 +# 1001 +# 1010 +# 1011 +# 1100 +# 1101 +# 1111 + +RUN_MASK=31 +LAST_BIT_POS=5 + +RUN_MASK_HIGHEST=0 + +for ((i=$LAST_BIT_POS; i>0; i--)); do + b=$((2**$((i-1)))) + if [ $((b & $RUN_MASK)) -gt 0 ]; then + RUN_MASK_HIGHEST=$i + break + fi +done + +echo $RUN_MASK_HIGHEST + + + +bit=1 +for ((i=0; i<$LAST_BIT_POS; i++)); do + runlevel="RUNLEVEL $bit" + if [[ $((RUN_MASK & $bit)) -eq ${bit} ]]; then + s="$runlevel - ${description[$i]}" + >&2 echo -e "\033[;96mRUNNING $s\033[;39m" + # source $((i+1))_${files[$i]}.sh + if [ $? -ne "0" ]; then + >&2 echo -e "\033[;31mFAILED $s\033[;39m" + exit 1; + fi + >&2 echo -e "\033[;32mSUCCEEDED $s\033[;39m" + >&2 echo -e "\033[;96mConfiguration state after $runlevel execution\033[;39m" + # confini-dump --schema-dir ./config + fi + bit=$((bit*2)) +done From 09dfdbb38ae4c98d7dd4717a5d72015c7c3392ed Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Tue, 2 Nov 2021 17:45:09 +0000 Subject: [PATCH 5/8] bug: False 404 in cic-meta logging --- apps/cic-meta/scripts/server/server.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/cic-meta/scripts/server/server.ts b/apps/cic-meta/scripts/server/server.ts index 66fde62a..29aa923e 100755 --- a/apps/cic-meta/scripts/server/server.ts +++ b/apps/cic-meta/scripts/server/server.ts @@ -193,6 +193,7 @@ async function processRequest(req, res) { res.end(); return; } + content = ''; break; case 'get:automerge:client': @@ -251,7 +252,7 @@ async function processRequest(req, res) { } if (content === undefined) { - console.error('empty content', data); + console.error('empty content', mod, digest, data); res.writeHead(404, {"Content-Type": "text/plain"}); res.end(); return; From a42efb2529bb3a188121d1a4294d83cab2324f99 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 8 Nov 2021 10:43:02 +0100 Subject: [PATCH 6/8] Non-checksum address in erc20 tx_caches --- apps/cic-eth/cic_eth/eth/erc20.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/apps/cic-eth/cic_eth/eth/erc20.py b/apps/cic-eth/cic_eth/eth/erc20.py index 4fbe2e57..40e2e0e8 100644 --- a/apps/cic-eth/cic_eth/eth/erc20.py +++ b/apps/cic-eth/cic_eth/eth/erc20.py @@ -382,6 +382,8 @@ def cache_transfer_data( sender_address = tx_normalize.wallet_address(tx['from']) recipient_address = tx_normalize.wallet_address(tx_data[0]) token_value = tx_data[1] + source_token_address = tx_normalize.executable_address(tx['to']) + destination_token_address = source_token_address session = SessionBase.create_session() @@ -389,8 +391,8 @@ def cache_transfer_data( 'hash': tx_hash_hex, 'from': sender_address, 'to': recipient_address, - 'source_token': tx['to'], - 'destination_token': tx['to'], + 'source_token': source_token_address, + 'destination_token': destination_token_address, 'from_value': token_value, 'to_value': token_value, } @@ -422,14 +424,16 @@ def cache_transfer_from_data( spender_address = tx_data[0] recipient_address = tx_data[1] token_value = tx_data[2] + source_token_address = tx_normalize.executable_address(tx['to']) + destination_token_address = source_token_address session = SessionBase.create_session() tx_dict = { 'hash': tx_hash_hex, 'from': tx['from'], 'to': recipient_address, - 'source_token': tx['to'], - 'destination_token': tx['to'], + 'source_token': source_token_address, + 'destination_token': destination_token_address, 'from_value': token_value, 'to_value': token_value, } @@ -461,14 +465,16 @@ def cache_approve_data( sender_address = tx_normalize.wallet_address(tx['from']) recipient_address = tx_normalize.wallet_address(tx_data[0]) token_value = tx_data[1] + source_token_address = tx_normalize.executable_address(tx['to']) + destination_token_address = source_token_address session = SessionBase.create_session() tx_dict = { 'hash': tx_hash_hex, 'from': sender_address, 'to': recipient_address, - 'source_token': tx['to'], - 'destination_token': tx['to'], + 'source_token': source_token_address, + 'destination_token': destination_token_address, 'from_value': token_value, 'to_value': token_value, } From 8600e282b31e59023def30275c61262a4e3bfca2 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 8 Nov 2021 10:56:11 +0100 Subject: [PATCH 7/8] Remove incoming balance in network check --- apps/cic-eth/cic_eth/queue/balance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cic-eth/cic_eth/queue/balance.py b/apps/cic-eth/cic_eth/queue/balance.py index f2f87fb5..8c48ab91 100644 --- a/apps/cic-eth/cic_eth/queue/balance.py +++ b/apps/cic-eth/cic_eth/queue/balance.py @@ -72,7 +72,7 @@ def __balance_incoming_compatible(token_address, receiver_address): status_compare = dead() q = q.filter(Otx.status.op('&')(status_compare)==0) # TODO: this can change the result for the recipient if tx is later obsoleted and resubmission is delayed. - q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK) + #q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK) q = q.filter(TxCache.destination_token_address==token_address) delta = 0 for r in q.all(): From b31433dcb5a7b6d7fd958743cce330b85c274991 Mon Sep 17 00:00:00 2001 From: William Luke Date: Tue, 9 Nov 2021 10:26:04 +0300 Subject: [PATCH 8/8] add database debug --- .envrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index 9daa87cb..e88b1bce 100644 --- a/.envrc +++ b/.envrc @@ -3,4 +3,5 @@ export DOCKER_BUILDKIT=1 export FAUCET_AMOUNT=100000000000 export TOKEN_TYPE=giftable_erc20_token export TOKEN_NAME=Coffee -export TOKEN_SYMBOL=COFE \ No newline at end of file +export TOKEN_SYMBOL=COFE +export DATABASE_DEBUG=1 \ No newline at end of file