diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index cea7fd57..db2e537a 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -16,7 +16,6 @@ from cic_eth.api.base import ApiBase from cic_eth.enum import LockEnum app = celery.current_app - #logg = logging.getLogger(__name__) logg = logging.getLogger() @@ -430,13 +429,13 @@ class Api(ApiBase): return t - def balance(self, address, token_symbols, include_pending=True): + def balance(self, address, token_symbol, include_pending=True): """Calls the provided callback with the current token balance of the given address. :param address: Ethereum address of holder :type address: str, 0x-hex - :param token_symbols: ERC20 token symbols of tokens to send - :type token_symbols: list[str] + :param token_symbol: ERC20 token symbol of tokens to send + :type token_symbol: str :param include_pending: If set, will include transactions that have not yet been fully processed :type include_pending: bool :returns: uuid of root task @@ -448,7 +447,7 @@ class Api(ApiBase): s_tokens = celery.signature( 'cic_eth.eth.erc20.resolve_tokens_by_symbol', [ - token_symbols, + [token_symbol], self.chain_spec.asdict(), ], queue=self.queue, diff --git a/apps/cic-eth/cic_eth/graphql/helpers.py b/apps/cic-eth/cic_eth/graphql/helpers.py index f35f9513..3c2e62ef 100644 --- a/apps/cic-eth/cic_eth/graphql/helpers.py +++ b/apps/cic-eth/cic_eth/graphql/helpers.py @@ -1,3 +1,4 @@ +from cic_eth.graphql.schema import schema query_with_argument = """ { @@ -34,7 +35,7 @@ print(m_result) balance_query = """ query { - balance(address:"0xcf41bd087b71c72d748fe2b8b4c88b2367c37df3", tokenSymbols:["GFT", "COFE"]){ + balance(address:"0x643c99d5ab51b5e06d55a68e6b24fd36e9d6b1c9", tokenSymbols:["GFT", "COFE"]){ balanceNetwork balanceIncoming balanceOutgoing @@ -43,8 +44,8 @@ query { } """ -# balance_query_result = schema.execute(balance_query) -# print(balance_query_result) +balance_query_result = schema.execute(balance_query) +print(balance_query_result) transfer_mutation = """ @@ -58,8 +59,8 @@ tokenSymbol: "COFE" ){ } """ -transfer_mutation_result = schema.execute(transfer_mutation) -print(transfer_mutation_result) +# transfer_mutation_result = schema.execute(transfer_mutation) +# print(transfer_mutation_result) balance_query = """ diff --git a/apps/cic-eth/cic_eth/graphql/schema.py b/apps/cic-eth/cic_eth/graphql/schema.py index cec83593..652d9055 100644 --- a/apps/cic-eth/cic_eth/graphql/schema.py +++ b/apps/cic-eth/cic_eth/graphql/schema.py @@ -3,6 +3,7 @@ import json import sys import uuid +import celery import cic_eth.cli import redis from cic_eth.api.api_task import Api @@ -17,7 +18,7 @@ argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -celery_app = cic_eth.cli.CeleryApp.from_config(config) +celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) chain_spec = config.get('CHAIN_SPEC') redis_host = config.get('REDIS_HOST') redis_port = config.get('REDIS_PORT') @@ -43,7 +44,7 @@ def call(method, *args): ps = r.pubsub() ps.subscribe(redis_channel) ps.get_message() # Subscription Object - + print(f"Channel init {redis_channel}") api = init_api(redis_channel) print(args) @@ -91,10 +92,10 @@ class Query(ObjectType): return Token(**data) balance = Field(List(TokenBalance), address=String(), - token_symbols=List(String), include_pending=Boolean()) + token_symbol=String(), include_pending=Boolean()) - def resolve_balance(root, info, address, token_symbols, include_pending=True): - data = call('balance', address, token_symbols, include_pending) + def resolve_balance(root, info, address, token_symbol, include_pending=True): + data = call('balance', address, token_symbol, include_pending) print(data) #[{'address': '3ff776b6f888980def9d4220858803f9dc5e341e', 'converters': [], 'balance_network': 0}] return map(lambda token: TokenBalance( diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 370e9bd7..c9a796d2 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -2,7 +2,6 @@ celery==4.4.7 chainlib-eth>=0.0.10a16,<0.1.0 semver==2.13.0 crypto-dev-signer>=0.4.15rc2,<0.5.0 -uwsgi==2.0.19.1 graphene>=2.0 flask>=2.0.2 Flask-GraphQL>=2.0.1 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d76a27f2..bfc1072f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,8 @@ volumes: postgres-db: {} signer-data: {} bloxberg-data: {} - contract-config: {} + contract-config: {} + services: evm: @@ -23,13 +24,13 @@ services: postgres: image: postgres:12.5-alpine environment: - POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging. + POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging. # PGDATA: /tmp/cic/postgres ports: - ${DEV_POSTGRES_PORT:-63432}:5432 - command: ["-c", "max_connections=200"] + command: [ "-c", "max_connections=200" ] volumes: - - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql + - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql - postgres-db:/var/lib/postgresql/data redis: @@ -38,16 +39,17 @@ services: - ${DEV_REDIS_PORT:-63379}:6379 command: "--loglevel verbose" + bootstrap: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/contract-migration:${TAG:-latest} build: context: apps/contract-migration dockerfile: docker/Dockerfile args: - DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY - PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config} DEV_CONFIG_RESET: $DEV_CONFIG_RESET @@ -85,9 +87,11 @@ services: volumes: - contract-config:/tmp/cic/config + + cic-signer: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/funga-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/funga-eth:${TAG:-latest} + build: context: apps/cic-signer dockerfile: Dockerfile args: @@ -116,8 +120,10 @@ services: # queue handling for outgoing transactions and incoming transactions cic-eth-tasker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + ports: + - 5000:5000 + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -146,9 +152,7 @@ services: SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-signer:8000} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} - restart: unless-stopped - ports: - - 5000:5000 + restart: unless-stopped depends_on: - evm - postgres @@ -164,11 +168,12 @@ services: set -a if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi set +a - ./start_tasker.sh --aux-all -q cic-eth -vv & python /root/cic_eth/graphql/app.py + ./start_tasker.sh --aux-all -q cic-eth -vv + cic-eth-tracker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -197,7 +202,7 @@ services: SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} SYNCER_OFFSET: ${SYNCER_OFFSET:-0} TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -213,9 +218,10 @@ services: set +a ./start_tracker.sh -vv + cic-eth-dispatcher: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -240,7 +246,7 @@ services: CELERY_DEBUG: ${CELERY_DEBUG:-1} CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} DISPATCHER_LOOP_INTERVAL: ${DISPATCHER_LOOP_INTERVAL:-1} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -256,9 +262,10 @@ services: set +a ./start_dispatcher.sh -vv + cic-eth-retrier: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-eth:${TAG:-latest} + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -285,7 +292,7 @@ services: SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-20} RETRY_DELAY: ${RETRY_DELAY:-60} RETRY_BATCH_SIZE: ${RETRY_BATCH_SIZE:-100} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -301,9 +308,11 @@ services: set +a ./start_retry.sh -vv + + cic-cache-tracker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-cache:${TAG:-latest} + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -332,7 +341,7 @@ services: SYNCER_OFFSET: ${SYNCER_OFFSET:-0} SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -348,9 +357,10 @@ services: set +a ./start_tracker.sh -vv + cic-cache-tasker: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-cache:${TAG:-latest} + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -375,7 +385,7 @@ services: CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -390,14 +400,15 @@ services: if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi set +a /usr/local/bin/cic-cache-taskerd -vv - # "/usr/local/bin/uwsgi" \ - # --wsgi-file /root/cic_cache/runnable/daemons/server.py \ - # --http :8000 \ - # --pyargv "-vv" +# "/usr/local/bin/uwsgi" \ +# --wsgi-file /root/cic_cache/runnable/daemons/server.py \ +# --http :8000 \ +# --pyargv "-vv" + cic-cache-server: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-cache:${TAG:-latest} + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -416,7 +427,7 @@ services: DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 SERVER_PORT: 8000 - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_CACHE_SERVER_PORT:-63313}:8000 depends_on: @@ -436,6 +447,7 @@ services: volumes: - contract-config:/tmp/cic/config/:ro + # metadata replacement server for swarm cic-meta-server: @@ -467,7 +479,7 @@ services: PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_META_SERVER_PORT:-63380}:8000 depends_on: @@ -475,6 +487,7 @@ services: volumes: - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp + cic-user-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-user:${TAG:-latest} build: @@ -499,7 +512,7 @@ services: REDIS_HOST: redis PGP_PASSPHRASE: merman CIC_META_URL: ${CIC_META_URL:-http://meta:8000} - restart: unless-stopped + restart: unless-stopped depends_on: - postgres - redis @@ -510,6 +523,7 @@ services: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" + cic-user-server: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-user:${TAG:-latest} build: @@ -530,7 +544,7 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_USER_SERVER_PORT:-63415}:9500 depends_on: @@ -538,6 +552,7 @@ services: - redis command: "/root/start_cic_user_server.sh -vv" + cic-user-ussd-server: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-user:${TAG:-latest} build: @@ -575,6 +590,7 @@ services: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_ussd_server.sh -vv" + cic-notify-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/cic-notify:${TAG:-latest} build: @@ -607,56 +623,57 @@ services: command: "/root/start_tasker.sh -q cic-notify -vv" data-seeding: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/data-seeding:${TAG:-latest} - build: - context: apps/data-seeding - dockerfile: docker/Dockerfile - args: - DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} - PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS - environment: - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - DATABASE_POOL_SIZE: 0 - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} - OUT_DIR: out - NUMBER_OF_USERS: 10 - CONFIG: config - CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} - TOKEN_SYMBOL: GFT - KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c - USSD_HOST: cic-user-ussd-server - USSD_PORT: 9000 - INCLUDE_BALANCES: y - USSD_SSL: n - NOTIFY_DATABASE_NAME: cic_notify - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_DB: 0 - META_HOST: meta - META_PORT: 8000 - META_URL: http://meta:8000 - USSD_PROVIDER: http://cic-user-ussd-server:9000 - CELERY_QUEUE: cic-import-ussd - EXCLUSIONS: ussd - command: - - /bin/bash - - -c - - | - set -a - if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - set +a - ./import_ussd.sh - volumes: - - contract-config:/tmp/cic/config/:ro + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}/data-seeding:${TAG:-latest} + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + OUT_DIR: out + NUMBER_OF_USERS: 10 + CONFIG: config + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + TOKEN_SYMBOL: GFT + #KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c + WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c} + USSD_HOST: cic-user-ussd-server + USSD_PORT: 9000 + INCLUDE_BALANCES: y + USSD_SSL: n + NOTIFY_DATABASE_NAME: cic_notify + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + META_HOST: meta + META_PORT: 8000 + META_URL: http://meta:8000 + USSD_PROVIDER: http://cic-user-ussd-server:9000 + CELERY_QUEUE: cic-import-ussd + EXCLUSIONS: ussd + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./import_ussd.sh + volumes: + - contract-config:/tmp/cic/config/:ro