From 4a1008e75e567934a1c79808f663cf758f03725b Mon Sep 17 00:00:00 2001 From: William Luke Date: Mon, 25 Oct 2021 17:51:45 +0300 Subject: [PATCH] feat(cic-eth): initial server setup --- apps/cic-eth/cic_eth/runnable/create.py | 4 - .../cic_eth/runnable/daemons/server.py | 99 ++++++++++ apps/cic-eth/docker/Dockerfile | 4 - apps/cic-eth/requirements.txt | 1 + docker-compose.yml | 184 ++++++++---------- 5 files changed, 185 insertions(+), 107 deletions(-) create mode 100644 apps/cic-eth/cic_eth/runnable/daemons/server.py diff --git a/apps/cic-eth/cic_eth/runnable/create.py b/apps/cic-eth/cic_eth/runnable/create.py index 31dd1991..dfe6921b 100644 --- a/apps/cic-eth/cic_eth/runnable/create.py +++ b/apps/cic-eth/cic_eth/runnable/create.py @@ -1,15 +1,11 @@ #!/usr/bin/python import sys -import os import logging import uuid import json -import argparse # external imports import redis -from xdg.BaseDirectory import xdg_config_home -from chainlib.chain import ChainSpec # local imports import cic_eth.cli diff --git a/apps/cic-eth/cic_eth/runnable/daemons/server.py b/apps/cic-eth/cic_eth/runnable/daemons/server.py new file mode 100644 index 00000000..52494882 --- /dev/null +++ b/apps/cic-eth/cic_eth/runnable/daemons/server.py @@ -0,0 +1,99 @@ +# standard imports +import json +import logging +import os +import sys +import uuid +from urllib.parse import parse_qsl, urlparse + +import cic_eth.cli +import redis +from cic_eth.api.api_task import Api + +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() + +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) + +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) + +celery_app = cic_eth.cli.CeleryApp.from_config(config) + +# uwsgi application + + +def application(env, start_response): + parsed_url = urlparse(env['REQUEST_URI']) # /api + path = parsed_url.path + params = dict(parse_qsl(parsed_url.query)) + request_method = env['REQUEST_METHOD'] + chain_spec = config.get('CHAIN_SPEC') + redis_host = config.get('REDIS_HOST') + redis_port = config.get('REDIS_PORT') + redis_db = config.get('REDIS_DB') + celery_queue = config.get('CELERY_QUEUE') + + redis_channel = str(uuid.uuid4()) + + r = redis.Redis(redis_host, redis_port, redis_db) + + ps = r.pubsub() + ps.subscribe(redis_channel) + ps.get_message() # Subscription Object + + api = Api( + chain_spec, + queue=celery_queue, + callback_param='{}:{}:{}:{}'.format( + redis_host, redis_port, redis_db, redis_channel), + callback_task='cic_eth.callbacks.redis.redis', + callback_queue=celery_queue, + ) + if path == '/list': + address = params.get('address') + print('address', address) + # address, limit=10, external_task=None, external_queue=None + api.list(address) + elif path == '/balance': + api.balance(**params) + elif path == '/create_account': + api.create_account(**params) + elif path == '/ping': + api.ping(**params) + elif path == '/transfer': + api.transfer(**params) + elif path == '/transfer_from': + api.transfer_from(**params) + elif path == '/token': + api.token(**params) + elif path == '/tokens': + api.tokens(**params) + elif path == '/default_token': + api.default_token() + + ps.get_message() # returns None !? + try: + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) + except TimeoutError as e: + sys.stderr.write( + 'got no new address from cic-eth before timeout: {}\n'.format(e)) + sys.exit(1) + ps.unsubscribe() + m = json.loads(o['data']) + print(m['result']) + + data = {"path": path, + "request_method": request_method, "result": m} + json_data = json.dumps(data) + content = json_data.encode('utf-8') + headers = [] + headers.append(('Content-Length', str(len(content))),) + headers.append(('Access-Control-Allow-Origin', '*',)) + headers.append(('Content-Type', 'application/json',)) + start_response('200 OK', headers) + + return [content] diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 59c8d65e..0dea21f4 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -15,8 +15,6 @@ RUN apt-get install libffi-dev RUN pip install --index-url $PIP_INDEX_URL \ --pre \ - --force-reinstall \ - --no-cache \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ cic-eth-aux-erc20-demurrage-token~=0.0.2a7 @@ -24,8 +22,6 @@ RUN pip install --index-url $PIP_INDEX_URL \ COPY *requirements.txt ./ RUN pip install --index-url $PIP_INDEX_URL \ --pre \ - --force-reinstall \ - --no-cache \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt \ -r services_requirements.txt \ diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index bfb64a1d..823e02e1 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -2,3 +2,4 @@ celery==4.4.7 chainlib-eth>=0.0.10a4,<0.1.0 semver==2.13.0 crypto-dev-signer>=0.4.15rc2,<0.5.0 +uwsgi==2.0.19.1 diff --git a/docker-compose.yml b/docker-compose.yml index 39a257cd..92141f12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,7 @@ volumes: postgres-db: {} signer-data: {} bloxberg-data: {} - contract-config: {} - + contract-config: {} services: evm: @@ -24,13 +23,13 @@ services: postgres: image: postgres:12.5-alpine environment: - POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging. + POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging. # PGDATA: /tmp/cic/postgres ports: - ${DEV_POSTGRES_PORT:-63432}:5432 - command: [ "-c", "max_connections=200" ] + command: ["-c", "max_connections=200"] volumes: - - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql + - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql - postgres-db:/var/lib/postgresql/data redis: @@ -39,17 +38,16 @@ services: - ${DEV_REDIS_PORT:-63379}:6379 command: "--loglevel verbose" - bootstrap: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/contract-migration:${TAG:-latest} build: context: apps/contract-migration dockerfile: docker/Dockerfile args: - DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY - PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config} DEV_CONFIG_RESET: $DEV_CONFIG_RESET @@ -81,11 +79,9 @@ services: volumes: - contract-config:/tmp/cic/config - - cic-signer: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/funga-eth:${TAG:-latest} - build: + build: context: apps/cic-signer dockerfile: Dockerfile args: @@ -115,7 +111,7 @@ services: # queue handling for outgoing transactions and incoming transactions cic-eth-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -144,7 +140,9 @@ services: SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-signer:8000} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} - restart: unless-stopped + restart: unless-stopped + ports: + - 8080:8000 depends_on: - evm - postgres @@ -162,10 +160,9 @@ services: set +a ./start_tasker.sh --aux-all -q cic-eth -vv - cic-eth-tracker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -194,7 +191,7 @@ services: SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} SYNCER_OFFSET: ${SYNCER_OFFSET:-0} TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -210,10 +207,9 @@ services: set +a ./start_tracker.sh -vv - cic-eth-dispatcher: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -238,7 +234,7 @@ services: CELERY_DEBUG: ${CELERY_DEBUG:-1} CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} DISPATCHER_LOOP_INTERVAL: ${DISPATCHER_LOOP_INTERVAL:-1} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -254,10 +250,9 @@ services: set +a ./start_dispatcher.sh -vv - cic-eth-retrier: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} - build: + build: context: apps/cic-eth dockerfile: docker/Dockerfile args: @@ -284,7 +279,7 @@ services: SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-20} RETRY_DELAY: ${RETRY_DELAY:-60} RETRY_BATCH_SIZE: ${RETRY_BATCH_SIZE:-100} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -300,11 +295,9 @@ services: set +a ./start_retry.sh -vv - - cic-cache-tracker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -333,7 +326,7 @@ services: SYNCER_OFFSET: ${SYNCER_OFFSET:-0} SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -349,10 +342,9 @@ services: set +a ./start_tracker.sh -vv - cic-cache-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -377,7 +369,7 @@ services: CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} - restart: unless-stopped + restart: unless-stopped depends_on: - evm - postgres @@ -392,15 +384,14 @@ services: if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi set +a /usr/local/bin/cic-cache-taskerd -vv -# "/usr/local/bin/uwsgi" \ -# --wsgi-file /root/cic_cache/runnable/daemons/server.py \ -# --http :8000 \ -# --pyargv "-vv" + # "/usr/local/bin/uwsgi" \ + # --wsgi-file /root/cic_cache/runnable/daemons/server.py \ + # --http :8000 \ + # --pyargv "-vv" - cic-cache-server: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} - build: + build: context: apps/cic-cache dockerfile: docker/Dockerfile args: @@ -419,7 +410,7 @@ services: DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 SERVER_PORT: 8000 - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_CACHE_SERVER_PORT:-63313}:8000 depends_on: @@ -439,7 +430,6 @@ services: volumes: - contract-config:/tmp/cic/config/:ro - # metadata replacement server for swarm cic-meta-server: @@ -470,7 +460,7 @@ services: PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_META_SERVER_PORT:-63380}:8000 depends_on: @@ -478,7 +468,6 @@ services: volumes: - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp - cic-user-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest} build: @@ -503,7 +492,7 @@ services: REDIS_HOST: redis PGP_PASSPHRASE: merman CIC_META_URL: ${CIC_META_URL:-http://meta:8000} - restart: unless-stopped + restart: unless-stopped depends_on: - postgres - redis @@ -514,7 +503,6 @@ services: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" - cic-user-server: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest} build: @@ -535,7 +523,7 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 - restart: on-failure + restart: on-failure ports: - ${DEV_CIC_USER_SERVER_PORT:-63415}:9500 depends_on: @@ -543,7 +531,6 @@ services: - redis command: "/root/start_cic_user_server.sh -vv" - cic-user-ussd-server: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest} build: @@ -581,7 +568,6 @@ services: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_ussd_server.sh -vv" - cic-notify-tasker: image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-notify:${TAG:-latest} build: @@ -614,56 +600,56 @@ services: command: "/root/start_tasker.sh -q cic-notify -vv" data-seeding: - image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/data-seeding:${TAG:-latest} - build: - context: apps/data-seeding - dockerfile: docker/Dockerfile - args: - DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} - PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS - environment: - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - DATABASE_POOL_SIZE: 0 - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} - OUT_DIR: out - NUMBER_OF_USERS: 10 - CONFIG: config - CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} - TOKEN_SYMBOL: GFT - KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c - USSD_HOST: cic-user-ussd-server - USSD_PORT: 9000 - INCLUDE_BALANCES: y - USSD_SSL: n - NOTIFY_DATABASE_NAME: cic_notify - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_DB: 0 - META_HOST: meta - META_PORT: 8000 - META_URL: http://meta:8000 - USSD_PROVIDER: http://cic-user-ussd-server:9000 - CELERY_QUEUE: cic-import-ussd - EXCLUSIONS: ussd - command: - - /bin/bash - - -c - - | - set -a - if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - set +a - ./import_ussd.sh - volumes: - - contract-config:/tmp/cic/config/:ro \ No newline at end of file + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/data-seeding:${TAG:-latest} + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + OUT_DIR: out + NUMBER_OF_USERS: 10 + CONFIG: config + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + TOKEN_SYMBOL: GFT + KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c + USSD_HOST: cic-user-ussd-server + USSD_PORT: 9000 + INCLUDE_BALANCES: y + USSD_SSL: n + NOTIFY_DATABASE_NAME: cic_notify + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + META_HOST: meta + META_PORT: 8000 + META_URL: http://meta:8000 + USSD_PROVIDER: http://cic-user-ussd-server:9000 + CELERY_QUEUE: cic-import-ussd + EXCLUSIONS: ussd + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./import_ussd.sh + volumes: + - contract-config:/tmp/cic/config/:ro