kamikazechaser
/
cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(cic-eth): initial server setup

This commit is contained in:
William Luke 2021-10-25 17:51:45 +03:00
parent d8f51c5bdd
commit 4a1008e75e
5 changed files with 185 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/cic-eth/cic_eth/runnable/create.py
View File

@ -1,15 +1,11 @@
#!/usr/bin/python
import sys
import os
import logging
import uuid
import json
import argparse
# external imports
import redis
from xdg.BaseDirectory import xdg_config_home
from chainlib.chain import ChainSpec
# local imports
import cic_eth.cli

99
apps/cic-eth/cic_eth/runnable/daemons/server.py Normal file
View File

@ -0,0 +1,99 @@
# standard imports
import json
import logging
import os
import sys
import uuid
from urllib.parse import parse_qsl, urlparse
import cic_eth.cli
import redis
from cic_eth.api.api_task import Api
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
arg_flags = cic_eth.cli.argflag_std_base
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
celery_app = cic_eth.cli.CeleryApp.from_config(config)
# uwsgi application
def application(env, start_response):
parsed_url = urlparse(env['REQUEST_URI']) # /api
path = parsed_url.path
params = dict(parse_qsl(parsed_url.query))
request_method = env['REQUEST_METHOD']
chain_spec = config.get('CHAIN_SPEC')
redis_host = config.get('REDIS_HOST')
redis_port = config.get('REDIS_PORT')
redis_db = config.get('REDIS_DB')
celery_queue = config.get('CELERY_QUEUE')
redis_channel = str(uuid.uuid4())
r = redis.Redis(redis_host, redis_port, redis_db)
ps = r.pubsub()
ps.subscribe(redis_channel)
ps.get_message() # Subscription Object
api = Api(
chain_spec,
queue=celery_queue,
callback_param='{}:{}:{}:{}'.format(
redis_host, redis_port, redis_db, redis_channel),
callback_task='cic_eth.callbacks.redis.redis',
callback_queue=celery_queue,
)
if path == '/list':
address = params.get('address')
print('address', address)
# address, limit=10, external_task=None, external_queue=None
api.list(address)
elif path == '/balance':
api.balance(**params)
elif path == '/create_account':
api.create_account(**params)
elif path == '/ping':
api.ping(**params)
elif path == '/transfer':
api.transfer(**params)
elif path == '/transfer_from':
api.transfer_from(**params)
elif path == '/token':
api.token(**params)
elif path == '/tokens':
api.tokens(**params)
elif path == '/default_token':
api.default_token()
ps.get_message() # returns None !?
try:
o = ps.get_message(timeout=config.get('REDIS_TIMEOUT'))
except TimeoutError as e:
sys.stderr.write(
'got no new address from cic-eth before timeout: {}\n'.format(e))
sys.exit(1)
ps.unsubscribe()
m = json.loads(o['data'])
print(m['result'])
data = {"path": path,
"request_method": request_method, "result": m}
json_data = json.dumps(data)
content = json_data.encode('utf-8')
headers = []
headers.append(('Content-Length', str(len(content))),)
headers.append(('Access-Control-Allow-Origin', '*',))
headers.append(('Content-Type', 'application/json',))
start_response('200 OK', headers)
return [content]

4
apps/cic-eth/docker/Dockerfile
View File

@ -15,8 +15,6 @@ RUN apt-get install libffi-dev
RUN pip install --index-url $PIP_INDEX_URL \
--pre \
--force-reinstall \
--no-cache \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
cic-eth-aux-erc20-demurrage-token~=0.0.2a7
@ -24,8 +22,6 @@ RUN pip install --index-url $PIP_INDEX_URL \
COPY *requirements.txt ./
RUN pip install --index-url $PIP_INDEX_URL \
--pre \
--force-reinstall \
--no-cache \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt \
-r services_requirements.txt \

1
apps/cic-eth/requirements.txt
View File

@ -2,3 +2,4 @@ celery==4.4.7
chainlib-eth>=0.0.10a4,<0.1.0
semver==2.13.0
crypto-dev-signer>=0.4.15rc2,<0.5.0
uwsgi==2.0.19.1

184
docker-compose.yml
View File

@ -4,8 +4,7 @@ volumes:
postgres-db: {}
signer-data: {}
bloxberg-data: {}
contract-config: {}
contract-config: {}
services:
evm:
@ -24,13 +23,13 @@ services:
postgres:
image: postgres:12.5-alpine
environment:
POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging.
POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging.
# PGDATA: /tmp/cic/postgres
ports:
- ${DEV_POSTGRES_PORT:-63432}:5432
command: [ "-c", "max_connections=200" ]
command: ["-c", "max_connections=200"]
volumes:
- ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql
- ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql
- postgres-db:/var/lib/postgresql/data
redis:
@ -39,17 +38,16 @@ services:
- ${DEV_REDIS_PORT:-63379}:6379
command: "--loglevel verbose"
bootstrap:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/contract-migration:${TAG:-latest}
build:
context: apps/contract-migration
dockerfile: docker/Dockerfile
args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}
DEV_CONFIG_RESET: $DEV_CONFIG_RESET
@ -81,11 +79,9 @@ services:
volumes:
- contract-config:/tmp/cic/config
cic-signer:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/funga-eth:${TAG:-latest}
build:
build:
context: apps/cic-signer
dockerfile: Dockerfile
args:
@ -115,7 +111,7 @@ services:
# queue handling for outgoing transactions and incoming transactions
cic-eth-tasker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest}
build:
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
args:
@ -144,7 +140,9 @@ services:
SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-signer:8000}
SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
restart: unless-stopped
restart: unless-stopped
ports:
- 8080:8000
depends_on:
- evm
- postgres
@ -162,10 +160,9 @@ services:
set +a
./start_tasker.sh --aux-all -q cic-eth -vv
cic-eth-tracker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest}
build:
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
args:
@ -194,7 +191,7 @@ services:
SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1}
SYNCER_OFFSET: ${SYNCER_OFFSET:-0}
TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
restart: unless-stopped
restart: unless-stopped
depends_on:
- evm
- postgres
@ -210,10 +207,9 @@ services:
set +a
./start_tracker.sh -vv
cic-eth-dispatcher:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest}
build:
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
args:
@ -238,7 +234,7 @@ services:
CELERY_DEBUG: ${CELERY_DEBUG:-1}
CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth}
DISPATCHER_LOOP_INTERVAL: ${DISPATCHER_LOOP_INTERVAL:-1}
restart: unless-stopped
restart: unless-stopped
depends_on:
- evm
- postgres
@ -254,10 +250,9 @@ services:
set +a
./start_dispatcher.sh -vv
cic-eth-retrier:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest}
build:
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
args:
@ -284,7 +279,7 @@ services:
SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-20}
RETRY_DELAY: ${RETRY_DELAY:-60}
RETRY_BATCH_SIZE: ${RETRY_BATCH_SIZE:-100}
restart: unless-stopped
restart: unless-stopped
depends_on:
- evm
- postgres
@ -300,11 +295,9 @@ services:
set +a
./start_retry.sh -vv
cic-cache-tracker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest}
build:
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
args:
@ -333,7 +326,7 @@ services:
SYNCER_OFFSET: ${SYNCER_OFFSET:-0}
SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1}
TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
restart: unless-stopped
restart: unless-stopped
depends_on:
- evm
- postgres
@ -349,10 +342,9 @@ services:
set +a
./start_tracker.sh -vv
cic-cache-tasker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest}
build:
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
args:
@ -377,7 +369,7 @@ services:
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
CELERY_DEBUG: ${CELERY_DEBUG:-1}
CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth}
restart: unless-stopped
restart: unless-stopped
depends_on:
- evm
- postgres
@ -392,15 +384,14 @@ services:
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
set +a
/usr/local/bin/cic-cache-taskerd -vv
# "/usr/local/bin/uwsgi" \
# --wsgi-file /root/cic_cache/runnable/daemons/server.py \
# --http :8000 \
# --pyargv "-vv"
# "/usr/local/bin/uwsgi" \
# --wsgi-file /root/cic_cache/runnable/daemons/server.py \
# --http :8000 \
# --pyargv "-vv"
cic-cache-server:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest}
build:
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
args:
@ -419,7 +410,7 @@ services:
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
SERVER_PORT: 8000
restart: on-failure
restart: on-failure
ports:
- ${DEV_CIC_CACHE_SERVER_PORT:-63313}:8000
depends_on:
@ -439,7 +430,6 @@ services:
volumes:
- contract-config:/tmp/cic/config/:ro
# metadata replacement server for swarm
cic-meta-server:
@ -470,7 +460,7 @@ services:
PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc
PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc
SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql
restart: on-failure
restart: on-failure
ports:
- ${DEV_CIC_META_SERVER_PORT:-63380}:8000
depends_on:
@ -478,7 +468,6 @@ services:
volumes:
- ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp
cic-user-tasker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest}
build:
@ -503,7 +492,7 @@ services:
REDIS_HOST: redis
PGP_PASSPHRASE: merman
CIC_META_URL: ${CIC_META_URL:-http://meta:8000}
restart: unless-stopped
restart: unless-stopped
depends_on:
- postgres
- redis
@ -514,7 +503,6 @@ services:
- ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/
command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv"
cic-user-server:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest}
build:
@ -535,7 +523,7 @@ services:
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
restart: on-failure
restart: on-failure
ports:
- ${DEV_CIC_USER_SERVER_PORT:-63415}:9500
depends_on:
@ -543,7 +531,6 @@ services:
- redis
command: "/root/start_cic_user_server.sh -vv"
cic-user-ussd-server:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest}
build:
@ -581,7 +568,6 @@ services:
- ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/
command: "/root/start_cic_user_ussd_server.sh -vv"
cic-notify-tasker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-notify:${TAG:-latest}
build:
@ -614,56 +600,56 @@ services:
command: "/root/start_tasker.sh -q cic-notify -vv"
data-seeding:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/data-seeding:${TAG:-latest}
build:
context: apps/data-seeding
dockerfile: docker/Dockerfile
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DATABASE_NAME:-cic_ussd}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
OUT_DIR: out
NUMBER_OF_USERS: 10
CONFIG: config
CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg}
TOKEN_SYMBOL: GFT
KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
USSD_HOST: cic-user-ussd-server
USSD_PORT: 9000
INCLUDE_BALANCES: y
USSD_SSL: n
NOTIFY_DATABASE_NAME: cic_notify
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_DB: 0
META_HOST: meta
META_PORT: 8000
META_URL: http://meta:8000
USSD_PROVIDER: http://cic-user-ussd-server:9000
CELERY_QUEUE: cic-import-ussd
EXCLUSIONS: ussd
command:
- /bin/bash
- -c
- |
set -a
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
set +a
./import_ussd.sh
volumes:
- contract-config:/tmp/cic/config/:ro
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/data-seeding:${TAG:-latest}
build:
context: apps/data-seeding
dockerfile: docker/Dockerfile
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DATABASE_NAME:-cic_ussd}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
OUT_DIR: out
NUMBER_OF_USERS: 10
CONFIG: config
CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg}
TOKEN_SYMBOL: GFT
KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
USSD_HOST: cic-user-ussd-server
USSD_PORT: 9000
INCLUDE_BALANCES: y
USSD_SSL: n
NOTIFY_DATABASE_NAME: cic_notify
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_DB: 0
META_HOST: meta
META_PORT: 8000
META_URL: http://meta:8000
USSD_PROVIDER: http://cic-user-ussd-server:9000
CELERY_QUEUE: cic-import-ussd
EXCLUSIONS: ussd
command:
- /bin/bash
- -c
- |
set -a
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
set +a
./import_ussd.sh
volumes:
- contract-config:/tmp/cic/config/:ro