diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index fa12ecc6..cce9caa7 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -15,7 +15,6 @@ from cic_eth.enum import LockEnum from hexathon import strip_0x app = celery.current_app -print(app.backend) #logg = logging.getLogger(__name__) logg = logging.getLogger() diff --git a/apps/cic-eth/cic_eth/runnable/daemons/server.py b/apps/cic-eth/cic_eth/runnable/daemons/server.py index e0e7b6ae..59198c73 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/server.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/server.py @@ -1,107 +1,40 @@ # standard imports -import json import logging -from os import path -from urllib.parse import parse_qsl, urlparse +from urllib.parse import urlparse -from cic_eth.server.celery_helper import call -from cic_eth.server.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest -from openapi_core import create_spec -from openapi_core.validation.request.validators import RequestValidator -from openapi_spec_validator.schemas import read_yaml_file +import redis +from cic_eth.server import args, cache, config, openapi, routes -spec_dict = read_yaml_file(path.join(path.dirname( - __file__), '../../server/openapi/server.yaml')) -spec = create_spec(spec_dict) +# define log levels +if args.vv: + logging.getLogger().setLevel(logging.DEBUG) +elif args.v: + logging.getLogger().setLevel(logging.INFO) +log = logging.getLogger(__name__) -logging.basicConfig(level=logging.WARNING) -log = logging.getLogger() +# TODO Censor relevant things +log.debug(f"{config}") -# TODO Implement wei conversions +# define universal redis cache access +cache.Cache.store = redis.StrictRedis(host=config.get('REDIS_HOST'), + port=config.get('REDIS_PORT'), + db=config.get('REDIS_DB'), + decode_responses=True) # uwsgi application def application(env, start_response): # Validate incoming request against the open api spec - oAPIRequest = UWSGIOpenAPIRequest(env) - validator = RequestValidator(spec) - result = validator.validate(oAPIRequest) - - if result.errors: - json_data = json.dumps(list(map(lambda e: str(e), result.errors))) - content = json_data.encode('utf-8') - headers = [] - headers.append(('Content-Length', str(len(content))),) - headers.append(('Access-Control-Allow-Origin', '*',)) - headers.append(('Content-Type', 'application/json',)) - start_response('400 Invalid Request', headers) - return [content] - + errors, request = openapi.validate.request(env, start_response) + if errors: + return errors parsed_url = urlparse(env.get('REQUEST_URI')) path = parsed_url.path - params = dict(parse_qsl(parsed_url.query)) - - if path == '/transactions': - address = params.pop('address') - # address, limit=10 - data = call('list', address, **params) - - elif path == '/balance': - address = params.pop('address') - token_symbol = params.pop('token_symbol') - data = call('balance', address, token_symbol, **params) - for b in data: - b.update({ - "balance_available": int(b['balance_network']) + int(b['balance_incoming']) - int(b['balance_outgoing']) - }) - elif path == '/create_account': - data = call('create_account', **params) - - elif path == '/refill_gas': - address = params.pop('address') - data = call('refill_gas', address) - - elif path == '/ping': - data = call('ping', **params) - - elif path == '/transfer': - from_address = params.pop('from_address') - to_address = params.pop('to_address') - value = params.pop('value') - token_symbol = params.pop('token_symbol') - - data = call('transfer', from_address, to_address, value, token_symbol) - - elif path == '/transfer_from': - from_address = params.pop('from_address') - to_address = params.pop('to_address') - value = params.pop('value') - token_symbol = params.pop('token_symbol') - spender_address = params.pop('spender_address') - data = call('transfer_from', from_address, to_address, - value, token_symbol, spender_address) - - elif path == '/token': - token_symbol = params.pop('token_symbol') - data = call('token', token_symbol, **params) - - elif path == '/tokens': - token_symbols = params.pop('token_symbols') - data = call('tokens', token_symbols, **params) - - elif path == '/default_token': - data = call('default_token') - else: - start_response('404 This is not the path you\'re looking for', []) - return [] - json_data = json.dumps(data) - content = json_data.encode('utf-8') - headers = [] - headers.append(('Content-Length', str(len(content))),) - headers.append(('Access-Control-Allow-Origin', '*',)) - headers.append(('Content-Type', 'application/json',)) - start_response('200 OK', headers) - - return [content] + query = dict(request.parameters.query) + handler = routes.get(path) + if handler: + return handler(start_response, query) + start_response('404 This is not the path you\'re looking for', []) + return [] diff --git a/apps/cic-eth/cic_eth/server/__init__.py b/apps/cic-eth/cic_eth/server/__init__.py index e69de29b..a567f984 100644 --- a/apps/cic-eth/cic_eth/server/__init__.py +++ b/apps/cic-eth/cic_eth/server/__init__.py @@ -0,0 +1,5 @@ +from . import converters +from . import cache +from . import uwsgi +from .routes import routes +from .config import config, args \ No newline at end of file diff --git a/apps/cic-eth/cic_eth/server/cache.py b/apps/cic-eth/cic_eth/server/cache.py new file mode 100644 index 00000000..d27859ea --- /dev/null +++ b/apps/cic-eth/cic_eth/server/cache.py @@ -0,0 +1,121 @@ +# standard imports +import hashlib +import json +import logging +from typing import Optional, Union + +# external imports +from cic_types.condiments import MetadataPointer +from redis import Redis + +logg = logging.getLogger(__file__) + + +class Cache: + store: Redis = None + +def get_token_data(token_symbol: str): + """ + :param token_symbol: + :type token_symbol: + :return: + :rtype: + """ + identifier = [token_symbol.encode('utf-8')] + key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA) + logg.debug(f'Retrieving token data for: {token_symbol} at: {key}') + token_data_str = get_cached_data(key=key) + if(token_data_str is None): + logg.debug(f'No token data found for: {token_symbol}') + return None + else: + token_data = json.loads(token_data_str) + logg.debug(f'Retrieved token data: {token_data}') + return token_data + + +def set_token_data(token_symbol: str, data: dict): + """ + :param token_symbol: + :type token_symbol: + :return: + :rtype: + """ + identifier = [token_symbol.encode('utf-8')] + key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA) + cache_data(key, json.dumps(data)) + logg.debug(f'Cached token data for: {token_symbol} at: {key}') + +def get_default_token() -> Optional[str]: + """This function attempts to retrieve the default token's data from the redis cache. + :param chain_str: chain name and network id. + :type chain_str: str + :return: + :rtype: + """ + logg.debug(f'Retrieving default token from cache for chain:') + key = cache_data_key(identifier="ff".encode('utf-8'), + salt=MetadataPointer.TOKEN_DEFAULT) + token_data = json.loads(get_cached_data(key=key)) + logg.debug(f'Retrieved default token data: {token_data}') + return token_data + + +def set_default_token(default_token: dict): + """ + :param default_token: + :type default_token: + :return: + :rtype: + """ + logg.debug(f'Setting default token in cache') + key = cache_data_key(identifier="ff".encode('utf-8'), + salt=MetadataPointer.TOKEN_DEFAULT) + cache_data(key, json.dumps(default_token)) + + + + +def cache_data(key: str, data: str): + """ + :param key: + :type key: + :param data: + :type data: + :return: + :rtype: + """ + cache = Cache.store + cache.set(name=key, value=data) + cache.persist(name=key) + logg.debug(f'caching: {data} with key: {key}.') + + +def get_cached_data(key: str): + """ + :param key: + :type key: + :return: + :rtype: + """ + cache = Cache.store + return cache.get(name=key) + + +def cache_data_key(identifier: Union[list, bytes], salt: MetadataPointer): + """ + :param identifier: + :type identifier: + :param salt: + :type salt: + :return: + :rtype: + """ + hash_object = hashlib.new("sha256") + if isinstance(identifier, list): + for identity in identifier: + hash_object.update(identity) + else: + hash_object.update(identifier) + hash_object.update(salt.value.encode(encoding="utf-8")) + return hash_object.digest().hex() diff --git a/apps/cic-eth/cic_eth/server/celery_helper.py b/apps/cic-eth/cic_eth/server/celery.py similarity index 83% rename from apps/cic-eth/cic_eth/server/celery_helper.py rename to apps/cic-eth/cic_eth/server/celery.py index 1a2519bc..847f032b 100644 --- a/apps/cic-eth/cic_eth/server/celery_helper.py +++ b/apps/cic-eth/cic_eth/server/celery.py @@ -9,8 +9,6 @@ from cic_eth.api.api_task import Api from cic_eth.server.config import config log = logging.getLogger(__name__) -# TODO Remove -log.setLevel(logging.DEBUG) celery_app = cic_eth.cli.CeleryApp.from_config(config) celery_app.set_default() @@ -45,11 +43,11 @@ def call(method, *args, **kwargs): o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) except TimeoutError as e: sys.stderr.write( - f"cic_eth.api.{method}({', '.join(args)}, {', '.join(f'{key}={value}' for key, value in kwargs.items())}) timed out:\n {e}") + f"cic_eth.api.{method}({args}, {kwargs}) timed out:\n {e}") sys.exit(1) log.debug( - f"cic_eth.api.{method}({', '.join(args)}, {', '.join(f'{key}={value}' for key, value in kwargs.items())})\n {o}") + f"cic_eth.api.{method}(args={args}, kwargs={kwargs})\n {o}") ps.unsubscribe() try: diff --git a/apps/cic-eth/cic_eth/server/converters.py b/apps/cic-eth/cic_eth/server/converters.py new file mode 100644 index 00000000..8f0113f9 --- /dev/null +++ b/apps/cic-eth/cic_eth/server/converters.py @@ -0,0 +1,39 @@ +# Stolen from ussd + +from math import trunc + +def from_wei(decimals: int, value: int) -> float: + """This function converts values in Wei to a token in the cic network. + :param decimals: The decimals required for wei values. + :type decimals: int + :param value: Value in Wei + :type value: int + :return: SRF equivalent of value in Wei + :rtype: float + """ + value = float(value) / (10**decimals) + return truncate(value=value, decimals=2) + + +def to_wei(decimals: int, value: int) -> int: + """This functions converts values from a token in the cic network to Wei. + :param decimals: The decimals required for wei values. + :type decimals: int + :param value: Value in SRF + :type value: int + :return: Wei equivalent of value in SRF + :rtype: int + """ + return int(value * (10**decimals)) + +def truncate(value: float, decimals: int) -> float: + """This function truncates a value to a specified number of decimals places. + :param value: The value to be truncated. + :type value: float + :param decimals: The number of decimals for the value to be truncated to + :type decimals: int + :return: The truncated value. + :rtype: int + """ + stepper = 10.0**decimals + return trunc(stepper*value) / stepper diff --git a/apps/cic-eth/cic_eth/server/UWSGIOpenAPIRequest.py b/apps/cic-eth/cic_eth/server/openapi/UWSGIOpenAPIRequest.py similarity index 78% rename from apps/cic-eth/cic_eth/server/UWSGIOpenAPIRequest.py rename to apps/cic-eth/cic_eth/server/openapi/UWSGIOpenAPIRequest.py index 0161c8a6..4d8c492f 100644 --- a/apps/cic-eth/cic_eth/server/UWSGIOpenAPIRequest.py +++ b/apps/cic-eth/cic_eth/server/openapi/UWSGIOpenAPIRequest.py @@ -1,18 +1,14 @@ """OpenAPI core contrib requests requests module""" -from urllib.parse import parse_qs -from urllib.parse import urlparse +from urllib.parse import parse_qs, urlparse +from openapi_core.validation.request.datatypes import (OpenAPIRequest, + RequestParameters) from requests import Request -from werkzeug.datastructures import Headers -from werkzeug.datastructures import ImmutableMultiDict - -from openapi_core.validation.request.datatypes import OpenAPIRequest -from openapi_core.validation.request.datatypes import RequestParameters +from werkzeug.datastructures import Headers, ImmutableMultiDict from werkzeug.wrappers import Request - -class UWSGIOpenAPIRequestFactory: +class OpenAPIRequestFactory: @classmethod def create(cls, env): """ @@ -21,11 +17,9 @@ class UWSGIOpenAPIRequestFactory: """ request = Request(env) - # Method method = request.method.lower() - # Preparing a request formats the URL with params, strip them out again o = urlparse(request.url) params = parse_qs(o.query) @@ -59,4 +53,5 @@ class UWSGIOpenAPIRequestFactory: mimetype=mimetype, ) -UWSGIOpenAPIRequest = UWSGIOpenAPIRequestFactory.create \ No newline at end of file + +UWSGIOpenAPIRequest = OpenAPIRequestFactory.create diff --git a/apps/cic-eth/cic_eth/server/openapi/__init__.py b/apps/cic-eth/cic_eth/server/openapi/__init__.py new file mode 100644 index 00000000..d5b8da38 --- /dev/null +++ b/apps/cic-eth/cic_eth/server/openapi/__init__.py @@ -0,0 +1,2 @@ +from . import validate +from . import UWSGIOpenAPIRequest \ No newline at end of file diff --git a/apps/cic-eth/cic_eth/server/openapi/validate.py b/apps/cic-eth/cic_eth/server/openapi/validate.py new file mode 100644 index 00000000..8c6ba4f1 --- /dev/null +++ b/apps/cic-eth/cic_eth/server/openapi/validate.py @@ -0,0 +1,31 @@ +import json +from os import path +from typing import Tuple, Type, Union, List + +from openapi_core.validation.request.datatypes import OpenAPIRequest + +from cic_eth.server.openapi.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest +from openapi_core import create_spec +from openapi_core.validation.request.validators import RequestValidator +from openapi_spec_validator.schemas import read_yaml_file + +spec_dict = read_yaml_file(path.join(path.dirname( + __file__), './server.yaml')) +spec = create_spec(spec_dict) + + +def request(env, start_response) -> Tuple[Union[List[bytes], None], Type[OpenAPIRequest]]: + oAPIRequest = UWSGIOpenAPIRequest(env) + validator = RequestValidator(spec) + result = validator.validate(oAPIRequest) + + if result.errors: + json_data = json.dumps(list(map(lambda e: str(e), result.errors))) + content = json_data.encode('utf-8') + headers = [] + headers.append(('Content-Length', str(len(content))),) + headers.append(('Access-Control-Allow-Origin', '*',)) + headers.append(('Content-Type', 'application/json',)) + start_response('400 Invalid Request', headers) + return ([content], oAPIRequest) + return (None, oAPIRequest) diff --git a/apps/cic-eth/cic_eth/server/routes.py b/apps/cic-eth/cic_eth/server/routes.py new file mode 100644 index 00000000..b130f31f --- /dev/null +++ b/apps/cic-eth/cic_eth/server/routes.py @@ -0,0 +1,131 @@ +""" +Contains all handlers for valid routes for the server +""" + +# standard imports +import logging +from typing import Any, Callable, Dict, List, Type, Union + +from cic_eth.server import cache, celery, converters, uwsgi +from cic_eth.server.openapi.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest + +log = logging.getLogger(__name__) + + +def handle_transactions(start_response, query: dict) -> List[bytes]: + address = query.pop('address') + data = celery.call('list', address, **query) + return uwsgi.respond(start_response, data) + + +def handle_balance(start_response, query: dict) -> List[bytes]: + address = query.pop('address') + token_symbol = query.pop('token_symbol') + log.info(f"token_symbol: {token_symbol}") + log.info(f"query: {query}") + data = celery.call('balance', address, token_symbol, + **query) + for b in data: + token_data = get_token_data(token_symbol) + b['balance_network'] = converters.from_wei(token_data.get( + 'decimals'), int(b['balance_network'])) + b['balance_incoming'] = converters.from_wei(token_data.get( + 'decimals'), int(b['balance_incoming'])) + b['balance_outgoing'] = converters.from_wei(token_data.get( + 'decimals'), int(b['balance_outgoing'])) + + b.update({ + "balance_available": int(b['balance_network']) + int(b['balance_incoming']) - int(b['balance_outgoing']) + }) + return uwsgi.respond(start_response, data) + + +def handle_create_account(start_response, query: dict) -> List[bytes]: + data = celery.call('create_account', **query) + return uwsgi.respond(start_response, data) + + +def handle_refill_gas(start_response, query: dict) -> List[bytes]: + address = query.pop('address') + data = celery.call('refill_gas', address) + return uwsgi.respond(start_response, data) + + +def handle_ping(start_response, query: dict) -> List[bytes]: + data = celery.call('ping', **query) + return uwsgi.respond(start_response, data) + + +def handle_transfer(start_response, query: dict) -> List[bytes]: + from_address = query.pop('from_address') + to_address = query.pop('to_address') + value = query.pop('value') + token_symbol = query.pop('token_symbol') + wei_value = converters.to_wei(get_token_data( + token_symbol).get('decimals'), int(value)) + data = celery.call('transfer', from_address, + to_address, wei_value, token_symbol) + return uwsgi.respond(start_response, data) + + +def handle_transfer_from(start_response, query: dict) -> List[bytes]: + from_address = query.pop('from_address') + to_address = query.pop('to_address') + value = query.pop('value') + token_symbol = query.pop('token_symbol') + spender_address = query.pop('spender_address') + wei_value = converters.to_wei(get_token_data( + token_symbol).get('decimals'), int(value)) + data = celery.call('transfer_from', from_address, to_address, + wei_value, token_symbol, spender_address) + return uwsgi.respond(start_response, data) + + +def handle_token(start_response, query: dict) -> List[bytes]: + token_symbol = query.pop('token_symbol') + data = cache.get_token_data(token_symbol) + if data == None: + data = celery.call('token', token_symbol, **query) + cache.set_token_data(token_symbol, data) + return uwsgi.respond(start_response, data) + + +def handle_tokens(start_response, query: dict) -> List[bytes]: + token_symbols = query.pop('token_symbols') + data = celery.call('tokens', token_symbols, **query) + return uwsgi.respond(start_response, data) + + +def handle_default_token(start_response, _query) -> List[bytes]: + data = cache.get_default_token() + if data == None: + data = celery.call('default_token') + cache.set_default_token(data) + cache.set_token_data(data.get('token_symbol'), data) + return uwsgi.respond(start_response, data) + + +def get_token_data(token_symbol: str) -> List[bytes]: + data = cache.get_token_data(token_symbol) + log.debug(f"cached token data: {data}") + if data == None: + data = celery.call('token', token_symbol) + log.debug( + f"No token data setting token data for: {token_symbol} to {data}") + cache.set_token_data(token_symbol, data) + # TODO What is the second item in the list + return data[0] + + +routes: Dict[str, Callable[[Any, Union[None, dict]], List[bytes]]] = { + "/transactions": handle_transactions, + "/balance": handle_balance, + "/create_account": handle_create_account, + "/refill_gas": handle_refill_gas, + "/ping": handle_ping, + "/transfer": handle_transfer, + "/transfer_from": handle_transfer_from, + "/token": handle_token, + "/tokens": handle_tokens, + "/default_token": handle_default_token, +} diff --git a/apps/cic-eth/cic_eth/server/uwsgi.py b/apps/cic-eth/cic_eth/server/uwsgi.py new file mode 100644 index 00000000..92e1e5a9 --- /dev/null +++ b/apps/cic-eth/cic_eth/server/uwsgi.py @@ -0,0 +1,18 @@ +# standard imports +import json + +DEFAULT_HEADERS = [ + ('Access-Control-Allow-Origin', '*'), + ('Content-Type', 'application/json'), +] + + +def respond(start_response, data): + json_data = json.dumps(data) + content = json_data.encode('utf-8') + headers = [ + *DEFAULT_HEADERS, + ('Content-Type', 'application/json',) + ] + start_response('200 OK', headers) + return [content] diff --git a/apps/cic-eth/config/docker/redis.ini b/apps/cic-eth/config/docker/redis.ini new file mode 100644 index 00000000..d9f54d0b --- /dev/null +++ b/apps/cic-eth/config/docker/redis.ini @@ -0,0 +1,5 @@ +[redis] +host=redis +database=0 +password= +port=6379 diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index dca25af6..192e172b 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,4 @@ +cic-types~=0.2.1a7 celery==4.4.7 chainlib-eth>=0.0.10a16,<0.1.0 semver==2.13.0 @@ -6,4 +7,4 @@ uwsgi==2.0.19.1 setuptools >= 21.0.0 openapi-core >= 0.14.2 openapi-spec-validator >= 0.3.1 -Werkzeug>=2.0.2 \ No newline at end of file +Werkzeug>=2.0.2 diff --git a/apps/cic-eth/tests/test_server.py b/apps/cic-eth/tests/test_server.py index 0b675000..20f34da3 100644 --- a/apps/cic-eth/tests/test_server.py +++ b/apps/cic-eth/tests/test_server.py @@ -64,9 +64,9 @@ def test_account(testserver): assert (balance[0] == { "address": default_token.get('address').lower(), - "balance_available": 30000000000, + "balance_available": 30000, "balance_incoming": 0, - "balance_network": 30000000000, + "balance_network": 30000, "balance_outgoing": 0, "converters": [] }) @@ -95,10 +95,10 @@ def test_account(testserver): balance_after_transfer = response.json() assert (balance_after_transfer[0] == { "address": default_token.get('address').lower(), - "balance_available": 29900000000, + "balance_available": 29900, "balance_incoming": 0, - "balance_network": 30000000000, - "balance_outgoing": 100000000, + "balance_network": 30000, + "balance_outgoing": 100, "converters": [] }) @@ -111,5 +111,9 @@ def test_account(testserver): testserver.url + '/transactions', params=params) transactions = response.json() - log.debug(transactions) - exit(1) + ## TODO: What are the other 2 transactions + assert len(transactions) == 3 + ## Check the transaction is correct + # TODO wtf is READSEND (Ready to send? Or already sent) + assert transactions[0].status == 'READYSEND' + exit(1) # Forcing it to fail to i get logs out of pytest diff --git a/docker-compose.yml b/docker-compose.yml index 20a669a8..c58a44f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -210,6 +210,7 @@ services: - cic-eth-tracker - cic-eth-dispatcher volumes: + # - ./apps/cic-eth/:/root Useful for developing locally - signer-data:/run/crypto-dev-signer - contract-config:/tmp/cic/config/:ro command: