feat: add wei conversions and caching

apps/cic-eth/cic_eth/api/api_task.py

@@ -15,7 +15,6 @@ from cic_eth.enum import LockEnum
 from hexathon import strip_0x
 
 app = celery.current_app
-print(app.backend)
 
 #logg = logging.getLogger(__name__)
 logg = logging.getLogger()

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -1,107 +1,40 @@
 # standard imports
-import json
 import logging
-from os import path
-from urllib.parse import parse_qsl, urlparse
+from urllib.parse import urlparse
 
-from cic_eth.server.celery_helper import call
-from cic_eth.server.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest
-from openapi_core import create_spec
-from openapi_core.validation.request.validators import RequestValidator
-from openapi_spec_validator.schemas import read_yaml_file
+import redis
+from cic_eth.server import args, cache, config, openapi, routes
 
-spec_dict = read_yaml_file(path.join(path.dirname(
-    __file__), '../../server/openapi/server.yaml'))
-spec = create_spec(spec_dict)
+# define log levels
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
 
+log = logging.getLogger(__name__)
 
-logging.basicConfig(level=logging.WARNING)
-log = logging.getLogger()
+# TODO Censor relevant things 
+log.debug(f"{config}")
 
-# TODO Implement wei conversions
+# define universal redis cache access
+cache.Cache.store = redis.StrictRedis(host=config.get('REDIS_HOST'),
+                                      port=config.get('REDIS_PORT'),
+                                      db=config.get('REDIS_DB'),
+                                      decode_responses=True)
 
 # uwsgi application
 
 
 def application(env, start_response):
     # Validate incoming request against the open api spec
-    oAPIRequest = UWSGIOpenAPIRequest(env)
-    validator = RequestValidator(spec)
-    result = validator.validate(oAPIRequest)
-
-    if result.errors:
-        json_data = json.dumps(list(map(lambda e: str(e), result.errors)))
-        content = json_data.encode('utf-8')
-        headers = []
-        headers.append(('Content-Length', str(len(content))),)
-        headers.append(('Access-Control-Allow-Origin', '*',))
-        headers.append(('Content-Type', 'application/json',))
-        start_response('400 Invalid Request', headers)
-        return [content]
-
+    errors, request = openapi.validate.request(env, start_response)
+    if errors:
+        return errors
     parsed_url = urlparse(env.get('REQUEST_URI'))
     path = parsed_url.path
-    params = dict(parse_qsl(parsed_url.query))
-
-    if path == '/transactions':
-        address = params.pop('address')
-        # address, limit=10
-        data = call('list', address, **params)
-
-    elif path == '/balance':
-        address = params.pop('address')
-        token_symbol = params.pop('token_symbol')
-        data = call('balance', address, token_symbol, **params)
-        for b in data:
-            b.update({
-                "balance_available": int(b['balance_network']) + int(b['balance_incoming']) - int(b['balance_outgoing'])
-            })
-    elif path == '/create_account':
-        data = call('create_account', **params)
-
-    elif path == '/refill_gas':
-        address = params.pop('address')
-        data = call('refill_gas', address)
-
-    elif path == '/ping':
-        data = call('ping', **params)
-
-    elif path == '/transfer':
-        from_address = params.pop('from_address')
-        to_address = params.pop('to_address')
-        value = params.pop('value')
-        token_symbol = params.pop('token_symbol')
-
-        data = call('transfer', from_address, to_address, value, token_symbol)
-
-    elif path == '/transfer_from':
-        from_address = params.pop('from_address')
-        to_address = params.pop('to_address')
-        value = params.pop('value')
-        token_symbol = params.pop('token_symbol')
-        spender_address = params.pop('spender_address')
-        data = call('transfer_from', from_address, to_address,
-                    value, token_symbol, spender_address)
-
-    elif path == '/token':
-        token_symbol = params.pop('token_symbol')
-        data = call('token', token_symbol, **params)
-
-    elif path == '/tokens':
-        token_symbols = params.pop('token_symbols')
-        data = call('tokens', token_symbols, **params)
-
-    elif path == '/default_token':
-        data = call('default_token')
-    else:
-        start_response('404 This is not the path you\'re looking for', [])
-        return []
-    json_data = json.dumps(data)
-    content = json_data.encode('utf-8')
-    headers = []
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',))
-    headers.append(('Content-Type', 'application/json',))
-    start_response('200 OK', headers)
-
-    return [content]
+    query = dict(request.parameters.query)
+    handler = routes.get(path)
+    if handler:
+        return handler(start_response, query)
+    start_response('404 This is not the path you\'re looking for', [])
+    return []

apps/cic-eth/cic_eth/server/__init__.py

@@ -0,0 +1,5 @@
+from . import converters
+from . import cache
+from . import uwsgi
+from .routes import routes
+from .config import config, args

apps/cic-eth/cic_eth/server/cache.py

@@ -0,0 +1,121 @@
+# standard imports
+import hashlib
+import json
+import logging
+from typing import Optional, Union
+
+# external imports
+from cic_types.condiments import MetadataPointer
+from redis import Redis
+
+logg = logging.getLogger(__file__)
+
+
+class Cache:
+    store: Redis = None
+
+def get_token_data(token_symbol: str):
+    """
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    identifier = [token_symbol.encode('utf-8')]
+    key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA)
+    logg.debug(f'Retrieving token data for: {token_symbol} at: {key}')
+    token_data_str = get_cached_data(key=key)
+    if(token_data_str is None):
+        logg.debug(f'No token data found for: {token_symbol}')
+        return None
+    else:
+        token_data = json.loads(token_data_str)
+        logg.debug(f'Retrieved token data: {token_data}')
+        return token_data
+
+
+def set_token_data(token_symbol: str, data: dict):
+    """
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    identifier = [token_symbol.encode('utf-8')]
+    key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA)
+    cache_data(key, json.dumps(data))
+    logg.debug(f'Cached token data for: {token_symbol} at: {key}')
+
+def get_default_token() -> Optional[str]:
+    """This function attempts to retrieve the default token's data from the redis cache.
+    :param chain_str: chain name and network id.
+    :type chain_str: str
+    :return:
+    :rtype:
+    """
+    logg.debug(f'Retrieving default token from cache for chain:')
+    key = cache_data_key(identifier="ff".encode('utf-8'),
+                         salt=MetadataPointer.TOKEN_DEFAULT)
+    token_data = json.loads(get_cached_data(key=key))
+    logg.debug(f'Retrieved default token data: {token_data}')
+    return token_data
+
+
+def set_default_token(default_token: dict):
+    """
+    :param default_token:
+    :type default_token:
+    :return:
+    :rtype:
+    """
+    logg.debug(f'Setting default token in cache')
+    key = cache_data_key(identifier="ff".encode('utf-8'),
+                         salt=MetadataPointer.TOKEN_DEFAULT)
+    cache_data(key, json.dumps(default_token))
+
+
+
+
+def cache_data(key: str, data: str):
+    """
+    :param key:
+    :type key:
+    :param data:
+    :type data:
+    :return:
+    :rtype:
+    """
+    cache = Cache.store
+    cache.set(name=key, value=data)
+    cache.persist(name=key)
+    logg.debug(f'caching: {data} with key: {key}.')
+
+
+def get_cached_data(key: str):
+    """
+    :param key:
+    :type key:
+    :return:
+    :rtype:
+    """
+    cache = Cache.store
+    return cache.get(name=key)
+
+
+def cache_data_key(identifier: Union[list, bytes], salt: MetadataPointer):
+    """
+    :param identifier:
+    :type identifier:
+    :param salt:
+    :type salt:
+    :return:
+    :rtype:
+    """
+    hash_object = hashlib.new("sha256")
+    if isinstance(identifier, list):
+        for identity in identifier:
+            hash_object.update(identity)
+    else:
+        hash_object.update(identifier)
+    hash_object.update(salt.value.encode(encoding="utf-8"))
+    return hash_object.digest().hex()

apps/cic-eth/cic_eth/server/celery.py

@@ -9,8 +9,6 @@ from cic_eth.api.api_task import Api
 from cic_eth.server.config import config
 
 log = logging.getLogger(__name__)
-# TODO Remove
-log.setLevel(logging.DEBUG)
 
 celery_app = cic_eth.cli.CeleryApp.from_config(config)
 celery_app.set_default()
@@ -45,11 +43,11 @@ def call(method, *args, **kwargs):
         o = ps.get_message(timeout=config.get('REDIS_TIMEOUT'))
     except TimeoutError as e:
         sys.stderr.write(
-            f"cic_eth.api.{method}({', '.join(args)}, {', '.join(f'{key}={value}' for key, value in kwargs.items())}) timed out:\n {e}")
+            f"cic_eth.api.{method}({args}, {kwargs}) timed out:\n {e}")
         sys.exit(1)
 
     log.debug(
-        f"cic_eth.api.{method}({', '.join(args)}, {', '.join(f'{key}={value}' for key, value in kwargs.items())})\n {o}")
+        f"cic_eth.api.{method}(args={args}, kwargs={kwargs})\n {o}")
 
     ps.unsubscribe()
     try:

apps/cic-eth/cic_eth/server/converters.py

@@ -0,0 +1,39 @@
+# Stolen from ussd
+
+from math import trunc
+
+def from_wei(decimals: int, value: int) -> float:
+    """This function converts values in Wei to a token in the cic network.
+    :param decimals: The decimals required for wei values.
+    :type decimals: int
+    :param value: Value in Wei
+    :type value: int
+    :return: SRF equivalent of value in Wei
+    :rtype: float
+    """
+    value = float(value) / (10**decimals)
+    return truncate(value=value, decimals=2)
+
+
+def to_wei(decimals: int, value: int) -> int:
+    """This functions converts values from a token in the cic network to Wei.
+    :param decimals: The decimals required for wei values.
+    :type decimals: int
+    :param value: Value in SRF
+    :type value: int
+    :return: Wei equivalent of value in SRF
+    :rtype: int
+    """
+    return int(value * (10**decimals))
+
+def truncate(value: float, decimals: int) -> float:
+    """This function truncates a value to a specified number of decimals places.
+    :param value: The value to be truncated.
+    :type value: float
+    :param decimals: The number of decimals for the value to be truncated to
+    :type decimals: int
+    :return: The truncated value.
+    :rtype: int
+    """
+    stepper = 10.0**decimals
+    return trunc(stepper*value) / stepper

apps/cic-eth/cic_eth/server/openapi/UWSGIOpenAPIRequest.py

@@ -1,18 +1,14 @@
 """OpenAPI core contrib requests requests module"""
 
-from urllib.parse import parse_qs
-from urllib.parse import urlparse
+from urllib.parse import parse_qs, urlparse
 
+from openapi_core.validation.request.datatypes import (OpenAPIRequest,
+                                                       RequestParameters)
 from requests import Request
-from werkzeug.datastructures import Headers
-from werkzeug.datastructures import ImmutableMultiDict
-
-from openapi_core.validation.request.datatypes import OpenAPIRequest
-from openapi_core.validation.request.datatypes import RequestParameters
+from werkzeug.datastructures import Headers, ImmutableMultiDict
 from werkzeug.wrappers import Request
 
-
-class UWSGIOpenAPIRequestFactory:
+class OpenAPIRequestFactory:
     @classmethod
     def create(cls, env):
         """
@@ -21,11 +17,9 @@ class UWSGIOpenAPIRequestFactory:
         """
         request = Request(env)
 
-
         # Method
         method = request.method.lower()
 
-
         # Preparing a request formats the URL with params, strip them out again
         o = urlparse(request.url)
         params = parse_qs(o.query)
@@ -59,4 +53,5 @@ class UWSGIOpenAPIRequestFactory:
             mimetype=mimetype,
         )
 
-UWSGIOpenAPIRequest = UWSGIOpenAPIRequestFactory.create
+
+UWSGIOpenAPIRequest = OpenAPIRequestFactory.create

apps/cic-eth/cic_eth/server/openapi/__init__.py

@@ -0,0 +1,2 @@
+from . import validate
+from . import UWSGIOpenAPIRequest

apps/cic-eth/cic_eth/server/openapi/validate.py

@@ -0,0 +1,31 @@
+import json
+from os import path
+from typing import Tuple, Type, Union, List
+
+from openapi_core.validation.request.datatypes import OpenAPIRequest
+
+from cic_eth.server.openapi.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest
+from openapi_core import create_spec
+from openapi_core.validation.request.validators import RequestValidator
+from openapi_spec_validator.schemas import read_yaml_file
+
+spec_dict = read_yaml_file(path.join(path.dirname(
+    __file__), './server.yaml'))
+spec = create_spec(spec_dict)
+
+
+def request(env, start_response) -> Tuple[Union[List[bytes], None], Type[OpenAPIRequest]]:
+    oAPIRequest = UWSGIOpenAPIRequest(env)
+    validator = RequestValidator(spec)
+    result = validator.validate(oAPIRequest)
+
+    if result.errors:
+        json_data = json.dumps(list(map(lambda e: str(e), result.errors)))
+        content = json_data.encode('utf-8')
+        headers = []
+        headers.append(('Content-Length', str(len(content))),)
+        headers.append(('Access-Control-Allow-Origin', '*',))
+        headers.append(('Content-Type', 'application/json',))
+        start_response('400 Invalid Request', headers)
+        return ([content], oAPIRequest)
+    return (None, oAPIRequest)

apps/cic-eth/cic_eth/server/routes.py

@@ -0,0 +1,131 @@
+"""
+Contains all handlers for valid routes for the server
+"""
+
+# standard imports
+import logging
+from typing import Any, Callable, Dict, List, Type, Union
+
+from cic_eth.server import cache, celery, converters, uwsgi
+from cic_eth.server.openapi.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest
+
+log = logging.getLogger(__name__)
+
+
+def handle_transactions(start_response, query: dict) -> List[bytes]:
+    address = query.pop('address')
+    data = celery.call('list', address, **query)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_balance(start_response, query: dict) -> List[bytes]:
+    address = query.pop('address')
+    token_symbol = query.pop('token_symbol')
+    log.info(f"token_symbol: {token_symbol}")
+    log.info(f"query: {query}")
+    data = celery.call('balance', address, token_symbol,
+                       **query)
+    for b in data:
+        token_data = get_token_data(token_symbol)
+        b['balance_network'] = converters.from_wei(token_data.get(
+            'decimals'), int(b['balance_network']))
+        b['balance_incoming'] = converters.from_wei(token_data.get(
+            'decimals'), int(b['balance_incoming']))
+        b['balance_outgoing'] = converters.from_wei(token_data.get(
+            'decimals'), int(b['balance_outgoing']))
+
+        b.update({
+            "balance_available": int(b['balance_network']) + int(b['balance_incoming']) - int(b['balance_outgoing'])
+        })
+    return uwsgi.respond(start_response, data)
+
+
+def handle_create_account(start_response, query: dict) -> List[bytes]:
+    data = celery.call('create_account', **query)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_refill_gas(start_response, query: dict) -> List[bytes]:
+    address = query.pop('address')
+    data = celery.call('refill_gas', address)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_ping(start_response, query: dict) -> List[bytes]:
+    data = celery.call('ping', **query)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_transfer(start_response, query: dict) -> List[bytes]:
+    from_address = query.pop('from_address')
+    to_address = query.pop('to_address')
+    value = query.pop('value')
+    token_symbol = query.pop('token_symbol')
+    wei_value = converters.to_wei(get_token_data(
+        token_symbol).get('decimals'), int(value))
+    data = celery.call('transfer', from_address,
+                       to_address, wei_value, token_symbol)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_transfer_from(start_response, query: dict) -> List[bytes]:
+    from_address = query.pop('from_address')
+    to_address = query.pop('to_address')
+    value = query.pop('value')
+    token_symbol = query.pop('token_symbol')
+    spender_address = query.pop('spender_address')
+    wei_value = converters.to_wei(get_token_data(
+        token_symbol).get('decimals'), int(value))
+    data = celery.call('transfer_from', from_address, to_address,
+                       wei_value, token_symbol, spender_address)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_token(start_response, query: dict) -> List[bytes]:
+    token_symbol = query.pop('token_symbol')
+    data = cache.get_token_data(token_symbol)
+    if data == None:
+        data = celery.call('token', token_symbol, **query)
+        cache.set_token_data(token_symbol, data)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_tokens(start_response, query: dict) -> List[bytes]:
+    token_symbols = query.pop('token_symbols')
+    data = celery.call('tokens', token_symbols, **query)
+    return uwsgi.respond(start_response, data)
+
+
+def handle_default_token(start_response, _query) -> List[bytes]:
+    data = cache.get_default_token()
+    if data == None:
+        data = celery.call('default_token')
+        cache.set_default_token(data)
+        cache.set_token_data(data.get('token_symbol'), data)
+    return uwsgi.respond(start_response, data)
+
+
+def get_token_data(token_symbol: str) -> List[bytes]:
+    data = cache.get_token_data(token_symbol)
+    log.debug(f"cached token data: {data}")
+    if data == None:
+        data = celery.call('token', token_symbol)
+        log.debug(
+            f"No token data setting token data for: {token_symbol} to {data}")
+        cache.set_token_data(token_symbol, data)
+    # TODO What is the second item in the list
+    return data[0]
+
+
+routes: Dict[str, Callable[[Any, Union[None, dict]], List[bytes]]] = {
+    "/transactions": handle_transactions,
+    "/balance": handle_balance,
+    "/create_account": handle_create_account,
+    "/refill_gas": handle_refill_gas,
+    "/ping": handle_ping,
+    "/transfer": handle_transfer,
+    "/transfer_from": handle_transfer_from,
+    "/token": handle_token,
+    "/tokens": handle_tokens,
+    "/default_token": handle_default_token,
+}

apps/cic-eth/cic_eth/server/uwsgi.py

@@ -0,0 +1,18 @@
+# standard imports
+import json
+
+DEFAULT_HEADERS = [
+    ('Access-Control-Allow-Origin', '*'),
+    ('Content-Type', 'application/json'),
+]
+
+
+def respond(start_response, data):
+    json_data = json.dumps(data)
+    content = json_data.encode('utf-8')
+    headers = [
+        *DEFAULT_HEADERS,
+        ('Content-Type', 'application/json',)
+    ]
+    start_response('200 OK', headers)
+    return [content]

apps/cic-eth/config/docker/redis.ini

@@ -0,0 +1,5 @@
+[redis]
+host=redis
+database=0
+password=
+port=6379

apps/cic-eth/requirements.txt

@@ -1,3 +1,4 @@
+cic-types~=0.2.1a7
 celery==4.4.7
 chainlib-eth>=0.0.10a16,<0.1.0
 semver==2.13.0
@@ -6,4 +7,4 @@ uwsgi==2.0.19.1
 setuptools >= 21.0.0
 openapi-core >= 0.14.2
 openapi-spec-validator >= 0.3.1
-Werkzeug>=2.0.2
+Werkzeug>=2.0.2

apps/cic-eth/tests/test_server.py

@@ -64,9 +64,9 @@ def test_account(testserver):
 
     assert (balance[0] == {
         "address": default_token.get('address').lower(),
-        "balance_available": 30000000000,
+        "balance_available": 30000,
         "balance_incoming": 0,
-        "balance_network": 30000000000,
+        "balance_network": 30000,
         "balance_outgoing": 0,
         "converters": []
     })
@@ -95,10 +95,10 @@ def test_account(testserver):
     balance_after_transfer = response.json()
     assert (balance_after_transfer[0] == {
         "address": default_token.get('address').lower(),
-        "balance_available": 29900000000,
+        "balance_available": 29900,
         "balance_incoming": 0,
-        "balance_network": 30000000000,
-        "balance_outgoing": 100000000,
+        "balance_network": 30000,
+        "balance_outgoing": 100,
         "converters": []
     })
 
@@ -111,5 +111,9 @@ def test_account(testserver):
         testserver.url + '/transactions',
         params=params)
     transactions = response.json()
-    log.debug(transactions)
-    exit(1)
+    ## TODO: What are the other 2 transactions
+    assert len(transactions) == 3
+    ## Check the transaction is correct 
+    # TODO wtf is READSEND (Ready to send? Or already sent)
+    assert transactions[0].status == 'READYSEND'
+    exit(1) # Forcing it to fail to i get logs out of pytest

docker-compose.yml

@@ -210,6 +210,7 @@ services:
       - cic-eth-tracker
       - cic-eth-dispatcher
     volumes:
+      # - ./apps/cic-eth/:/root Useful for developing locally 
       - signer-data:/run/crypto-dev-signer
       - contract-config:/tmp/cic/config/:ro
     command: