feat: add wei conversions and caching

This commit is contained in:
William Luke 2021-12-02 15:07:49 +03:00
parent 253725c24f
commit b3df49f89d
15 changed files with 400 additions and 117 deletions

View File

@ -15,7 +15,6 @@ from cic_eth.enum import LockEnum
from hexathon import strip_0x
app = celery.current_app
print(app.backend)
#logg = logging.getLogger(__name__)
logg = logging.getLogger()

View File

@ -1,107 +1,40 @@
# standard imports
import json
import logging
from os import path
from urllib.parse import parse_qsl, urlparse
from urllib.parse import urlparse
from cic_eth.server.celery_helper import call
from cic_eth.server.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest
from openapi_core import create_spec
from openapi_core.validation.request.validators import RequestValidator
from openapi_spec_validator.schemas import read_yaml_file
import redis
from cic_eth.server import args, cache, config, openapi, routes
spec_dict = read_yaml_file(path.join(path.dirname(
__file__), '../../server/openapi/server.yaml'))
spec = create_spec(spec_dict)
# define log levels
if args.vv:
logging.getLogger().setLevel(logging.DEBUG)
elif args.v:
logging.getLogger().setLevel(logging.INFO)
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.WARNING)
log = logging.getLogger()
# TODO Censor relevant things
log.debug(f"{config}")
# TODO Implement wei conversions
# define universal redis cache access
cache.Cache.store = redis.StrictRedis(host=config.get('REDIS_HOST'),
port=config.get('REDIS_PORT'),
db=config.get('REDIS_DB'),
decode_responses=True)
# uwsgi application
def application(env, start_response):
# Validate incoming request against the open api spec
oAPIRequest = UWSGIOpenAPIRequest(env)
validator = RequestValidator(spec)
result = validator.validate(oAPIRequest)
if result.errors:
json_data = json.dumps(list(map(lambda e: str(e), result.errors)))
content = json_data.encode('utf-8')
headers = []
headers.append(('Content-Length', str(len(content))),)
headers.append(('Access-Control-Allow-Origin', '*',))
headers.append(('Content-Type', 'application/json',))
start_response('400 Invalid Request', headers)
return [content]
errors, request = openapi.validate.request(env, start_response)
if errors:
return errors
parsed_url = urlparse(env.get('REQUEST_URI'))
path = parsed_url.path
params = dict(parse_qsl(parsed_url.query))
if path == '/transactions':
address = params.pop('address')
# address, limit=10
data = call('list', address, **params)
elif path == '/balance':
address = params.pop('address')
token_symbol = params.pop('token_symbol')
data = call('balance', address, token_symbol, **params)
for b in data:
b.update({
"balance_available": int(b['balance_network']) + int(b['balance_incoming']) - int(b['balance_outgoing'])
})
elif path == '/create_account':
data = call('create_account', **params)
elif path == '/refill_gas':
address = params.pop('address')
data = call('refill_gas', address)
elif path == '/ping':
data = call('ping', **params)
elif path == '/transfer':
from_address = params.pop('from_address')
to_address = params.pop('to_address')
value = params.pop('value')
token_symbol = params.pop('token_symbol')
data = call('transfer', from_address, to_address, value, token_symbol)
elif path == '/transfer_from':
from_address = params.pop('from_address')
to_address = params.pop('to_address')
value = params.pop('value')
token_symbol = params.pop('token_symbol')
spender_address = params.pop('spender_address')
data = call('transfer_from', from_address, to_address,
value, token_symbol, spender_address)
elif path == '/token':
token_symbol = params.pop('token_symbol')
data = call('token', token_symbol, **params)
elif path == '/tokens':
token_symbols = params.pop('token_symbols')
data = call('tokens', token_symbols, **params)
elif path == '/default_token':
data = call('default_token')
else:
start_response('404 This is not the path you\'re looking for', [])
return []
json_data = json.dumps(data)
content = json_data.encode('utf-8')
headers = []
headers.append(('Content-Length', str(len(content))),)
headers.append(('Access-Control-Allow-Origin', '*',))
headers.append(('Content-Type', 'application/json',))
start_response('200 OK', headers)
return [content]
query = dict(request.parameters.query)
handler = routes.get(path)
if handler:
return handler(start_response, query)
start_response('404 This is not the path you\'re looking for', [])
return []

View File

@ -0,0 +1,5 @@
from . import converters
from . import cache
from . import uwsgi
from .routes import routes
from .config import config, args

View File

@ -0,0 +1,121 @@
# standard imports
import hashlib
import json
import logging
from typing import Optional, Union
# external imports
from cic_types.condiments import MetadataPointer
from redis import Redis
logg = logging.getLogger(__file__)
class Cache:
store: Redis = None
def get_token_data(token_symbol: str):
"""
:param token_symbol:
:type token_symbol:
:return:
:rtype:
"""
identifier = [token_symbol.encode('utf-8')]
key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA)
logg.debug(f'Retrieving token data for: {token_symbol} at: {key}')
token_data_str = get_cached_data(key=key)
if(token_data_str is None):
logg.debug(f'No token data found for: {token_symbol}')
return None
else:
token_data = json.loads(token_data_str)
logg.debug(f'Retrieved token data: {token_data}')
return token_data
def set_token_data(token_symbol: str, data: dict):
"""
:param token_symbol:
:type token_symbol:
:return:
:rtype:
"""
identifier = [token_symbol.encode('utf-8')]
key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA)
cache_data(key, json.dumps(data))
logg.debug(f'Cached token data for: {token_symbol} at: {key}')
def get_default_token() -> Optional[str]:
"""This function attempts to retrieve the default token's data from the redis cache.
:param chain_str: chain name and network id.
:type chain_str: str
:return:
:rtype:
"""
logg.debug(f'Retrieving default token from cache for chain:')
key = cache_data_key(identifier="ff".encode('utf-8'),
salt=MetadataPointer.TOKEN_DEFAULT)
token_data = json.loads(get_cached_data(key=key))
logg.debug(f'Retrieved default token data: {token_data}')
return token_data
def set_default_token(default_token: dict):
"""
:param default_token:
:type default_token:
:return:
:rtype:
"""
logg.debug(f'Setting default token in cache')
key = cache_data_key(identifier="ff".encode('utf-8'),
salt=MetadataPointer.TOKEN_DEFAULT)
cache_data(key, json.dumps(default_token))
def cache_data(key: str, data: str):
"""
:param key:
:type key:
:param data:
:type data:
:return:
:rtype:
"""
cache = Cache.store
cache.set(name=key, value=data)
cache.persist(name=key)
logg.debug(f'caching: {data} with key: {key}.')
def get_cached_data(key: str):
"""
:param key:
:type key:
:return:
:rtype:
"""
cache = Cache.store
return cache.get(name=key)
def cache_data_key(identifier: Union[list, bytes], salt: MetadataPointer):
"""
:param identifier:
:type identifier:
:param salt:
:type salt:
:return:
:rtype:
"""
hash_object = hashlib.new("sha256")
if isinstance(identifier, list):
for identity in identifier:
hash_object.update(identity)
else:
hash_object.update(identifier)
hash_object.update(salt.value.encode(encoding="utf-8"))
return hash_object.digest().hex()

View File

@ -9,8 +9,6 @@ from cic_eth.api.api_task import Api
from cic_eth.server.config import config
log = logging.getLogger(__name__)
# TODO Remove
log.setLevel(logging.DEBUG)
celery_app = cic_eth.cli.CeleryApp.from_config(config)
celery_app.set_default()
@ -45,11 +43,11 @@ def call(method, *args, **kwargs):
o = ps.get_message(timeout=config.get('REDIS_TIMEOUT'))
except TimeoutError as e:
sys.stderr.write(
f"cic_eth.api.{method}({', '.join(args)}, {', '.join(f'{key}={value}' for key, value in kwargs.items())}) timed out:\n {e}")
f"cic_eth.api.{method}({args}, {kwargs}) timed out:\n {e}")
sys.exit(1)
log.debug(
f"cic_eth.api.{method}({', '.join(args)}, {', '.join(f'{key}={value}' for key, value in kwargs.items())})\n {o}")
f"cic_eth.api.{method}(args={args}, kwargs={kwargs})\n {o}")
ps.unsubscribe()
try:

View File

@ -0,0 +1,39 @@
# Stolen from ussd
from math import trunc
def from_wei(decimals: int, value: int) -> float:
"""This function converts values in Wei to a token in the cic network.
:param decimals: The decimals required for wei values.
:type decimals: int
:param value: Value in Wei
:type value: int
:return: SRF equivalent of value in Wei
:rtype: float
"""
value = float(value) / (10**decimals)
return truncate(value=value, decimals=2)
def to_wei(decimals: int, value: int) -> int:
"""This functions converts values from a token in the cic network to Wei.
:param decimals: The decimals required for wei values.
:type decimals: int
:param value: Value in SRF
:type value: int
:return: Wei equivalent of value in SRF
:rtype: int
"""
return int(value * (10**decimals))
def truncate(value: float, decimals: int) -> float:
"""This function truncates a value to a specified number of decimals places.
:param value: The value to be truncated.
:type value: float
:param decimals: The number of decimals for the value to be truncated to
:type decimals: int
:return: The truncated value.
:rtype: int
"""
stepper = 10.0**decimals
return trunc(stepper*value) / stepper

View File

@ -1,18 +1,14 @@
"""OpenAPI core contrib requests requests module"""
from urllib.parse import parse_qs
from urllib.parse import urlparse
from urllib.parse import parse_qs, urlparse
from openapi_core.validation.request.datatypes import (OpenAPIRequest,
RequestParameters)
from requests import Request
from werkzeug.datastructures import Headers
from werkzeug.datastructures import ImmutableMultiDict
from openapi_core.validation.request.datatypes import OpenAPIRequest
from openapi_core.validation.request.datatypes import RequestParameters
from werkzeug.datastructures import Headers, ImmutableMultiDict
from werkzeug.wrappers import Request
class UWSGIOpenAPIRequestFactory:
class OpenAPIRequestFactory:
@classmethod
def create(cls, env):
"""
@ -21,11 +17,9 @@ class UWSGIOpenAPIRequestFactory:
"""
request = Request(env)
# Method
method = request.method.lower()
# Preparing a request formats the URL with params, strip them out again
o = urlparse(request.url)
params = parse_qs(o.query)
@ -59,4 +53,5 @@ class UWSGIOpenAPIRequestFactory:
mimetype=mimetype,
)
UWSGIOpenAPIRequest = UWSGIOpenAPIRequestFactory.create
UWSGIOpenAPIRequest = OpenAPIRequestFactory.create

View File

@ -0,0 +1,2 @@
from . import validate
from . import UWSGIOpenAPIRequest

View File

@ -0,0 +1,31 @@
import json
from os import path
from typing import Tuple, Type, Union, List
from openapi_core.validation.request.datatypes import OpenAPIRequest
from cic_eth.server.openapi.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest
from openapi_core import create_spec
from openapi_core.validation.request.validators import RequestValidator
from openapi_spec_validator.schemas import read_yaml_file
spec_dict = read_yaml_file(path.join(path.dirname(
__file__), './server.yaml'))
spec = create_spec(spec_dict)
def request(env, start_response) -> Tuple[Union[List[bytes], None], Type[OpenAPIRequest]]:
oAPIRequest = UWSGIOpenAPIRequest(env)
validator = RequestValidator(spec)
result = validator.validate(oAPIRequest)
if result.errors:
json_data = json.dumps(list(map(lambda e: str(e), result.errors)))
content = json_data.encode('utf-8')
headers = []
headers.append(('Content-Length', str(len(content))),)
headers.append(('Access-Control-Allow-Origin', '*',))
headers.append(('Content-Type', 'application/json',))
start_response('400 Invalid Request', headers)
return ([content], oAPIRequest)
return (None, oAPIRequest)

View File

@ -0,0 +1,131 @@
"""
Contains all handlers for valid routes for the server
"""
# standard imports
import logging
from typing import Any, Callable, Dict, List, Type, Union
from cic_eth.server import cache, celery, converters, uwsgi
from cic_eth.server.openapi.UWSGIOpenAPIRequest import UWSGIOpenAPIRequest
log = logging.getLogger(__name__)
def handle_transactions(start_response, query: dict) -> List[bytes]:
address = query.pop('address')
data = celery.call('list', address, **query)
return uwsgi.respond(start_response, data)
def handle_balance(start_response, query: dict) -> List[bytes]:
address = query.pop('address')
token_symbol = query.pop('token_symbol')
log.info(f"token_symbol: {token_symbol}")
log.info(f"query: {query}")
data = celery.call('balance', address, token_symbol,
**query)
for b in data:
token_data = get_token_data(token_symbol)
b['balance_network'] = converters.from_wei(token_data.get(
'decimals'), int(b['balance_network']))
b['balance_incoming'] = converters.from_wei(token_data.get(
'decimals'), int(b['balance_incoming']))
b['balance_outgoing'] = converters.from_wei(token_data.get(
'decimals'), int(b['balance_outgoing']))
b.update({
"balance_available": int(b['balance_network']) + int(b['balance_incoming']) - int(b['balance_outgoing'])
})
return uwsgi.respond(start_response, data)
def handle_create_account(start_response, query: dict) -> List[bytes]:
data = celery.call('create_account', **query)
return uwsgi.respond(start_response, data)
def handle_refill_gas(start_response, query: dict) -> List[bytes]:
address = query.pop('address')
data = celery.call('refill_gas', address)
return uwsgi.respond(start_response, data)
def handle_ping(start_response, query: dict) -> List[bytes]:
data = celery.call('ping', **query)
return uwsgi.respond(start_response, data)
def handle_transfer(start_response, query: dict) -> List[bytes]:
from_address = query.pop('from_address')
to_address = query.pop('to_address')
value = query.pop('value')
token_symbol = query.pop('token_symbol')
wei_value = converters.to_wei(get_token_data(
token_symbol).get('decimals'), int(value))
data = celery.call('transfer', from_address,
to_address, wei_value, token_symbol)
return uwsgi.respond(start_response, data)
def handle_transfer_from(start_response, query: dict) -> List[bytes]:
from_address = query.pop('from_address')
to_address = query.pop('to_address')
value = query.pop('value')
token_symbol = query.pop('token_symbol')
spender_address = query.pop('spender_address')
wei_value = converters.to_wei(get_token_data(
token_symbol).get('decimals'), int(value))
data = celery.call('transfer_from', from_address, to_address,
wei_value, token_symbol, spender_address)
return uwsgi.respond(start_response, data)
def handle_token(start_response, query: dict) -> List[bytes]:
token_symbol = query.pop('token_symbol')
data = cache.get_token_data(token_symbol)
if data == None:
data = celery.call('token', token_symbol, **query)
cache.set_token_data(token_symbol, data)
return uwsgi.respond(start_response, data)
def handle_tokens(start_response, query: dict) -> List[bytes]:
token_symbols = query.pop('token_symbols')
data = celery.call('tokens', token_symbols, **query)
return uwsgi.respond(start_response, data)
def handle_default_token(start_response, _query) -> List[bytes]:
data = cache.get_default_token()
if data == None:
data = celery.call('default_token')
cache.set_default_token(data)
cache.set_token_data(data.get('token_symbol'), data)
return uwsgi.respond(start_response, data)
def get_token_data(token_symbol: str) -> List[bytes]:
data = cache.get_token_data(token_symbol)
log.debug(f"cached token data: {data}")
if data == None:
data = celery.call('token', token_symbol)
log.debug(
f"No token data setting token data for: {token_symbol} to {data}")
cache.set_token_data(token_symbol, data)
# TODO What is the second item in the list
return data[0]
routes: Dict[str, Callable[[Any, Union[None, dict]], List[bytes]]] = {
"/transactions": handle_transactions,
"/balance": handle_balance,
"/create_account": handle_create_account,
"/refill_gas": handle_refill_gas,
"/ping": handle_ping,
"/transfer": handle_transfer,
"/transfer_from": handle_transfer_from,
"/token": handle_token,
"/tokens": handle_tokens,
"/default_token": handle_default_token,
}

View File

@ -0,0 +1,18 @@
# standard imports
import json
DEFAULT_HEADERS = [
('Access-Control-Allow-Origin', '*'),
('Content-Type', 'application/json'),
]
def respond(start_response, data):
json_data = json.dumps(data)
content = json_data.encode('utf-8')
headers = [
*DEFAULT_HEADERS,
('Content-Type', 'application/json',)
]
start_response('200 OK', headers)
return [content]

View File

@ -0,0 +1,5 @@
[redis]
host=redis
database=0
password=
port=6379

View File

@ -1,3 +1,4 @@
cic-types~=0.2.1a7
celery==4.4.7
chainlib-eth>=0.0.10a16,<0.1.0
semver==2.13.0

View File

@ -64,9 +64,9 @@ def test_account(testserver):
assert (balance[0] == {
"address": default_token.get('address').lower(),
"balance_available": 30000000000,
"balance_available": 30000,
"balance_incoming": 0,
"balance_network": 30000000000,
"balance_network": 30000,
"balance_outgoing": 0,
"converters": []
})
@ -95,10 +95,10 @@ def test_account(testserver):
balance_after_transfer = response.json()
assert (balance_after_transfer[0] == {
"address": default_token.get('address').lower(),
"balance_available": 29900000000,
"balance_available": 29900,
"balance_incoming": 0,
"balance_network": 30000000000,
"balance_outgoing": 100000000,
"balance_network": 30000,
"balance_outgoing": 100,
"converters": []
})
@ -111,5 +111,9 @@ def test_account(testserver):
testserver.url + '/transactions',
params=params)
transactions = response.json()
log.debug(transactions)
exit(1)
## TODO: What are the other 2 transactions
assert len(transactions) == 3
## Check the transaction is correct
# TODO wtf is READSEND (Ready to send? Or already sent)
assert transactions[0].status == 'READYSEND'
exit(1) # Forcing it to fail to i get logs out of pytest

View File

@ -210,6 +210,7 @@ services:
- cic-eth-tracker
- cic-eth-dispatcher
volumes:
# - ./apps/cic-eth/:/root Useful for developing locally
- signer-data:/run/crypto-dev-signer
- contract-config:/tmp/cic/config/:ro
command: