From b3adee69e7bf5c76c490f923204d8caa508fdf2d Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 18 May 2021 13:20:02 +0200 Subject: [PATCH] Add data api --- apps/cic-cache/cic_cache/cache.py | 43 +++++++- apps/cic-cache/cic_cache/db/list.py | 20 ++++ .../cic_cache/runnable/daemons/query.py | 104 ++++++++++++++++++ .../cic_cache/runnable/daemons/server.py | 74 ++----------- apps/cic-cache/tests/conftest.py | 13 +++ apps/cic-cache/tests/test_api.py | 20 ++++ apps/cic-cache/tests/test_cache.py | 21 ++++ 7 files changed, 224 insertions(+), 71 deletions(-) create mode 100644 apps/cic-cache/cic_cache/runnable/daemons/query.py create mode 100644 apps/cic-cache/tests/test_api.py diff --git a/apps/cic-cache/cic_cache/cache.py b/apps/cic-cache/cic_cache/cache.py index 341c7c9d..f4436eb8 100644 --- a/apps/cic-cache/cic_cache/cache.py +++ b/apps/cic-cache/cic_cache/cache.py @@ -1,22 +1,27 @@ # standard imports import logging -# third-party imports +# external imports import moolb # local imports -from cic_cache.db import list_transactions_mined -from cic_cache.db import list_transactions_account_mined +from cic_cache.db.list import ( + list_transactions_mined, + list_transactions_account_mined, + list_transactions_mined_with_data, + ) logg = logging.getLogger() -class BloomCache: +class Cache: def __init__(self, session): self.session = session +class BloomCache(Cache): + @staticmethod def __get_filter_size(n): n = 8192 * 8 @@ -87,3 +92,33 @@ class BloomCache: f_blocktx.add(block + tx) logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block)) return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),) + + +class DataCache(Cache): + + def load_transactions_with_data(self, offset, limit): + rows = list_transactions_mined_with_data(self.session, offset, limit) + tx_cache = [] + highest_block = -1; + lowest_block = -1; + for r in rows: + if highest_block == -1: + highest_block = r['block_number'] + lowest_block = r['block_number'] + tx_type = 'unknown' + if r['value'] != None: + tx_type = '{}.{}'.format(r['domain'], r['value']) + o = { + 'block_number': r['block_number'], + 'tx_hash': r['tx_hash'], + 'date_block': r['date_block'], + 'sender': r['sender'], + 'recipient': r['recipient'], + 'from_value': r['from_value'], + 'to_value': r['to_value'], + 'source_token': r['source_token'], + 'destination_token': r['destination_token'], + 'tx_type': tx_type, + } + tx_cache.append(o) + return (lowest_block, highest_block, tx_cache) diff --git a/apps/cic-cache/cic_cache/db/list.py b/apps/cic-cache/cic_cache/db/list.py index efcf1cfb..871630b8 100644 --- a/apps/cic-cache/cic_cache/db/list.py +++ b/apps/cic-cache/cic_cache/db/list.py @@ -28,6 +28,26 @@ def list_transactions_mined( return r +def list_transactions_mined_with_data( + session, + offset, + limit, + ): + """Executes db query to return all confirmed transactions according to the specified offset and limit. + + :param offset: Offset in data set to return transactions from + :type offset: int + :param limit: Max number of transactions to retrieve + :type limit: int + :result: Result set + :rtype: SQLAlchemy.ResultProxy + """ + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id" + r = session.execute(s) + return r + + + def list_transactions_account_mined( session, address, diff --git a/apps/cic-cache/cic_cache/runnable/daemons/query.py b/apps/cic-cache/cic_cache/runnable/daemons/query.py new file mode 100644 index 00000000..9e136f39 --- /dev/null +++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py @@ -0,0 +1,104 @@ +# standard imports +import json +import re + +# local imports +from cic_cache.cache import ( + BloomCache, + DataCache, + ) + +re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?' +re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?' +re_transactions_all_data = re_transactions_all_bloom + +DEFAULT_LIMIT = 100 + + +def process_transactions_account_bloom(session, env): + r = re.match(re_transactions_account_bloom, env.get('PATH_INFO')) + if not r: + return None + + address = r[1] + if r[2] == None: + address = '0x' + address + offset = DEFAULT_LIMIT + if r.lastindex > 2: + offset = r[3] + limit = 0 + if r.lastindex > 3: + limit = r[4] + + c = BloomCache(session) + (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit) + + o = { + 'alg': 'sha256', + 'low': lowest_block, + 'high': highest_block, + 'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'), + 'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'), + 'filter_rounds': 3, + } + + j = json.dumps(o) + + return ('application/json', j.encode('utf-8'),) + + +def process_transactions_all_bloom(session, env): + r = re.match(re_transactions_all_bloom, env.get('PATH_INFO')) + if not r: + return None + + offset = DEFAULT_LIMIT + if r.lastindex > 0: + offset = r[1] + limit = 0 + if r.lastindex > 1: + limit = r[2] + + c = BloomCache(session) + (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit) + + o = { + 'alg': 'sha256', + 'low': lowest_block, + 'high': highest_block, + 'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'), + 'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'), + 'filter_rounds': 3, + } + + j = json.dumps(o) + + return ('application/json', j.encode('utf-8'),) + + +def process_transactions_all_data(session, env): + r = re.match(re_transactions_all_data, env.get('PATH_INFO')) + if not r: + return None + if env.get('HTTP_X_CIC_CACHE_MODE') != 'all': + return None + + offset = DEFAULT_LIMIT + if r.lastindex > 0: + offset = r[1] + limit = 0 + if r.lastindex > 1: + limit = r[2] + + c = DataCache(session) + (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit) + + o = { + 'low': lowest_block, + 'high': highest_block, + 'data': tx_cache, + } + + j = json.dumps(o) + + return ('application/json', j.encode('utf-8'),) diff --git a/apps/cic-cache/cic_cache/runnable/daemons/server.py b/apps/cic-cache/cic_cache/runnable/daemons/server.py index e0e8465a..7d139f63 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/server.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/server.py @@ -6,13 +6,18 @@ import argparse import json import base64 -# third-party imports +# external imports import confini # local imports from cic_cache import BloomCache from cic_cache.db import dsn_from_config from cic_cache.db.models.base import SessionBase +from .query import ( + process_transactions_account_bloom, + process_transactions_all_bloom, + process_transactions_all_data, + ) logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -44,72 +49,6 @@ logg.debug('config:\n{}'.format(config)) dsn = dsn_from_config(config) SessionBase.connect(dsn, config.true('DATABASE_DEBUG')) -re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?' -re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?' - -DEFAULT_LIMIT = 100 - - -def process_transactions_account_bloom(session, env): - r = re.match(re_transactions_account_bloom, env.get('PATH_INFO')) - if not r: - return None - - address = r[1] - if r[2] == None: - address = '0x' + address - offset = DEFAULT_LIMIT - if r.lastindex > 2: - offset = r[3] - limit = 0 - if r.lastindex > 3: - limit = r[4] - - c = BloomCache(session) - (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit) - - o = { - 'alg': 'sha256', - 'low': lowest_block, - 'high': highest_block, - 'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'), - 'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'), - 'filter_rounds': 3, - } - - j = json.dumps(o) - - return ('application/json', j.encode('utf-8'),) - - -def process_transactions_all_bloom(session, env): - r = re.match(re_transactions_all_bloom, env.get('PATH_INFO')) - if not r: - return None - - offset = DEFAULT_LIMIT - if r.lastindex > 0: - offset = r[1] - limit = 0 - if r.lastindex > 1: - limit = r[2] - - c = BloomCache(session) - (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit) - - o = { - 'alg': 'sha256', - 'low': lowest_block, - 'high': highest_block, - 'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'), - 'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'), - 'filter_rounds': 3, - } - - j = json.dumps(o) - - return ('application/json', j.encode('utf-8'),) - # uwsgi application def application(env, start_response): @@ -121,6 +60,7 @@ def application(env, start_response): for handler in [ process_transactions_all_bloom, process_transactions_account_bloom, + process_transactions_all_data, ]: r = handler(session, env) if r != None: diff --git a/apps/cic-cache/tests/conftest.py b/apps/cic-cache/tests/conftest.py index 00db9ae0..f5e8edd4 100644 --- a/apps/cic-cache/tests/conftest.py +++ b/apps/cic-cache/tests/conftest.py @@ -88,3 +88,16 @@ def txs( tx_hash_first, tx_hash_second, ] + + +@pytest.fixture(scope='function') +def tag_txs( + init_database, + txs, + ): + + db.add_tag(init_database, 'taag', domain='test') + init_database.commit() + + db.tag_transaction(init_database, txs[1], 'taag', domain='test') + diff --git a/apps/cic-cache/tests/test_api.py b/apps/cic-cache/tests/test_api.py new file mode 100644 index 00000000..d7379680 --- /dev/null +++ b/apps/cic-cache/tests/test_api.py @@ -0,0 +1,20 @@ +# standard imports +import json + +# local imports +from cic_cache.runnable.daemons.query import process_transactions_all_data + + +def test_api_all_data( + init_database, + txs, + ): + + env = { + 'PATH_INFO': '/tx/0/100', + 'HTTP_X_CIC_CACHE_MODE': 'all', + } + j = process_transactions_all_data(init_database, env) + o = json.loads(j[1]) + + assert len(o['data']) == 2 diff --git a/apps/cic-cache/tests/test_cache.py b/apps/cic-cache/tests/test_cache.py index 3c772649..c35fa3a5 100644 --- a/apps/cic-cache/tests/test_cache.py +++ b/apps/cic-cache/tests/test_cache.py @@ -9,6 +9,7 @@ import pytest # local imports from cic_cache import BloomCache +from cic_cache.cache import DataCache logg = logging.getLogger() @@ -33,3 +34,23 @@ def test_cache( assert b[0] == list_defaults['block'] - 1 + +def test_cache_data( + init_database, + list_defaults, + list_actors, + list_tokens, + txs, + tag_txs, + ): + + session = init_database + + c = DataCache(session) + b = c.load_transactions_with_data(0, 100) + + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == txs[0] + assert b[2][0]['tx_type'] == 'unknown' + assert b[2][1]['tx_type'] == 'test.taag' +