grassrootseconomics/cic-stack
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: Improve cic-cache

Browse Source
This commit is contained in:
Louis Holbrook
2022-01-04 16:01:01 +00:00
committed by Philip Wafula
parent b5653a704c
commit 104ff8a76a
74 changed files with 1057 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-cache/cic_cache/data/config/cic.ini
View File

@@ -1,4 +1,4 @@
[cic]
registry_address =
trust_address =
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
health_modules =

3
apps/cic-cache/cic_cache/data/config/database.ini
View File

@@ -3,7 +3,8 @@ engine =
driver =
host =
port =
name = cic-cache
#name = cic-cache
prefix =
user =
password =
debug = 0

11
apps/cic-cache/cic_cache/db/__init__.py
View File

@@ -9,21 +9,26 @@ from .list import (
tag_transaction,
add_tag,
)
from cic_cache.db.models.base import SessionBase
logg = logging.getLogger()
def dsn_from_config(config):
def dsn_from_config(config, name):
scheme = config.get('DATABASE_ENGINE')
if config.get('DATABASE_DRIVER') != None:
scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
database_name = name
if config.get('DATABASE_PREFIX'):
database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
dsn = ''
if config.get('DATABASE_ENGINE') == 'sqlite':
SessionBase.poolable = False
dsn = '{}:///{}'.format(
scheme,
config.get('DATABASE_NAME'),
database_name,
)
else:
@@ -33,7 +38,7 @@ def dsn_from_config(config):
config.get('DATABASE_PASSWORD'),
config.get('DATABASE_HOST'),
config.get('DATABASE_PORT'),
config.get('DATABASE_NAME'),
database_name,
)
logg.debug('parsed dsn from config: {}'.format(dsn))
return dsn

121
apps/cic-cache/cic_cache/runnable/daemons/query.py
View File

@@ -5,7 +5,11 @@ import re
import base64
# external imports
from hexathon import add_0x
from hexathon import (
add_0x,
strip_0x,
)
from chainlib.encode import TxHexNormalizer
# local imports
from cic_cache.cache import (
@@ -16,27 +20,72 @@ from cic_cache.cache import (
logg = logging.getLogger(__name__)
#logg = logging.getLogger()
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
re_default_limit = r'/defaultlimit/?'
DEFAULT_LIMIT = 100
tx_normalize = TxHexNormalizer()
def parse_query_account(r):
address = strip_0x(r[1])
#address = tx_normalize.wallet_address(address)
limit = DEFAULT_LIMIT
g = r.groups()
if len(g) > 3:
limit = int(r[4])
if limit == 0:
limit = DEFAULT_LIMIT
offset = 0
if len(g) > 4:
offset = int(r[6])
logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
return (address, offset, limit,)
# r is an re.Match
def parse_query_any(r):
limit = DEFAULT_LIMIT
offset = 0
block_offset = None
block_end = None
if r.lastindex != None:
if r.lastindex > 0:
limit = int(r[1])
if r.lastindex > 1:
offset = int(r[2])
if r.lastindex > 2:
block_offset = int(r[3])
if r.lastindex > 3:
block_end = int(r[4])
if block_end < block_offset:
raise ValueError('cart before the horse, dude')
logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
return (offset, limit, block_offset, block_end,)
def process_default_limit(session, env):
r = re.match(re_default_limit, env.get('PATH_INFO'))
if not r:
return None
return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
def process_transactions_account_bloom(session, env):
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
if not r:
return None
logg.debug('match account bloom')
address = r[1]
if r[2] == None:
address = add_0x(address)
offset = 0
if r.lastindex > 2:
offset = r[4]
limit = DEFAULT_LIMIT
if r.lastindex > 4:
limit = r[6]
(address, offset, limit,) = parse_query_account(r)
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
@@ -59,13 +108,9 @@ def process_transactions_all_bloom(session, env):
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
if not r:
return None
logg.debug('match all bloom')
offset = DEFAULT_LIMIT
if r.lastindex > 0:
offset = r[1]
limit = 0
if r.lastindex > 1:
limit = r[2]
(limit, offset, block_offset, block_end,) = parse_query_any(r)
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
@@ -88,17 +133,16 @@ def process_transactions_all_data(session, env):
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
if not r:
return None
if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
return None
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
# return None
logg.debug('match all data')
logg.debug('got data request {}'.format(env))
block_offset = r[1]
block_end = r[2]
if int(r[2]) < int(r[1]):
raise ValueError('cart before the horse, dude')
(offset, limit, block_offset, block_end) = parse_query_any(r)
c = DataCache(session)
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
for r in tx_cache:
r['date_block'] = r['date_block'].timestamp()
@@ -113,3 +157,30 @@ def process_transactions_all_data(session, env):
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_account_data(session, env):
r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
if not r:
return None
logg.debug('match account data')
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
# return None
(address, offset, limit,) = parse_query_account(r)
c = DataCache(session)
(lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
for r in tx_cache:
r['date_block'] = r['date_block'].timestamp()
o = {
'low': lowest_block,
'high': highest_block,
'data': tx_cache,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)

17
apps/cic-cache/cic_cache/runnable/daemons/server.py
View File

@@ -12,21 +12,20 @@ import cic_cache.cli
from cic_cache.db import dsn_from_config
from cic_cache.db.models.base import SessionBase
from cic_cache.runnable.daemons.query import (
process_default_limit,
process_transactions_account_bloom,
process_transactions_account_data,
process_transactions_all_bloom,
process_transactions_all_data,
)
import cic_cache.cli
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
migrationsdir = os.path.join(dbdir, 'migrations')
# process args
arg_flags = cic_cache.cli.argflag_std_base
local_arg_flags = cic_cache.cli.argflag_local_task
arg_flags = cic_cache.cli.argflag_std_read
local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task
argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
@@ -35,7 +34,7 @@ args = argparser.parse_args()
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
# connect to database
dsn = dsn_from_config(config)
dsn = dsn_from_config(config, 'cic_cache')
SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
@@ -47,9 +46,11 @@ def application(env, start_response):
session = SessionBase.create_session()
for handler in [
process_transactions_account_data,
process_transactions_account_bloom,
process_transactions_all_data,
process_transactions_all_bloom,
process_transactions_account_bloom,
process_default_limit,
]:
r = None
try:

3
apps/cic-cache/cic_cache/runnable/daemons/tasker.py
View File

@@ -3,6 +3,7 @@ import logging
import os
import sys
import argparse
import tempfile
# third-party imports
import celery
@@ -28,7 +29,7 @@ args = argparser.parse_args()
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
# connect to database
dsn = dsn_from_config(config)
dsn = dsn_from_config(config, 'cic_cache')
SessionBase.connect(dsn)
# set up celery

2
apps/cic-cache/cic_cache/runnable/daemons/tracker.py
View File

@@ -50,7 +50,7 @@ args = argparser.parse_args()
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
# connect to database
dsn = dsn_from_config(config)
dsn = dsn_from_config(config, 'cic_cache')
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
# set up rpc

2
apps/cic-cache/cic_cache/version.py
View File

@@ -5,7 +5,7 @@ version = (
0,
2,
1,
'alpha.2',
'alpha.3',
)
version_object = semver.VersionInfo(