Add raw tx filestore, optional retrieve and cache of raw tx

This commit is contained in:
lash 2022-01-22 10:43:46 +00:00
parent 265a53f9ca
commit b44a50aaa2
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
4 changed files with 50 additions and 9 deletions

View File

@ -158,6 +158,8 @@ class CmdCtrl:
r = self.config.get(k, default) r = self.config.get(k, default)
if k in [ if k in [
'_FORCE', '_FORCE',
'_FORCE_ALL',
'_RAW_TX',
]: ]:
if r == None: if r == None:
return False return False

View File

@ -34,12 +34,16 @@ def process_args(argparser):
argparser.add_argument('-m', '--method', type=str, help='lookup method') argparser.add_argument('-m', '--method', type=str, help='lookup method')
argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from') argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries') argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries')
argparser.add_argument('-ff', '--force-update-all', dest='force_update_all', action='store_true', help='Update records of mutable entries and immutable entries')
argparser.add_argument('--raw-tx', dest='raw_tx', action='store_true', help='Also cache raw transaction data')
argparser.add_argument('identifier', type=str, help='user identifier') argparser.add_argument('identifier', type=str, help='user identifier')
def extra_args(): def extra_args():
return { return {
'raw_tx': '_RAW_TX',
'force_update': '_FORCE', 'force_update': '_FORCE',
'force_update_all': '_FORCE_ALL',
'method': 'META_LOOKUP_METHOD', 'method': 'META_LOOKUP_METHOD',
'meta_url': 'META_URL', 'meta_url': 'META_URL',
'identifier': '_IDENTIFIER', 'identifier': '_IDENTIFIER',
@ -60,7 +64,7 @@ def execute(ctrl):
store_path = '.clicada' store_path = '.clicada'
user_phone_file_label = 'phone' user_phone_file_label = 'phone'
user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter) user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER'))) ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE')) user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
@ -82,7 +86,7 @@ def execute(ctrl):
token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
user_address_file_label = 'address' user_address_file_label = 'address'
user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter) user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
ctrl.notify('resolving metadata for address {}'.format(user_address_normal)) ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
try: try:
@ -112,7 +116,11 @@ Tags: {}""".format(
) )
) )
tx_store = FileTxStore(store_path) raw_rpc = None
if ctrl.get('_RAW_TX'):
raw_rpc = ctrl.rpc
tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
tx_lines = [] tx_lines = []
seen_tokens = {} seen_tokens = {}
for tx_src in txs['data']: for tx_src in txs['data']:
@ -123,9 +131,9 @@ Tags: {}""".format(
tx_lines.append(tx) tx_lines.append(tx)
seen_tokens[tx.source_token_label] = tx.source_token seen_tokens[tx.source_token_label] = tx.source_token
seen_tokens[tx.destination_token_label] = tx.destination_token seen_tokens[tx.destination_token_label] = tx.destination_token
tx_store.put(tx_hash, str(tx_src)) tx_store.put(tx_hash, str(tx_src), overwrite=ctrl.get('_FORCE_ALL'))
ctrl.write("Balances:")
for k in seen_tokens.keys(): for k in seen_tokens.keys():
ctrl.notify('resolve token {}'.format(seen_tokens[k])) ctrl.notify('resolve token {}'.format(seen_tokens[k]))
(token_symbol, token_decimals) = token_store.by_address(seen_tokens[k]) (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
@ -133,7 +141,7 @@ Tags: {}""".format(
balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address) balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
fmt = '{:.' + str(token_decimals) + 'f}' fmt = '{:.' + str(token_decimals) + 'f}'
decimal_balance = fmt.format(balance / (10 ** token_decimals)) decimal_balance = fmt.format(balance / (10 ** token_decimals))
ctrl.write("Balances:\n {} {}".format(token_symbol, decimal_balance)) ctrl.write(" {} {}".format(token_symbol, decimal_balance))
print() print()
for l in tx_lines: for l in tx_lines:

View File

@ -1,20 +1,45 @@
# standard imports
import os import os
import logging
# external imports
from chainlib.eth.tx import transaction
from leveldir.numeric import NumDir from leveldir.numeric import NumDir
from leveldir.hex import HexDir from leveldir.hex import HexDir
from hexathon import strip_0x
logg = logging.getLogger(__name__)
class FileTxStore: class FileTxStore:
subdivision = 100000 subdivision = 100000
def __init__(self, store_base_path): def __init__(self, store_base_path, rpc=None, notifier=None):
tx_base_path = os.path.join(store_base_path, 'tx') tx_base_path = os.path.join(store_base_path, 'tx')
num_base_path = os.path.join(tx_base_path, 'blocks') num_base_path = os.path.join(tx_base_path, 'blocks')
hash_base_path = os.path.join(tx_base_path, 'hash') hash_base_path = os.path.join(tx_base_path, 'hash')
raw_base_path = os.path.join(tx_base_path, 'raw')
self.block_index_dir = NumDir(num_base_path) self.block_index_dir = NumDir(num_base_path)
self.hash_index_dir = HexDir(hash_base_path, 32) self.hash_index_dir = HexDir(hash_base_path, 32)
self.raw_index_dir = HexDir(raw_base_path, 32)
self.rpc = rpc
self.notifier = notifier
def put(self, k, v): def put(self, k, v, overwrite=False):
if self.notifier != None:
self.notifier.notify('caching tx data for {}'.format(k))
hsh = bytes.fromhex(k) hsh = bytes.fromhex(k)
if not overwrite and self.hash_index_dir.have(k):
logg.debug('tx store already has {}'.format(k))
return
self.hash_index_dir.add(hsh, v.encode('utf-8')) self.hash_index_dir.add(hsh, v.encode('utf-8'))
if self.rpc != None:
self.notifier.notify('retrieve and cache raw tx data for {}'.format(k))
o = transaction(k)
r = self.rpc.conn.do(o)
raw = bytes.fromhex(strip_0x(r['raw']))
self.raw_index_dir.add(hsh, raw)

View File

@ -65,7 +65,7 @@ class Account(Person):
class FileUserStore: class FileUserStore:
def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None): def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None, notifier=None):
invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl) invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
self.invalidate_before = int(invalidate_before.timestamp()) self.invalidate_before = int(invalidate_before.timestamp())
self.have_xattr = False self.have_xattr = False
@ -83,6 +83,7 @@ class FileUserStore:
self.metadata_opener = metadata_opener self.metadata_opener = metadata_opener
self.failed_entities = {} self.failed_entities = {}
self.encrypter = encrypter self.encrypter = encrypter
self.notifier = notifier
def __validate_dir(self): def __validate_dir(self):
@ -214,6 +215,8 @@ class FileUserStore:
logg.info(e) logg.info(e)
pass pass
self.notifier.notify('wallet address for phone {} not found locally, retrieve from metadata service'.format(phone))
getter = self.metadata_opener getter = self.metadata_opener
ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
r = None r = None
@ -259,6 +262,8 @@ class FileUserStore:
logg.info(e) logg.info(e)
pass pass
self.notifier.notify('metadata for wallet {} not found locally, retrieve from metadata service'.format(address))
getter = self.metadata_opener getter = self.metadata_opener
ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON) ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@ -276,6 +281,7 @@ class FileUserStore:
person = Account() person = Account()
person_data = person.deserialize(person_data=data) person_data = person.deserialize(person_data=data)
self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM) ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
r = None r = None
try: try: