feat: Threaded resolution of txs and tokens #12
@ -55,6 +55,8 @@ def extra_args():
|
|||||||
def apply_args(config, args):
|
def apply_args(config, args):
|
||||||
if config.get('META_LOOKUP_METHOD'):
|
if config.get('META_LOOKUP_METHOD'):
|
||||||
raise NotImplementedError('Sorry, currently only "phone" lookup method is implemented')
|
raise NotImplementedError('Sorry, currently only "phone" lookup method is implemented')
|
||||||
|
if config.true('_FORCE_ALL'):
|
||||||
|
config.add(True, '_FORCE', exists_ok=True)
|
||||||
|
|
||||||
|
|
||||||
def validate(config, args):
|
def validate(config, args):
|
||||||
@ -122,19 +124,36 @@ Tags: {}""".format(
|
|||||||
if ctrl.get('_RAW_TX'):
|
if ctrl.get('_RAW_TX'):
|
||||||
raw_rpc = ctrl.rpc
|
raw_rpc = ctrl.rpc
|
||||||
|
|
||||||
tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
|
|
||||||
tx_lines = []
|
|
||||||
seen_tokens = {}
|
|
||||||
|
|
||||||
tx_threads = []
|
|
||||||
|
|
||||||
token_resolver_queue = Queue()
|
token_resolver_queue = Queue()
|
||||||
token_result_queue = Queue()
|
token_result_queue = Queue()
|
||||||
tx_queue = Queue()
|
|
||||||
|
|
||||||
token_resolver_worker = TokenResolverWorker(user_address, ctrl, token_store, token_resolver_queue, token_result_queue)
|
token_resolver_worker = TokenResolverWorker(user_address, ctrl, token_store, token_resolver_queue, token_result_queue)
|
||||||
token_resolver_worker.start()
|
token_resolver_worker.start()
|
||||||
|
|
||||||
|
wallets = []
|
||||||
|
for tx in txs['data']:
|
||||||
|
token_resolver_queue.put_nowait(tx['source_token'])
|
||||||
|
token_resolver_queue.put_nowait(tx['destination_token'])
|
||||||
|
if tx['sender'] not in wallets:
|
||||||
|
logg.info('adding wallet {} to metadata lookup'.format(tx['sender']))
|
||||||
|
wallets.append(tx['sender'])
|
||||||
|
if tx['recipient'] not in wallets:
|
||||||
|
wallets.append(tx['recipient'])
|
||||||
|
logg.info('registered wallet {} for metadata lookup'.format(tx['recipient']))
|
||||||
|
|
||||||
|
wallet_threads = []
|
||||||
|
for a in wallets:
|
||||||
|
thread_wallet = MetadataResolverWorker(a, ctrl, user_address_store)
|
||||||
|
thread_wallet.start()
|
||||||
|
wallet_threads.append(thread_wallet)
|
||||||
|
|
||||||
|
ctrl.notify('wait for metadata resolvers to finish work')
|
||||||
|
for t in wallet_threads:
|
||||||
|
t.join()
|
||||||
|
|
||||||
|
tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
|
||||||
|
tx_threads = []
|
||||||
|
tx_queue = Queue()
|
||||||
|
|
||||||
tx_n = 0
|
tx_n = 0
|
||||||
for tx_src in txs['data']:
|
for tx_src in txs['data']:
|
||||||
tx_hash = strip_0x(tx_src['tx_hash'])
|
tx_hash = strip_0x(tx_src['tx_hash'])
|
||||||
@ -182,6 +201,24 @@ Tags: {}""".format(
|
|||||||
ctrl.write(tx_buf[k])
|
ctrl.write(tx_buf[k])
|
||||||
|
|
||||||
|
|
||||||
|
class MetadataResolverWorker(threading.Thread):
|
||||||
|
|
||||||
|
def __init__(self, wallet_address, ctrl, user_address_store):
|
||||||
|
self.user_address_store = user_address_store
|
||||||
|
self.wallet_address = wallet_address
|
||||||
|
#tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update)
|
||||||
|
self.ctrl = ctrl
|
||||||
|
super(MetadataResolverWorker, self).__init__()
|
||||||
|
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.ctrl.notify('resolve metadata for {}'.format(self.wallet_address))
|
||||||
|
try:
|
||||||
|
self.user_address_store.by_address(self.wallet_address)
|
||||||
|
except MetadataNotFoundError:
|
||||||
|
logg.info('failed metadata lookup for {}'.format(self.wallet_address))
|
||||||
|
|
||||||
|
|
||||||
class TxResolverWorker(threading.Thread):
|
class TxResolverWorker(threading.Thread):
|
||||||
def __init__(self, tx_hash, tx_src, ctrl, tx_store, token_store, user_address_store, token_queue, tx_queue, show_decimals=True, update=None):
|
def __init__(self, tx_hash, tx_src, ctrl, tx_store, token_store, user_address_store, token_queue, tx_queue, show_decimals=True, update=None):
|
||||||
self.tx_hash = tx_hash
|
self.tx_hash = tx_hash
|
||||||
@ -200,13 +237,11 @@ class TxResolverWorker(threading.Thread):
|
|||||||
def run(self):
|
def run(self):
|
||||||
self.ctrl.notify('resolve details for tx {}'.format(self.tx_hash))
|
self.ctrl.notify('resolve details for tx {}'.format(self.tx_hash))
|
||||||
tx = ResolvedTokenTx.from_dict(self.tx_src)
|
tx = ResolvedTokenTx.from_dict(self.tx_src)
|
||||||
tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update)
|
tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update, lookup=False)
|
||||||
self.token_queue.put_nowait(tx.source_token)
|
#tx.resolve_tokens(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update)
|
||||||
self.token_queue.put_nowait(tx.destination_token)
|
#self.token_queue.put_nowait(tx.source_token)
|
||||||
try:
|
#self.token_queue.put_nowait(tx.destination_token)
|
||||||
self.tx_store.put(self.tx_hash, str(self.tx_src), overwrite=self.ctrl.get('_FORCE_ALL'))
|
self.tx_store.put(self.tx_hash, str(self.tx_src), overwrite=self.ctrl.get('_FORCE_ALL'))
|
||||||
except FileExistsError:
|
|
||||||
logg.debug('skip put of already existing tx entry {}'.format(self.tx_hash))
|
|
||||||
self.tx_queue.put(tx)
|
self.tx_queue.put(tx)
|
||||||
|
|
||||||
|
|
||||||
|
@ -52,7 +52,7 @@ class FileTokenStore:
|
|||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
def by_address(self, address):
|
def by_address(self, address, update=False, lookup=True):
|
||||||
address = tx_normalize.executable_address(address)
|
address = tx_normalize.executable_address(address)
|
||||||
|
|
||||||
token_symbol = self.memstore_symbol.get(address)
|
token_symbol = self.memstore_symbol.get(address)
|
||||||
@ -65,17 +65,30 @@ class FileTokenStore:
|
|||||||
try:
|
try:
|
||||||
f = open(p, 'r')
|
f = open(p, 'r')
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
p = self.__cache_token(address)
|
pass
|
||||||
|
|
||||||
|
if f == None:
|
||||||
|
if not lookup:
|
||||||
|
token_symbol = '???'
|
||||||
|
token_decimals = '???'
|
||||||
|
#self.memstore_symbol.put(address, token_symbol)
|
||||||
|
#self.memstore_decimals.put(address, token_decimals)
|
||||||
|
#logg.warning('token metadata not found and lookup deactivated. Will use 18 decimals as default')
|
||||||
|
#return (token_symbol, token_decimals,)
|
||||||
|
|
||||||
|
if token_symbol == None:
|
||||||
|
if f == None:
|
||||||
|
p = self.__cache_token(address)
|
||||||
|
f = open(p, 'r')
|
||||||
|
|
||||||
|
token_symbol = f.read()
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
p = os.path.join(self.store_path, token_symbol)
|
||||||
f = open(p, 'r')
|
f = open(p, 'r')
|
||||||
|
r = f.read()
|
||||||
token_symbol = f.read()
|
f.close()
|
||||||
f.close()
|
token_decimals = int(r)
|
||||||
|
|
||||||
p = os.path.join(self.store_path, token_symbol)
|
|
||||||
f = open(p, 'r')
|
|
||||||
r = f.read()
|
|
||||||
f.close()
|
|
||||||
token_decimals = int(r)
|
|
||||||
|
|
||||||
self.memstore_symbol.put(address, token_symbol)
|
self.memstore_symbol.put(address, token_symbol)
|
||||||
self.memstore_decimals.put(token_symbol, token_decimals)
|
self.memstore_decimals.put(token_symbol, token_decimals)
|
||||||
|
@ -16,12 +16,16 @@ from clicada.error import (
|
|||||||
ExpiredRecordError,
|
ExpiredRecordError,
|
||||||
MetadataNotFoundError,
|
MetadataNotFoundError,
|
||||||
)
|
)
|
||||||
|
from chainlib.eth.address import AddressChecksum
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
address_checksummer = AddressChecksum()
|
||||||
|
|
||||||
|
|
||||||
class ResolvedTokenTx(TokenTx):
|
class ResolvedTokenTx(TokenTx):
|
||||||
|
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(ResolvedTokenTx, self).__init__()
|
super(ResolvedTokenTx, self).__init__()
|
||||||
self.source_token_name = None
|
self.source_token_name = None
|
||||||
@ -33,8 +37,8 @@ class ResolvedTokenTx(TokenTx):
|
|||||||
self.recipient_entity = None
|
self.recipient_entity = None
|
||||||
|
|
||||||
|
|
||||||
def resolve_tokens(self, token_store, show_decimals=False, update=False):
|
def resolve_tokens(self, token_store, show_decimals=False, update=False, lookup=False):
|
||||||
(token_symbol, token_decimals) = token_store.by_address(self.source_token)
|
(token_symbol, token_decimals) = token_store.by_address(self.source_token, lookup=False)
|
||||||
self.source_token_decimals = token_decimals
|
self.source_token_decimals = token_decimals
|
||||||
self.source_token_label = token_symbol
|
self.source_token_label = token_symbol
|
||||||
token_value = self.to_value / (10 ** token_decimals)
|
token_value = self.to_value / (10 ** token_decimals)
|
||||||
@ -59,31 +63,31 @@ class ResolvedTokenTx(TokenTx):
|
|||||||
self.to_value_label = fmt.format(token_value)
|
self.to_value_label = fmt.format(token_value)
|
||||||
|
|
||||||
|
|
||||||
def resolve_entity(self, user_store, address):
|
def resolve_entity(self, user_store, address, update=False, lookup=True):
|
||||||
try:
|
try:
|
||||||
r = user_store.by_address(address)
|
r = user_store.by_address(address, update=update, lookup=lookup)
|
||||||
except MetadataNotFoundError:
|
except MetadataNotFoundError:
|
||||||
return address
|
return address_checksummer.sum(address)
|
||||||
return str(r)
|
return str(r)
|
||||||
|
|
||||||
|
|
||||||
def resolve_sender_entity(self, user_store, update=False):
|
def resolve_sender_entity(self, user_store, update=False, lookup=True):
|
||||||
if self.tx_type == TokenTxType.faucet_giveto.value:
|
if self.tx_type == TokenTxType.faucet_giveto.value:
|
||||||
return 'FAUCET'
|
return 'FAUCET'
|
||||||
return self.resolve_entity(user_store, self.sender)
|
return self.resolve_entity(user_store, self.sender, update=update, lookup=lookup)
|
||||||
|
|
||||||
|
|
||||||
def resolve_recipient_entity(self, user_store, update=False):
|
def resolve_recipient_entity(self, user_store, update=False, lookup=True):
|
||||||
return self.resolve_entity(user_store, self.recipient)
|
return self.resolve_entity(user_store, self.recipient, update=update, lookup=lookup)
|
||||||
|
|
||||||
|
|
||||||
def resolve_entities(self, user_store, update=False):
|
def resolve_entities(self, user_store, update=False, lookup=True):
|
||||||
self.sender_label = self.resolve_sender_entity(user_store, update=update)
|
self.sender_label = self.resolve_sender_entity(user_store, update=update, lookup=lookup)
|
||||||
self.recipient_label = self.resolve_recipient_entity(user_store, update=update)
|
self.recipient_label = self.resolve_recipient_entity(user_store, update=update, lookup=lookup)
|
||||||
|
|
||||||
|
|
||||||
def resolve(self, token_store, user_store, show_decimals=False, update=False):
|
def resolve(self, token_store, user_store, show_decimals=False, update=False, lookup=True):
|
||||||
self.resolve_tokens(token_store, show_decimals, update=update)
|
self.resolve_tokens(token_store, show_decimals, update=update, lookup=lookup)
|
||||||
self.resolve_entities(user_store, update=update)
|
self.resolve_entities(user_store, update=update)
|
||||||
|
|
||||||
|
|
||||||
|
@ -241,13 +241,13 @@ class FileUserStore:
|
|||||||
return person_data
|
return person_data
|
||||||
|
|
||||||
|
|
||||||
def by_address(self, address, update=False):
|
def by_address(self, address, update=False, lookup=True):
|
||||||
address = tx_normalize.wallet_address(address)
|
address = tx_normalize.wallet_address(address)
|
||||||
address = strip_0x(address)
|
address = strip_0x(address)
|
||||||
#if self.failed_entities.get(address):
|
#if self.failed_entities.get(address):
|
||||||
if self.is_dud(address):
|
if self.is_dud(address):
|
||||||
logg.debug('already tried and failed {}, skipping'.format(address))
|
logg.debug('already tried and failed {}, skipping'.format(address))
|
||||||
return address
|
raise MetadataNotFoundError()
|
||||||
|
|
||||||
ignore_expired = self.sticky(address)
|
ignore_expired = self.sticky(address)
|
||||||
|
|
||||||
@ -281,6 +281,7 @@ class FileUserStore:
|
|||||||
person = Account()
|
person = Account()
|
||||||
person_data = person.deserialize(person_data=data)
|
person_data = person.deserialize(person_data=data)
|
||||||
|
|
||||||
|
logg.debug('wallet {} resolved to {}'.format(address, str(person)))
|
||||||
self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
|
self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
|
||||||
ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
|
ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
|
||||||
r = None
|
r = None
|
||||||
|
Loading…
Reference in New Issue
Block a user