Only fetch metadata for each wallet address once

clicada/cli/user.py

@@ -55,6 +55,8 @@ def extra_args():
 def apply_args(config, args):
     if config.get('META_LOOKUP_METHOD'):
         raise NotImplementedError('Sorry, currently only "phone" lookup method is implemented')
+    if config.true('_FORCE_ALL'):
+        config.add(True, '_FORCE', exists_ok=True)
 
 
 def validate(config, args):
@@ -122,19 +124,36 @@ Tags: {}""".format(
     if ctrl.get('_RAW_TX'):
         raw_rpc = ctrl.rpc 
 
-    tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
-    tx_lines = []
-    seen_tokens = {}
-
-    tx_threads = []
-
     token_resolver_queue = Queue()
     token_result_queue = Queue()
-    tx_queue = Queue()
-
     token_resolver_worker = TokenResolverWorker(user_address, ctrl, token_store, token_resolver_queue, token_result_queue)
     token_resolver_worker.start()
 
+    wallets = []
+    for tx in txs['data']:
+        token_resolver_queue.put_nowait(tx['source_token'])
+        token_resolver_queue.put_nowait(tx['destination_token'])
+        if tx['sender'] not in wallets:
+            logg.info('adding wallet {} to metadata lookup'.format(tx['sender']))
+            wallets.append(tx['sender'])
+        if tx['recipient'] not in wallets:
+            wallets.append(tx['recipient'])
+            logg.info('registered wallet {} for metadata lookup'.format(tx['recipient']))
+
+    wallet_threads = []
+    for a in wallets:
+        thread_wallet = MetadataResolverWorker(a, ctrl, user_address_store)
+        thread_wallet.start()
+        wallet_threads.append(thread_wallet)
+    
+    ctrl.notify('wait for metadata resolvers to finish work')
+    for t in wallet_threads:
+        t.join()
+
+    tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
+    tx_threads = []
+    tx_queue = Queue()
+
     tx_n = 0
     for tx_src in txs['data']:
         tx_hash = strip_0x(tx_src['tx_hash'])
@@ -182,6 +201,24 @@ Tags: {}""".format(
         ctrl.write(tx_buf[k])
 
 
+class MetadataResolverWorker(threading.Thread):
+    
+    def __init__(self, wallet_address, ctrl, user_address_store):
+        self.user_address_store = user_address_store
+        self.wallet_address = wallet_address
+        #tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update)
+        self.ctrl = ctrl
+        super(MetadataResolverWorker, self).__init__()
+        
+
+    def run(self):
+        self.ctrl.notify('resolve metadata for {}'.format(self.wallet_address))
+        try:
+            self.user_address_store.by_address(self.wallet_address)
+        except MetadataNotFoundError:
+            logg.info('failed metadata lookup for {}'.format(self.wallet_address))
+
+
 class TxResolverWorker(threading.Thread):
     def __init__(self, tx_hash, tx_src, ctrl, tx_store, token_store, user_address_store, token_queue, tx_queue, show_decimals=True, update=None):
         self.tx_hash = tx_hash
@@ -200,13 +237,11 @@ class TxResolverWorker(threading.Thread):
     def run(self):
         self.ctrl.notify('resolve details for tx {}'.format(self.tx_hash))
         tx = ResolvedTokenTx.from_dict(self.tx_src)
-        tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update)
+        tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update, lookup=False)
-        self.token_queue.put_nowait(tx.source_token)
+        #tx.resolve_tokens(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update)
-        self.token_queue.put_nowait(tx.destination_token)
+        #self.token_queue.put_nowait(tx.source_token)
-        try:
+        #self.token_queue.put_nowait(tx.destination_token)
         self.tx_store.put(self.tx_hash, str(self.tx_src), overwrite=self.ctrl.get('_FORCE_ALL'))
-        except FileExistsError:
-            logg.debug('skip put of already existing tx entry {}'.format(self.tx_hash))
         self.tx_queue.put(tx)
 
 

clicada/token/token.py

@@ -52,7 +52,7 @@ class FileTokenStore:
         return p
    
 
-    def by_address(self, address):
+    def by_address(self, address, update=False, lookup=True):
         address = tx_normalize.executable_address(address)
 
         token_symbol = self.memstore_symbol.get(address)
@@ -65,6 +65,19 @@ class FileTokenStore:
         try:
             f = open(p, 'r')
         except FileNotFoundError:
+            pass
+
+        if f == None:
+            if not lookup:
+                token_symbol = '???'
+                token_decimals = '???'
+            #self.memstore_symbol.put(address, token_symbol)
+            #self.memstore_decimals.put(address, token_decimals)
+            #logg.warning('token metadata not found and lookup deactivated. Will use 18 decimals as default')
+            #return (token_symbol, token_decimals,)
+
+        if token_symbol == None:
+            if f == None:
                 p = self.__cache_token(address)
                 f = open(p, 'r')
            

clicada/tx/tx.py

@@ -16,12 +16,16 @@ from clicada.error import (
         ExpiredRecordError,
         MetadataNotFoundError,
         )
+from chainlib.eth.address import AddressChecksum
 
 logg = logging.getLogger(__name__)
 
+address_checksummer = AddressChecksum()
+
 
 class ResolvedTokenTx(TokenTx):
 
+
     def __init__(self):
         super(ResolvedTokenTx, self).__init__()
         self.source_token_name = None
@@ -33,8 +37,8 @@ class ResolvedTokenTx(TokenTx):
         self.recipient_entity = None
 
 
-    def resolve_tokens(self, token_store, show_decimals=False, update=False):
+    def resolve_tokens(self, token_store, show_decimals=False, update=False, lookup=False):
-        (token_symbol, token_decimals) = token_store.by_address(self.source_token)
+        (token_symbol, token_decimals) = token_store.by_address(self.source_token, lookup=False)
         self.source_token_decimals = token_decimals
         self.source_token_label = token_symbol
         token_value = self.to_value / (10 ** token_decimals)
@@ -59,31 +63,31 @@ class ResolvedTokenTx(TokenTx):
         self.to_value_label = fmt.format(token_value)
 
 
-    def resolve_entity(self, user_store, address):
+    def resolve_entity(self, user_store, address, update=False, lookup=True):
         try:
-            r = user_store.by_address(address)
+            r = user_store.by_address(address, update=update, lookup=lookup)
         except MetadataNotFoundError:
-            return address
+            return address_checksummer.sum(address)
         return str(r)
 
 
-    def resolve_sender_entity(self, user_store, update=False):
+    def resolve_sender_entity(self, user_store, update=False, lookup=True):
         if self.tx_type == TokenTxType.faucet_giveto.value:
             return 'FAUCET'
-        return self.resolve_entity(user_store, self.sender)
+        return self.resolve_entity(user_store, self.sender, update=update, lookup=lookup)
         
 
-    def resolve_recipient_entity(self, user_store, update=False):
+    def resolve_recipient_entity(self, user_store, update=False, lookup=True):
-        return self.resolve_entity(user_store, self.recipient)
+        return self.resolve_entity(user_store, self.recipient, update=update, lookup=lookup)
 
 
-    def resolve_entities(self, user_store, update=False):
+    def resolve_entities(self, user_store, update=False, lookup=True):
-        self.sender_label = self.resolve_sender_entity(user_store, update=update)
+        self.sender_label = self.resolve_sender_entity(user_store, update=update, lookup=lookup)
-        self.recipient_label = self.resolve_recipient_entity(user_store, update=update)
+        self.recipient_label = self.resolve_recipient_entity(user_store, update=update, lookup=lookup)
 
 
-    def resolve(self, token_store, user_store, show_decimals=False, update=False):
+    def resolve(self, token_store, user_store, show_decimals=False, update=False, lookup=True):
-        self.resolve_tokens(token_store, show_decimals, update=update)
+        self.resolve_tokens(token_store, show_decimals, update=update, lookup=lookup)
         self.resolve_entities(user_store, update=update)
 
 

clicada/user/file.py

@@ -241,13 +241,13 @@ class FileUserStore:
         return person_data
 
 
-    def by_address(self, address, update=False):
+    def by_address(self, address, update=False, lookup=True):
         address = tx_normalize.wallet_address(address)
         address = strip_0x(address)
         #if self.failed_entities.get(address):
         if self.is_dud(address):
             logg.debug('already tried and failed {}, skipping'.format(address))
-            return address
+            raise MetadataNotFoundError()
 
         ignore_expired = self.sticky(address)
 
@@ -281,6 +281,7 @@ class FileUserStore:
         person = Account()
         person_data = person.deserialize(person_data=data)
 
+        logg.debug('wallet {} resolved to {}'.format(address, str(person)))
         self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
         r = None