usability: WIP Friendly progress output (#4)

This MR adds colorized progress statements when resolving and retrieving data for user. It disables logging if loglevel is set to default (logging.WARNING at this time).

It also skips metadata lookups that have failed in the same session, speeding up retrievals when same address repeatedly occurs in transaction list.

closes #6
closes #5

Co-authored-by: lash <dev@holbrook.no>
Reviewed-on: #4
Co-authored-by: lash <accounts-grassrootseconomics@holbrook.no>
Co-committed-by: lash <accounts-grassrootseconomics@holbrook.no>

CHANGELOG

@@ -1,3 +1,10 @@
+- 0.0.5
+	* Replace logs with colorized progress output on default loglevel
+	* Do not repeat already failed metadata lookups
+- 0.0.4
+	* Resolve metadata to labels when loading from cache
+- 0.0.3
+	* Upgrade usumbufu to prevent missing bearer auth on https
 - 0.0.2
 	* Use ~/.config for default config override
 - 0.0.1-unreleased

clicada/cli/arg.py

@@ -1,8 +1,13 @@
+# import notifier
+from clicada.cli.notify import NotifyWriter
+notifier = NotifyWriter()
+notifier.notify('loading script')
+
 # standard imports
 import os
-#import argparse
 import logging
 import importlib
+import sys
 
 # external imports
 import confini
@@ -25,6 +30,20 @@ data_dir = os.path.join(script_dir, '..', 'data')
 base_config_dir = os.path.join(data_dir, 'config')
 
 
+class NullWriter:
+
+    def notify(self, v):
+        pass
+
+
+    def ouch(self, v):
+        pass
+
+
+    def write(self, v):
+        sys.stdout.write(str(v))
+
+
 class CmdCtrl:
 
     __cmd_alias = {
@@ -45,10 +64,12 @@ class CmdCtrl:
        
         self.config()
 
+        self.notifier()
+
         self.auth()
 
         self.blockchain()
-        
+
         self.remote_openers = {}
         if self.get('META_URL') != None:
             auth_client_session = PGPClientSession(self.__auth)
@@ -156,3 +177,27 @@ class CmdCtrl:
 
     def opener(self, k):
         return self.remote_openers[k]
+
+
+    def notifier(self):
+        if logg.root.level >= logging.WARNING:
+            logging.disable()
+            self.writer = notifier
+        else:
+            self.writer = NullWriter()
+
+
+    def notify(self, v):
+        self.writer.notify(v)
+
+
+    def ouch(self, v):
+        self.writer.ouch(v)
+        print()
+
+
+    def write(self, v):
+        self.writer.write("")
+        self.writer.write(v)
+        print()
+

clicada/cli/notify.py

@@ -0,0 +1,32 @@
+# standard imports
+import os
+import sys
+
+
+class NotifyWriter:
+
+    def __init__(self, writer=sys.stdout):
+        (c, r) = os.get_terminal_size()
+        self.cols = c
+        self.fmt = "\r{:" + "<{}".format(c) + "}"
+        self.w = writer
+        self.notify_max = self.cols - 4
+
+    
+    def notify(self, v):
+        if len(v) > self.notify_max:
+            v = v[:self.notify_max]
+        self.write('\x1b[0;36m... ' + v + '\x1b[0;39m')
+
+
+    def ouch(self, v):
+        if len(v) > self.notify_max:
+            v = v[:self.notify_max]
+        self.write('\x1b[0;91m!!! ' + v + '\x1b[0;39m')
+
+
+    def write(self, v):
+        s = str(v)
+        if len(s) > self.cols:
+            s = s[:self.cols]
+        self.w.write(self.fmt.format(s))

clicada/cli/user.py

@@ -19,6 +19,7 @@ from clicada.token import (
         token_balance,
         )
 from clicada.tx import ResolvedTokenTx
+from clicada.error import MetadataNotFoundError
 
 logg = logging.getLogger(__name__)
 
@@ -51,24 +52,27 @@ def validate(config, args):
 
 
 def execute(ctrl):
-    tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'))
+    tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'), 10)
 
     store_path = '.clicada'
     user_phone_file_label = 'phone'
     user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
 
+    ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
     user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
     if user_address == None:
-        sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
+        ctrl.ouch('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
         sys.exit(1)
     try:
         user_address = to_checksum_address(user_address)
     except ValueError:
-        sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
+        ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
         sys.exit(1)
 
     logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER')))
 
+    user_address_normal = tx_normalizer.wallet_address(user_address)
+    ctrl.notify('retrieving txs for address {}'.format(user_address_normal))
     txs = tx_getter.get(user_address)
 
     token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
@@ -76,12 +80,14 @@ def execute(ctrl):
     user_address_file_label = 'address'
     user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
 
-    user_address_normal = tx_normalizer.wallet_address(user_address)
+    ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
+    try:
+        r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
+    except MetadataNotFoundError as e:
+        ctrl.ouch('could not resolve metadata for user: {}'.format(e))
+        sys.exit(1)
 
-    r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
+    ctrl.write("""Phone: {}
-    print('r {}'.format(r))
-
-    print("""Phone: {}
 Network address: {}
 Chain: {}
 Name: {}
@@ -89,8 +95,7 @@ Registered: {}
 Gender: {}
 Location: {}
 Products: {}
-Tags: {}
+Tags: {}""".format(
-Balances:""".format(
     ctrl.get('_IDENTIFIER'),
     add_0x(user_address),
     ctrl.chain().common_name(),
@@ -106,6 +111,7 @@ Balances:""".format(
     tx_lines = []
     seen_tokens = {}
     for tx_src in txs['data']:
+        ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash']))
         tx = ResolvedTokenTx.from_dict(tx_src)
         tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
         tx_lines.append(tx)
@@ -113,12 +119,14 @@ Balances:""".format(
         seen_tokens[tx.destination_token_label] = tx.destination_token
 
     for k in seen_tokens.keys():
+        ctrl.notify('resolve token {}'.format(seen_tokens[k]))
         (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
+        ctrl.notify('get token balance for {} => {}'.format(token_symbol, seen_tokens[k]))
         balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
         fmt =  '{:.' + str(token_decimals) + 'f}'
         decimal_balance = fmt.format(balance / (10 ** token_decimals))
-        print("\t{} {}".format(token_symbol, decimal_balance))
+        ctrl.write("Balances:\n    {} {}".format(token_symbol, decimal_balance))
 
     print()
     for l in tx_lines:
-        print(l)
+        ctrl.write(l)

clicada/error.py

@@ -4,3 +4,7 @@ class ExpiredRecordError(Exception):
 
 class AuthError(Exception):
     pass
+
+
+class MetadataNotFoundError(Exception):
+    pass

clicada/tx/tx.py

@@ -12,7 +12,10 @@ from cic_types.models.tx import (
 
 # local imports
 from clicada.encode import tx_normalize
-from clicada.error import ExpiredRecordError
+from clicada.error import (
+        ExpiredRecordError,
+        MetadataNotFoundError,
+        )
 
 logg = logging.getLogger(__name__)
 
@@ -56,35 +59,22 @@ class ResolvedTokenTx(TokenTx):
         self.to_value_label = fmt.format(token_value)
 
 
-    def resolve_stored_entity(self, user_store, address, update=False):
+    def resolve_entity(self, user_store, address):
-        if update:
-            return None
-        address = tx_normalize.wallet_address(address)
         try:
-            v = user_store.get(address)
+            r = user_store.by_address(address)
-            return v
+        except MetadataNotFoundError:
-        except FileNotFoundError:
+            return address
-            return None
+        return str(r)
-        except ExpiredRecordError:
-            return None
 
 
     def resolve_sender_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.sender, update=update)
-        if v != None:
-            return v
         if self.tx_type == TokenTxType.faucet_giveto.value:
             return 'FAUCET'
-        r = user_store.by_address(self.sender)
+        return self.resolve_entity(user_store, self.sender)
-        return str(r)
+        
-
 
     def resolve_recipient_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.recipient, update=update)
+        return self.resolve_entity(user_store, self.recipient)
-        if v != None:
-            return v
-        r = user_store.by_address(self.recipient, update=update)
-        return str(r)
 
 
     def resolve_entities(self, user_store, update=False):
@@ -99,7 +89,7 @@ class ResolvedTokenTx(TokenTx):
 
     def __str__(self):
         if self.symmetric:
-            return '{}\t{} => {}\t{} {}'.format(
+            return '{}    {} => {}    {} {}'.format(
                     self.date_block_label,
                     self.sender_label,
                     self.recipient_label,

clicada/user/file.py

@@ -18,7 +18,10 @@ import phonenumbers
 # local imports
 from clicada.encode import tx_normalize
 from clicada.store.mem import MemDictStore
-from clicada.error import ExpiredRecordError
+from clicada.error import (
+        ExpiredRecordError,
+        MetadataNotFoundError,
+        )
 
 logg = logging.getLogger(__name__)
 
@@ -78,6 +81,7 @@ class FileUserStore:
         os.makedirs(self.store_path, exist_ok=True)
         self.__validate_dir()
         self.metadata_opener = metadata_opener
+        self.failed_entities = {}
 
 
     def __validate_dir(self):
@@ -86,6 +90,10 @@ class FileUserStore:
             logg.debug('using existing file store {} for {}'.format(self.store_path, self.label))
 
 
+    def is_dud(self, address):
+        return bool(self.failed_entities.get(address))
+
+
     def put(self, k, v, force=False):
         have_file = False
         p = os.path.join(self.store_path, k)
@@ -205,28 +213,37 @@ class FileUserStore:
         self.put(phone_file, user_address, force=update)
         return user_address
 
-    
+
+    def metadata_to_person(self, v):
+        person = Account()
+        try:
+            person_data = person.deserialize(person_data=v)
+        except Exception as e:
+            person_data = v
+        return person_data
+
+
     def by_address(self, address, update=False):
-        add = tx_normalize.wallet_address(address)
+        address = tx_normalize.wallet_address(address)
+        address = strip_0x(address)
+        #if self.failed_entities.get(address):
+        if self.is_dud(address):
+            logg.debug('already tried and failed {}, skipping'.format(address))
+            return address
+
         ignore_expired = self.sticky(address)
 
         if not update:
             try:
                 v = self.get(address, ignore_expired=ignore_expired)
                 v = json.loads(v)
-                person = Account()
+                return self.metadata_to_person(v)
-                try:
-                    person_data = person.deserialize(person_data=v)
-                except Exception as e:
-                    person_data = v
-                return person_data
             except FileNotFoundError:
                 pass
             except ExpiredRecordError as e:
                 logg.info(e)
                 pass
 
-        address = strip_0x(address)
         getter = self.metadata_opener
 
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@@ -235,7 +252,10 @@ class FileUserStore:
             r = getter.open(ptr)
         except Exception as e:
             logg.debug('no metadata found for {}: {}'.format(address, e))
-            return address
+        
+        if r == None:
+            self.failed_entities[address] = True
+            raise MetadataNotFoundError()
 
         data = json.loads(r)
         person = Account()

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = clicada
-version = 0.0.3
+version = 0.0.5a1
 description = CLI CRM tool for the cic-stack custodial wallet system
 author = Louis Holbrook
 author_email = dev@holbrook.no