From b0f8f39d15bfb8034f74c31e8ee9103d6a06b336 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 11:11:23 +0000 Subject: [PATCH] usability: WIP Friendly progress output (#4) This MR adds colorized progress statements when resolving and retrieving data for user. It disables logging if loglevel is set to default (logging.WARNING at this time). It also skips metadata lookups that have failed in the same session, speeding up retrievals when same address repeatedly occurs in transaction list. closes https://git.grassecon.net/grassrootseconomics/clicada/issues/6 closes https://git.grassecon.net/grassrootseconomics/clicada/issues/5 Co-authored-by: lash Reviewed-on: https://git.grassecon.net/grassrootseconomics/clicada/pulls/4 Co-authored-by: lash Co-committed-by: lash --- CHANGELOG | 7 +++++++ clicada/cli/arg.py | 49 +++++++++++++++++++++++++++++++++++++++++-- clicada/cli/notify.py | 32 ++++++++++++++++++++++++++++ clicada/cli/user.py | 32 +++++++++++++++++----------- clicada/error.py | 4 ++++ clicada/tx/tx.py | 36 ++++++++++++------------------- clicada/user/file.py | 42 +++++++++++++++++++++++++++---------- setup.cfg | 2 +- 8 files changed, 155 insertions(+), 49 deletions(-) create mode 100644 clicada/cli/notify.py diff --git a/CHANGELOG b/CHANGELOG index 41a8d68..b77a6f2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +- 0.0.5 + * Replace logs with colorized progress output on default loglevel + * Do not repeat already failed metadata lookups +- 0.0.4 + * Resolve metadata to labels when loading from cache +- 0.0.3 + * Upgrade usumbufu to prevent missing bearer auth on https - 0.0.2 * Use ~/.config for default config override - 0.0.1-unreleased diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index 5de057c..4c7564a 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -1,8 +1,13 @@ +# import notifier +from clicada.cli.notify import NotifyWriter +notifier = NotifyWriter() +notifier.notify('loading script') + # standard imports import os -#import argparse import logging import importlib +import sys # external imports import confini @@ -25,6 +30,20 @@ data_dir = os.path.join(script_dir, '..', 'data') base_config_dir = os.path.join(data_dir, 'config') +class NullWriter: + + def notify(self, v): + pass + + + def ouch(self, v): + pass + + + def write(self, v): + sys.stdout.write(str(v)) + + class CmdCtrl: __cmd_alias = { @@ -45,10 +64,12 @@ class CmdCtrl: self.config() + self.notifier() + self.auth() self.blockchain() - + self.remote_openers = {} if self.get('META_URL') != None: auth_client_session = PGPClientSession(self.__auth) @@ -156,3 +177,27 @@ class CmdCtrl: def opener(self, k): return self.remote_openers[k] + + + def notifier(self): + if logg.root.level >= logging.WARNING: + logging.disable() + self.writer = notifier + else: + self.writer = NullWriter() + + + def notify(self, v): + self.writer.notify(v) + + + def ouch(self, v): + self.writer.ouch(v) + print() + + + def write(self, v): + self.writer.write("") + self.writer.write(v) + print() + diff --git a/clicada/cli/notify.py b/clicada/cli/notify.py new file mode 100644 index 0000000..6b8d8ef --- /dev/null +++ b/clicada/cli/notify.py @@ -0,0 +1,32 @@ +# standard imports +import os +import sys + + +class NotifyWriter: + + def __init__(self, writer=sys.stdout): + (c, r) = os.get_terminal_size() + self.cols = c + self.fmt = "\r{:" + "<{}".format(c) + "}" + self.w = writer + self.notify_max = self.cols - 4 + + + def notify(self, v): + if len(v) > self.notify_max: + v = v[:self.notify_max] + self.write('\x1b[0;36m... ' + v + '\x1b[0;39m') + + + def ouch(self, v): + if len(v) > self.notify_max: + v = v[:self.notify_max] + self.write('\x1b[0;91m!!! ' + v + '\x1b[0;39m') + + + def write(self, v): + s = str(v) + if len(s) > self.cols: + s = s[:self.cols] + self.w.write(self.fmt.format(s)) diff --git a/clicada/cli/user.py b/clicada/cli/user.py index c035baf..c4a4794 100644 --- a/clicada/cli/user.py +++ b/clicada/cli/user.py @@ -19,6 +19,7 @@ from clicada.token import ( token_balance, ) from clicada.tx import ResolvedTokenTx +from clicada.error import MetadataNotFoundError logg = logging.getLogger(__name__) @@ -51,24 +52,27 @@ def validate(config, args): def execute(ctrl): - tx_getter = TxGetter(ctrl.get('TX_CACHE_URL')) + tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'), 10) store_path = '.clicada' user_phone_file_label = 'phone' user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) + ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER'))) user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE')) if user_address == None: - sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER'))) + ctrl.ouch('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER'))) sys.exit(1) try: user_address = to_checksum_address(user_address) except ValueError: - sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER'))) + ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER'))) sys.exit(1) logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER'))) + user_address_normal = tx_normalizer.wallet_address(user_address) + ctrl.notify('retrieving txs for address {}'.format(user_address_normal)) txs = tx_getter.get(user_address) token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) @@ -76,12 +80,14 @@ def execute(ctrl): user_address_file_label = 'address' user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) - user_address_normal = tx_normalizer.wallet_address(user_address) + ctrl.notify('resolving metadata for address {}'.format(user_address_normal)) + try: + r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE')) + except MetadataNotFoundError as e: + ctrl.ouch('could not resolve metadata for user: {}'.format(e)) + sys.exit(1) - r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE')) - print('r {}'.format(r)) - - print("""Phone: {} + ctrl.write("""Phone: {} Network address: {} Chain: {} Name: {} @@ -89,8 +95,7 @@ Registered: {} Gender: {} Location: {} Products: {} -Tags: {} -Balances:""".format( +Tags: {}""".format( ctrl.get('_IDENTIFIER'), add_0x(user_address), ctrl.chain().common_name(), @@ -106,6 +111,7 @@ Balances:""".format( tx_lines = [] seen_tokens = {} for tx_src in txs['data']: + ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash'])) tx = ResolvedTokenTx.from_dict(tx_src) tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE')) tx_lines.append(tx) @@ -113,12 +119,14 @@ Balances:""".format( seen_tokens[tx.destination_token_label] = tx.destination_token for k in seen_tokens.keys(): + ctrl.notify('resolve token {}'.format(seen_tokens[k])) (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k]) + ctrl.notify('get token balance for {} => {}'.format(token_symbol, seen_tokens[k])) balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address) fmt = '{:.' + str(token_decimals) + 'f}' decimal_balance = fmt.format(balance / (10 ** token_decimals)) - print("\t{} {}".format(token_symbol, decimal_balance)) + ctrl.write("Balances:\n {} {}".format(token_symbol, decimal_balance)) print() for l in tx_lines: - print(l) + ctrl.write(l) diff --git a/clicada/error.py b/clicada/error.py index e405133..f1ac2de 100644 --- a/clicada/error.py +++ b/clicada/error.py @@ -4,3 +4,7 @@ class ExpiredRecordError(Exception): class AuthError(Exception): pass + + +class MetadataNotFoundError(Exception): + pass diff --git a/clicada/tx/tx.py b/clicada/tx/tx.py index dad971b..2cbea44 100644 --- a/clicada/tx/tx.py +++ b/clicada/tx/tx.py @@ -12,7 +12,10 @@ from cic_types.models.tx import ( # local imports from clicada.encode import tx_normalize -from clicada.error import ExpiredRecordError +from clicada.error import ( + ExpiredRecordError, + MetadataNotFoundError, + ) logg = logging.getLogger(__name__) @@ -56,35 +59,22 @@ class ResolvedTokenTx(TokenTx): self.to_value_label = fmt.format(token_value) - def resolve_stored_entity(self, user_store, address, update=False): - if update: - return None - address = tx_normalize.wallet_address(address) + def resolve_entity(self, user_store, address): try: - v = user_store.get(address) - return v - except FileNotFoundError: - return None - except ExpiredRecordError: - return None + r = user_store.by_address(address) + except MetadataNotFoundError: + return address + return str(r) def resolve_sender_entity(self, user_store, update=False): - v = self.resolve_stored_entity(user_store, self.sender, update=update) - if v != None: - return v if self.tx_type == TokenTxType.faucet_giveto.value: return 'FAUCET' - r = user_store.by_address(self.sender) - return str(r) - + return self.resolve_entity(user_store, self.sender) + def resolve_recipient_entity(self, user_store, update=False): - v = self.resolve_stored_entity(user_store, self.recipient, update=update) - if v != None: - return v - r = user_store.by_address(self.recipient, update=update) - return str(r) + return self.resolve_entity(user_store, self.recipient) def resolve_entities(self, user_store, update=False): @@ -99,7 +89,7 @@ class ResolvedTokenTx(TokenTx): def __str__(self): if self.symmetric: - return '{}\t{} => {}\t{} {}'.format( + return '{} {} => {} {} {}'.format( self.date_block_label, self.sender_label, self.recipient_label, diff --git a/clicada/user/file.py b/clicada/user/file.py index 3eeda13..a84a4f9 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -18,7 +18,10 @@ import phonenumbers # local imports from clicada.encode import tx_normalize from clicada.store.mem import MemDictStore -from clicada.error import ExpiredRecordError +from clicada.error import ( + ExpiredRecordError, + MetadataNotFoundError, + ) logg = logging.getLogger(__name__) @@ -78,6 +81,7 @@ class FileUserStore: os.makedirs(self.store_path, exist_ok=True) self.__validate_dir() self.metadata_opener = metadata_opener + self.failed_entities = {} def __validate_dir(self): @@ -86,6 +90,10 @@ class FileUserStore: logg.debug('using existing file store {} for {}'.format(self.store_path, self.label)) + def is_dud(self, address): + return bool(self.failed_entities.get(address)) + + def put(self, k, v, force=False): have_file = False p = os.path.join(self.store_path, k) @@ -205,28 +213,37 @@ class FileUserStore: self.put(phone_file, user_address, force=update) return user_address - + + def metadata_to_person(self, v): + person = Account() + try: + person_data = person.deserialize(person_data=v) + except Exception as e: + person_data = v + return person_data + + def by_address(self, address, update=False): - add = tx_normalize.wallet_address(address) + address = tx_normalize.wallet_address(address) + address = strip_0x(address) + #if self.failed_entities.get(address): + if self.is_dud(address): + logg.debug('already tried and failed {}, skipping'.format(address)) + return address + ignore_expired = self.sticky(address) if not update: try: v = self.get(address, ignore_expired=ignore_expired) v = json.loads(v) - person = Account() - try: - person_data = person.deserialize(person_data=v) - except Exception as e: - person_data = v - return person_data + return self.metadata_to_person(v) except FileNotFoundError: pass except ExpiredRecordError as e: logg.info(e) pass - address = strip_0x(address) getter = self.metadata_opener ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON) @@ -235,7 +252,10 @@ class FileUserStore: r = getter.open(ptr) except Exception as e: logg.debug('no metadata found for {}: {}'.format(address, e)) - return address + + if r == None: + self.failed_entities[address] = True + raise MetadataNotFoundError() data = json.loads(r) person = Account() diff --git a/setup.cfg b/setup.cfg index 0f325fb..ecf7664 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = clicada -version = 0.0.3 +version = 0.0.5a1 description = CLI CRM tool for the cic-stack custodial wallet system author = Louis Holbrook author_email = dev@holbrook.no