diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index 2ea2cb2..5f82513 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -158,6 +158,8 @@ class CmdCtrl: r = self.config.get(k, default) if k in [ '_FORCE', + '_FORCE_ALL', + '_RAW_TX', ]: if r == None: return False diff --git a/clicada/cli/user.py b/clicada/cli/user.py index 2e591ac..41f2533 100644 --- a/clicada/cli/user.py +++ b/clicada/cli/user.py @@ -34,12 +34,16 @@ def process_args(argparser): argparser.add_argument('-m', '--method', type=str, help='lookup method') argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from') argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries') + argparser.add_argument('-ff', '--force-update-all', dest='force_update_all', action='store_true', help='Update records of mutable entries and immutable entries') + argparser.add_argument('--raw-tx', dest='raw_tx', action='store_true', help='Also cache raw transaction data') argparser.add_argument('identifier', type=str, help='user identifier') def extra_args(): return { + 'raw_tx': '_RAW_TX', 'force_update': '_FORCE', + 'force_update_all': '_FORCE_ALL', 'method': 'META_LOOKUP_METHOD', 'meta_url': 'META_URL', 'identifier': '_IDENTIFIER', @@ -60,7 +64,7 @@ def execute(ctrl): store_path = '.clicada' user_phone_file_label = 'phone' - user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter) + user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl) ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER'))) user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE')) @@ -82,7 +86,7 @@ def execute(ctrl): token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) user_address_file_label = 'address' - user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter) + user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl) ctrl.notify('resolving metadata for address {}'.format(user_address_normal)) try: @@ -112,7 +116,11 @@ Tags: {}""".format( ) ) - tx_store = FileTxStore(store_path) + raw_rpc = None + if ctrl.get('_RAW_TX'): + raw_rpc = ctrl.rpc + + tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl) tx_lines = [] seen_tokens = {} for tx_src in txs['data']: @@ -123,9 +131,9 @@ Tags: {}""".format( tx_lines.append(tx) seen_tokens[tx.source_token_label] = tx.source_token seen_tokens[tx.destination_token_label] = tx.destination_token - tx_store.put(tx_hash, str(tx_src)) - + tx_store.put(tx_hash, str(tx_src), overwrite=ctrl.get('_FORCE_ALL')) + ctrl.write("Balances:") for k in seen_tokens.keys(): ctrl.notify('resolve token {}'.format(seen_tokens[k])) (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k]) @@ -133,7 +141,7 @@ Tags: {}""".format( balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address) fmt = '{:.' + str(token_decimals) + 'f}' decimal_balance = fmt.format(balance / (10 ** token_decimals)) - ctrl.write("Balances:\n {} {}".format(token_symbol, decimal_balance)) + ctrl.write(" {} {}".format(token_symbol, decimal_balance)) print() for l in tx_lines: diff --git a/clicada/tx/file.py b/clicada/tx/file.py index 06be938..b795883 100644 --- a/clicada/tx/file.py +++ b/clicada/tx/file.py @@ -1,20 +1,45 @@ +# standard imports import os +import logging +# external imports +from chainlib.eth.tx import transaction from leveldir.numeric import NumDir from leveldir.hex import HexDir +from hexathon import strip_0x + +logg = logging.getLogger(__name__) + class FileTxStore: subdivision = 100000 - def __init__(self, store_base_path): + def __init__(self, store_base_path, rpc=None, notifier=None): tx_base_path = os.path.join(store_base_path, 'tx') num_base_path = os.path.join(tx_base_path, 'blocks') hash_base_path = os.path.join(tx_base_path, 'hash') + raw_base_path = os.path.join(tx_base_path, 'raw') self.block_index_dir = NumDir(num_base_path) self.hash_index_dir = HexDir(hash_base_path, 32) + self.raw_index_dir = HexDir(raw_base_path, 32) + self.rpc = rpc + self.notifier = notifier - def put(self, k, v): + def put(self, k, v, overwrite=False): + if self.notifier != None: + self.notifier.notify('caching tx data for {}'.format(k)) hsh = bytes.fromhex(k) + if not overwrite and self.hash_index_dir.have(k): + logg.debug('tx store already has {}'.format(k)) + return + self.hash_index_dir.add(hsh, v.encode('utf-8')) + + if self.rpc != None: + self.notifier.notify('retrieve and cache raw tx data for {}'.format(k)) + o = transaction(k) + r = self.rpc.conn.do(o) + raw = bytes.fromhex(strip_0x(r['raw'])) + self.raw_index_dir.add(hsh, raw) diff --git a/clicada/user/file.py b/clicada/user/file.py index 9ec3c81..a649adb 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -65,7 +65,7 @@ class Account(Person): class FileUserStore: - def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None): + def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None, notifier=None): invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl) self.invalidate_before = int(invalidate_before.timestamp()) self.have_xattr = False @@ -83,6 +83,7 @@ class FileUserStore: self.metadata_opener = metadata_opener self.failed_entities = {} self.encrypter = encrypter + self.notifier = notifier def __validate_dir(self): @@ -214,6 +215,8 @@ class FileUserStore: logg.info(e) pass + self.notifier.notify('wallet address for phone {} not found locally, retrieve from metadata service'.format(phone)) + getter = self.metadata_opener ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) r = None @@ -259,6 +262,8 @@ class FileUserStore: logg.info(e) pass + self.notifier.notify('metadata for wallet {} not found locally, retrieve from metadata service'.format(address)) + getter = self.metadata_opener ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON) @@ -276,6 +281,7 @@ class FileUserStore: person = Account() person_data = person.deserialize(person_data=data) + self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person))) ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM) r = None try: