Bump threaded version

This MR adds colorized progress statements when resolving and retrieving data for user. It disables logging if loglevel is set to default (logging.WARNING at this time).

It also skips metadata lookups that have failed in the same session, speeding up retrievals when same address repeatedly occurs in transaction list.

closes #6
closes #5

Co-authored-by: lash <dev@holbrook.no>
Reviewed-on: #4
Co-authored-by: lash <accounts-grassrootseconomics@holbrook.no>
Co-committed-by: lash <accounts-grassrootseconomics@holbrook.no>

CHANGELOG

@@ -1,3 +1,12 @@
+- 0.0.6
+	* Add cache encryption, with AES-CTR-128
+- 0.0.5
+	* Replace logs with colorized progress output on default loglevel
+	* Do not repeat already failed metadata lookups
+- 0.0.4
+	* Resolve metadata to labels when loading from cache
+- 0.0.3
+	* Upgrade usumbufu to prevent missing bearer auth on https
 - 0.0.2
 	* Use ~/.config for default config override
 - 0.0.1-unreleased

clicada/cli/arg.py

@@ -1,8 +1,13 @@
+# import notifier
+from clicada.cli.notify import NotifyWriter
+notifier = NotifyWriter()
+#notifier.notify('loading script')
+
 # standard imports
 import os
-#import argparse
 import logging
 import importlib
+import sys
 
 # external imports
 import confini
@@ -17,6 +22,7 @@ from clicada.cli.http import (
         HTTPSession,
         PGPClientSession,
         )
+from clicada.crypt.aes import AESCTREncrypt
 
 logg = logging.getLogger()
 
@@ -25,6 +31,20 @@ data_dir = os.path.join(script_dir, '..', 'data')
 base_config_dir = os.path.join(data_dir, 'config')
 
 
+class NullWriter:
+
+    def notify(self, v):
+        pass
+
+
+    def ouch(self, v):
+        pass
+
+
+    def write(self, v):
+        sys.stdout.write(str(v))
+
+
 class CmdCtrl:
 
     __cmd_alias = {
@@ -45,10 +65,12 @@ class CmdCtrl:
        
         self.config()
 
+        self.notifier()
+
         self.auth()
 
         self.blockchain()
-        
+
         self.remote_openers = {}
         if self.get('META_URL') != None:
             auth_client_session = PGPClientSession(self.__auth)
@@ -129,12 +151,17 @@ class CmdCtrl:
         auth_db_path = self.get('AUTH_DB_PATH', default_auth_db_path)
         self.__auth = PGPAuthCrypt(auth_db_path, self.get('AUTH_KEY'), self.get('AUTH_KEYRING_PATH'))
         self.__auth.get_secret(self.get('AUTH_PASSPHRASE'))
+        self.encrypter = AESCTREncrypt(auth_db_path, self.__auth.secret)
 
 
     def get(self, k, default=None):
         r = self.config.get(k, default)
         if k in [
                 '_FORCE',
+                '_FORCE_ALL',
+                '_RAW_TX',
+                '_NO_RESOLVE',
+                '_NO_TX',
                 ]:
             if r == None:
                 return False
@@ -156,3 +183,27 @@ class CmdCtrl:
 
     def opener(self, k):
         return self.remote_openers[k]
+
+
+    def notifier(self):
+        if logg.root.level >= logging.WARNING:
+            logging.disable()
+            self.writer = notifier
+        else:
+            self.writer = NullWriter()
+
+
+    def notify(self, v):
+        self.writer.notify(v)
+
+
+    def ouch(self, v):
+        self.writer.ouch(v)
+        print()
+
+
+    def write(self, v):
+        self.writer.write("")
+        self.writer.write(v)
+        print()
+

clicada/cli/auth.py

@@ -26,6 +26,7 @@ class PGPAuthCrypt:
             raise AuthError('invalid key {}'.format(auth_key))
         self.auth_key = auth_key
         self.gpg = gnupg.GPG(gnupghome=pgp_dir)
+        self.secret = None
 
 
     def get_secret(self, passphrase=''):
@@ -49,10 +50,11 @@ class PGPAuthCrypt:
             f.write(secret.data)
             f.close()
             f = open(p, 'rb')
-        self.secret = self.gpg.decrypt_file(f, passphrase=passphrase)
-        if not self.secret.ok:
+        secret = self.gpg.decrypt_file(f, passphrase=passphrase)
+        if not secret.ok:
             raise AuthError('could not decrypt encryption secret. wrong password?')
         f.close()
+        self.secret = secret.data
         self.__passphrase = passphrase
 
 

clicada/cli/http.py

@@ -3,6 +3,7 @@ import hashlib
 import urllib.parse
 import os
 import logging
+from socket import getservbyname
 
 # external imports
 from usumbufu.client.base import (
@@ -48,7 +49,15 @@ class HTTPSession:
     def __init__(self, url, auth=None, origin=None):
         self.base_url = url
         url_parts = urllib.parse.urlsplit(self.base_url)
-        url_parts_origin = (url_parts[0], url_parts[1], '', '', '',)
+        url_parts_origin_host = url_parts[1].split(":")
+        host = url_parts_origin_host[0]
+        try:
+            host = host + ':' + url_parts_origin_host[1]
+        except IndexError:
+            host = host + ':' + str(getservbyname(url_parts[0]))
+            logg.info('changed origin with missing port number from {} to {}'.format(url_parts[1], host))
+        url_parts_origin = (url_parts[0], host, '', '', '',)
+            
         self.origin = origin
         if self.origin == None:
             self.origin = urllib.parse.urlunsplit(url_parts_origin)

clicada/cli/notify.py

@@ -0,0 +1,33 @@
+# standard imports
+import os
+import sys
+import shutil
+
+
+class NotifyWriter:
+
+    def __init__(self, writer=sys.stdout):
+        (c, r) = shutil.get_terminal_size()
+        self.cols = c
+        self.fmt = "\r{:" + "<{}".format(c) + "}"
+        self.w = writer
+        self.notify_max = self.cols - 4
+
+    
+    def notify(self, v):
+        if len(v) > self.notify_max:
+            v = v[:self.notify_max]
+        self.write('\x1b[0;36m... ' + v + '\x1b[0;39m')
+
+
+    def ouch(self, v):
+        if len(v) > self.notify_max:
+            v = v[:self.notify_max]
+        self.write('\x1b[0;91m!!! ' + v + '\x1b[0;39m')
+
+
+    def write(self, v):
+        s = str(v)
+        if len(s) > self.cols:
+            s = s[:self.cols]
+        self.w.write(self.fmt.format(s))

clicada/cli/user.py

@@ -2,6 +2,7 @@
 import sys
 import logging
 import datetime
+from queue import SimpleQueue as Queue
 
 # external imports
 from cic_eth_registry import CICRegistry
@@ -9,16 +10,26 @@ from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
 from cic_types.models.person import Person
 from chainlib.eth.address import to_checksum_address
 from chainlib.encode import TxHexNormalizer
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
-from clicada.tx import TxGetter
-from clicada.user import FileUserStore
-from clicada.token import (
-        FileTokenStore,
-        token_balance,
+from clicada.tx import (
+        TxGetter,
+        FormattedTokenTx,
         )
+from clicada.user import FileUserStore
+from clicada.token import FileTokenStore
 from clicada.tx import ResolvedTokenTx
+from clicada.tx.file import FileTxStore
+from clicada.error import MetadataNotFoundError
+from clicada.cli.worker import (
+        MetadataResolverWorker,
+        TxResolverWorker,
+        TokenResolverWorker,
+        )
 
 logg = logging.getLogger(__name__)
 
@@ -29,21 +40,31 @@ def process_args(argparser):
     argparser.add_argument('-m', '--method', type=str, help='lookup method')
     argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
     argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries')
+    argparser.add_argument('-ff', '--force-update-all', dest='force_update_all', action='store_true', help='Update records of mutable entries and immutable entries')
+    argparser.add_argument('-N', '--no-resolve', dest='no_resolve', action='store_true', help='Resolve no metadata')
+    argparser.add_argument('--no-tx', dest='no_tx', action='store_true', help='Do not fetch transactions')
+    argparser.add_argument('--raw-tx', dest='raw_tx', action='store_true', help='Also cache raw transaction data')
     argparser.add_argument('identifier', type=str, help='user identifier')
 
 
 def extra_args():
     return {
+        'raw_tx': '_RAW_TX',
         'force_update': '_FORCE',
+        'force_update_all': '_FORCE_ALL',
         'method': 'META_LOOKUP_METHOD',
         'meta_url': 'META_URL',
         'identifier': '_IDENTIFIER',
+        'no_resolve': '_NO_RESOLVE',
+        'no_tx': '_NO_TX',
             }
 
 
 def apply_args(config, args):
     if config.get('META_LOOKUP_METHOD'):
         raise NotImplementedError('Sorry, currently only "phone" lookup method is implemented')
+    if config.true('_FORCE_ALL'):
+        config.add(True, '_FORCE', exists_ok=True)
 
 
 def validate(config, args):
@@ -51,74 +72,157 @@ def validate(config, args):
 
 
 def execute(ctrl):
-    tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'))
+    tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'), 50)
 
     store_path = '.clicada'
     user_phone_file_label = 'phone'
-    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
+    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
 
+    ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
     user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
     if user_address == None:
-        sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
+        ctrl.ouch('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
         sys.exit(1)
     try:
         user_address = to_checksum_address(user_address)
     except ValueError:
-        sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
+        ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
         sys.exit(1)
 
     logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER')))
 
+    user_address_normal = tx_normalizer.wallet_address(user_address)
+    ctrl.write("""Results for lookup by phone {}:
+
+Metadata:
+    Network address: {}""".format(
+        ctrl.get('_IDENTIFIER'),
+        add_0x(user_address),
+    )
+    )
+
+    if not ctrl.get('_NO_RESOLVE'):
+
+        token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
+
+        user_address_file_label = 'address'
+        user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
+
+        ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
+        try:
+            r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
+        except MetadataNotFoundError as e:
+            ctrl.ouch('could not resolve metadata for user: {}'.format(e))
+            sys.exit(1)
+
+        ctrl.write("""    Chain: {}
+    Name: {}
+    Registered: {}
+    Gender: {}
+    Location: {}
+    Products: {}
+    Tags: {}""".format(
+        ctrl.chain().common_name(),
+        str(r),
+        datetime.datetime.fromtimestamp(r.date_registered).ctime(),
+        r.gender,
+        r.location['area_name'],
+        ','.join(r.products),
+        ','.join(r.tags),
+        )
+    )
+
+    if ctrl.get('_NO_TX'):
+        sys.exit(0)
+
+    raw_rpc = None
+    if ctrl.get('_RAW_TX'):
+        raw_rpc = ctrl.rpc 
+
+    ctrl.notify('retrieving txs for address {}'.format(user_address_normal))
     txs = tx_getter.get(user_address)
 
-    token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
+    if ctrl.get('_NO_RESOLVE'):
+        for v in txs['data']:
+            tx = FormattedTokenTx.from_dict(v)
+            ctrl.write(tx)
+        sys.exit(0)
 
-    user_address_file_label = 'address'
-    user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
 
-    user_address_normal = tx_normalizer.wallet_address(user_address)
+    token_resolver_queue = Queue()
+    token_result_queue = Queue()
+    token_resolver_worker = TokenResolverWorker(user_address, ctrl, token_store, token_resolver_queue, token_result_queue)
+    token_resolver_worker.start()
 
-    r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
-    print('r {}'.format(r))
+    wallets = []
+    for tx in txs['data']:
+        token_resolver_queue.put_nowait(tx['source_token'])
+        token_resolver_queue.put_nowait(tx['destination_token'])
+        if tx['sender'] not in wallets:
+            logg.info('adding wallet {} to metadata lookup'.format(tx['sender']))
+            wallets.append(tx['sender'])
+        if tx['recipient'] not in wallets:
+            wallets.append(tx['recipient'])
+            logg.info('registered wallet {} for metadata lookup'.format(tx['recipient']))
 
-    print("""Phone: {}
-Network address: {}
-Chain: {}
-Name: {}
-Registered: {}
-Gender: {}
-Location: {}
-Products: {}
-Tags: {}
-Balances:""".format(
-    ctrl.get('_IDENTIFIER'),
-    add_0x(user_address),
-    ctrl.chain().common_name(),
-    str(r),
-    datetime.datetime.fromtimestamp(r.date_registered).ctime(),
-    r.gender,
-    r.location['area_name'],
-    ','.join(r.products),
-    ','.join(r.tags),
-    )
-)
+    wallet_threads = []
+    for a in wallets:
+        thread_wallet = MetadataResolverWorker(a, ctrl, user_address_store)
+        thread_wallet.start()
+        wallet_threads.append(thread_wallet)
+    
+    ctrl.notify('wait for metadata resolvers to finish work')
+    for t in wallet_threads:
+        t.join()
 
-    tx_lines = []
-    seen_tokens = {}
+    tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
+    tx_threads = []
+    tx_queue = Queue()
+
+    tx_n = 0
     for tx_src in txs['data']:
-        tx = ResolvedTokenTx.from_dict(tx_src)
-        tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
-        tx_lines.append(tx)
-        seen_tokens[tx.source_token_label] = tx.source_token
-        seen_tokens[tx.destination_token_label] = tx.destination_token
+        tx_hash = strip_0x(tx_src['tx_hash'])
+        tx_worker = TxResolverWorker(tx_hash, tx_src, ctrl, tx_store, token_store, user_address_store, token_resolver_queue, tx_queue, show_decimals=True, update=ctrl.get('_FORCE'))
+        tx_thread = tx_worker.start()
+        tx_threads.append(tx_worker)
+        tx_n += 1
+
+    tx_buf = {}
+
+    for i in range(0, tx_n):
+        tx = tx_queue.get()
+        if tx == None:
+            break
+        # ugh, ugly
+        #k = float('{}.{}'.format(tx.block_number, tx.tx_index))
+        # tx_index is missing, this is temporary sort measure
+        k = str(tx.block_number) + '.' + tx.tx_hash
+        tx_buf[k] = tx
+
+    ctrl.notify('wait for transaction getters to finish work')
+    for tx_thread in tx_threads:
+        tx_thread.join()
+
+    token_resolver_queue.put_nowait(None)
+
+    token_buf = ''
+    while True:
+        l = token_result_queue.get()
+        if l == None:
+            break
+        token_buf += '    {} {}\n'.format(l[0], l[1])
+
+    ctrl.notify('wait for token resolver to finish work')
+    token_resolver_worker.join()
+
+    ctrl.write('')
+    ctrl.write("Balances:")
+    ctrl.write(token_buf)
+    ks = list(tx_buf.keys())
+    ks.sort()
+    ks.reverse()
+    for k in ks:
+        ctrl.write(tx_buf[k])
+
 
-    for k in seen_tokens.keys():
-        (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
-        balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
-        fmt =  '{:.' + str(token_decimals) + 'f}'
-        decimal_balance = fmt.format(balance / (10 ** token_decimals))
-        print("\t{} {}".format(token_symbol, decimal_balance))
 
-    print()
-    for l in tx_lines:
-        print(l)

clicada/cli/worker.py

@@ -0,0 +1,87 @@
+# standard imports
+import threading
+import logging
+
+# external imports
+from hexathon import strip_0x
+
+# local imports
+from clicada.tx import ResolvedTokenTx
+from clicada.token import token_balance
+from clicada.error import MetadataNotFoundError
+
+logg = logging.getLogger(__name__)
+
+
+class MetadataResolverWorker(threading.Thread):
+    
+    def __init__(self, wallet_address, ctrl, user_address_store):
+        self.user_address_store = user_address_store
+        self.wallet_address = wallet_address
+        self.ctrl = ctrl
+        super(MetadataResolverWorker, self).__init__()
+        
+
+    def run(self):
+        self.ctrl.notify('resolve metadata for {}'.format(self.wallet_address))
+        try:
+            self.user_address_store.by_address(self.wallet_address)
+        except MetadataNotFoundError:
+            logg.info('failed metadata lookup for {}'.format(self.wallet_address))
+
+
+class TxResolverWorker(threading.Thread):
+    def __init__(self, tx_hash, tx_src, ctrl, tx_store, token_store, user_address_store, token_queue, tx_queue, show_decimals=True, update=None):
+        self.tx_hash = tx_hash
+        self.tx_src = tx_src
+        self.ctrl = ctrl
+        self.token_store = token_store
+        self.user_address_store = user_address_store
+        self.show_decimals = show_decimals
+        self.update = update
+        self.token_queue = token_queue
+        self.tx_store = tx_store
+        self.tx_queue = tx_queue
+        super(TxResolverWorker, self).__init__()
+
+
+    def run(self):
+        self.ctrl.notify('resolve details for tx {}'.format(self.tx_hash))
+        tx = ResolvedTokenTx.from_dict(self.tx_src)
+        tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update, lookup=False)
+        self.tx_store.put(self.tx_hash, str(self.tx_src), overwrite=self.ctrl.get('_FORCE_ALL'))
+        self.tx_queue.put(tx)
+
+
+class TokenResolverWorker(threading.Thread):
+
+    def __init__(self, wallet_address, ctrl, token_store, in_queue, out_queue):
+        super(TokenResolverWorker, self).__init__()
+        self.ctrl = ctrl
+        self.token_store = token_store
+        self.in_queue = in_queue
+        self.out_queue = out_queue
+        self.seen_tokens = {}
+        self.wallet_address = strip_0x(wallet_address)
+
+
+    def run(self):
+        while True:
+            token_address = self.in_queue.get()
+            if token_address == None:
+                logg.debug('token resolver end')
+                self.out_queue.put_nowait(None)
+                return
+            token_address = strip_0x(token_address)
+            if self.seen_tokens.get(token_address) != None:
+                continue
+            logg.debug('resolve token {}'.format(token_address))
+            self.ctrl.notify('resolve token {}'.format(token_address))
+            (token_symbol, token_decimals) = self.token_store.by_address(token_address)
+            self.seen_tokens[token_address] = token_address
+            self.ctrl.notify('get token balance for {} => {}'.format(token_symbol, self.seen_tokens[token_address]))
+            balance = token_balance(self.ctrl.chain(), self.ctrl.conn(), self.seen_tokens[token_address], self.wallet_address)
+            fmt =  '{:.' + str(token_decimals) + 'f}'
+            decimal_balance = fmt.format(balance / (10 ** token_decimals))
+            logg.debug('token balance for {} ({}) is {}'.format(token_symbol, token_address, decimal_balance))
+            self.out_queue.put((token_symbol, decimal_balance,))

clicada/crypt/aes.py

@@ -0,0 +1,42 @@
+# standard imports
+import os
+import logging
+import hashlib
+
+from Crypto.Cipher import AES
+from Crypto.Util import Counter
+
+from .base import Encrypter
+
+logg = logging.getLogger(__name__)
+
+
+class AESCTREncrypt(Encrypter):
+
+    aes_block_size = 1 << 7
+    counter_bytes = int(128 / 8)
+
+    def __init__(self, db_dir, secret):
+        self.secret = secret
+
+
+    def key_to_iv(self, k):
+        h = hashlib.sha256()
+        h.update(k.encode('utf-8'))
+        h.update(self.secret)
+        z = h.digest()
+        return int.from_bytes(z[:self.counter_bytes], 'big') 
+
+
+    def encrypt(self, k, v):
+        iv = self.key_to_iv(k)
+        ctr = Counter.new(self.aes_block_size, initial_value=iv)
+        cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr)
+        return cipher.encrypt(v)
+       
+
+    def decrypt(self, k, v):
+        iv = self.key_to_iv(k)
+        ctr = Counter.new(self.aes_block_size, initial_value=iv)
+        cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr)
+        return cipher.decrypt(v)

clicada/crypt/base.py

@@ -0,0 +1,8 @@
+class Encrypter:
+    
+    def encrypt(self, v):
+        raise NotImplementedError()
+
+
+    def decrypt(self, v):
+        raise NotImplementedError()

clicada/error.py

@@ -4,3 +4,7 @@ class ExpiredRecordError(Exception):
 
 class AuthError(Exception):
     pass
+
+
+class MetadataNotFoundError(Exception):
+    pass

clicada/token/token.py

@@ -52,7 +52,7 @@ class FileTokenStore:
         return p
    
 
-    def by_address(self, address):
+    def by_address(self, address, update=False, lookup=True):
         address = tx_normalize.executable_address(address)
 
         token_symbol = self.memstore_symbol.get(address)
@@ -65,17 +65,30 @@ class FileTokenStore:
         try:
             f = open(p, 'r')
         except FileNotFoundError:
-            p = self.__cache_token(address)
-            f = open(p, 'r')
-       
-        token_symbol = f.read()
-        f.close()
+            pass
 
-        p = os.path.join(self.store_path, token_symbol)
-        f = open(p, 'r')
-        r = f.read()
-        f.close()
-        token_decimals = int(r)
+        if f == None:
+            if not lookup:
+                token_symbol = '???'
+                token_decimals = '???'
+            #self.memstore_symbol.put(address, token_symbol)
+            #self.memstore_decimals.put(address, token_decimals)
+            #logg.warning('token metadata not found and lookup deactivated. Will use 18 decimals as default')
+            #return (token_symbol, token_decimals,)
+
+        if token_symbol == None:
+            if f == None:
+                p = self.__cache_token(address)
+                f = open(p, 'r')
+           
+            token_symbol = f.read()
+            f.close()
+
+            p = os.path.join(self.store_path, token_symbol)
+            f = open(p, 'r')
+            r = f.read()
+            f.close()
+            token_decimals = int(r)
 
         self.memstore_symbol.put(address, token_symbol)
         self.memstore_decimals.put(token_symbol, token_decimals)

clicada/tx/file.py

@@ -0,0 +1,45 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.tx import transaction
+from leveldir.numeric import NumDir
+from leveldir.hex import HexDir
+from hexathon import strip_0x
+
+logg = logging.getLogger(__name__)
+
+
+class FileTxStore:
+
+    subdivision = 100000
+
+    def __init__(self, store_base_path, rpc=None, notifier=None):
+        tx_base_path = os.path.join(store_base_path, 'tx')
+        num_base_path = os.path.join(tx_base_path, 'blocks')
+        hash_base_path = os.path.join(tx_base_path, 'hash')
+        raw_base_path = os.path.join(tx_base_path, 'raw')
+        self.block_index_dir = NumDir(num_base_path)
+        self.hash_index_dir = HexDir(hash_base_path, 32)
+        self.raw_index_dir = HexDir(raw_base_path, 32)
+        self.rpc = rpc
+        self.notifier = notifier
+
+
+    def put(self, k, v, overwrite=False):
+        if self.notifier != None:
+            self.notifier.notify('caching tx data for {}'.format(k))
+        hsh = bytes.fromhex(k)
+        if not overwrite and self.hash_index_dir.have(k):
+            logg.debug('tx store already has {}'.format(k))
+            return
+
+        self.hash_index_dir.add(hsh, v.encode('utf-8'))
+
+        if self.rpc != None:
+            self.notifier.notify('retrieve and cache raw tx data for {}'.format(k))
+            o = transaction(k) 
+            r = self.rpc.conn.do(o)
+            raw = bytes.fromhex(strip_0x(r['raw']))
+            self.raw_index_dir.add(hsh, raw)

clicada/tx/tx.py

@@ -12,12 +12,36 @@ from cic_types.models.tx import (
 
 # local imports
 from clicada.encode import tx_normalize
-from clicada.error import ExpiredRecordError
+from clicada.error import (
+        ExpiredRecordError,
+        MetadataNotFoundError,
+        )
+from chainlib.eth.address import AddressChecksum
 
 logg = logging.getLogger(__name__)
 
+address_checksummer = AddressChecksum()
+
+
+class FormattedTokenTx(TokenTx):
+
+    def __init__(self):
+        super(FormattedTokenTx, self).__init__()
+        self.symmetric = True
+
+    def __str__(self):
+        if self.symmetric:
+            return '{}    {} => {}    {} {}'.format(
+                    self.date_block_label,
+                    self.sender_label,
+                    self.recipient_label,
+                    self.destination_token_label,
+                    self.to_value_label,
+            )
+
+
+class ResolvedTokenTx(FormattedTokenTx):
 
-class ResolvedTokenTx(TokenTx):
 
     def __init__(self):
         super(ResolvedTokenTx, self).__init__()
@@ -25,13 +49,12 @@ class ResolvedTokenTx(TokenTx):
         self.destination_token_name = None
         self.source_token_decimals = None
         self.destination_token_decimals = None
-        self.symmetric = True
         self.sender_entity = None
         self.recipient_entity = None
 
 
-    def resolve_tokens(self, token_store, show_decimals=False, update=False):
-        (token_symbol, token_decimals) = token_store.by_address(self.source_token)
+    def resolve_tokens(self, token_store, show_decimals=False, update=False, lookup=False):
+        (token_symbol, token_decimals) = token_store.by_address(self.source_token, lookup=False)
         self.source_token_decimals = token_decimals
         self.source_token_label = token_symbol
         token_value = self.to_value / (10 ** token_decimals)
@@ -56,58 +79,34 @@ class ResolvedTokenTx(TokenTx):
         self.to_value_label = fmt.format(token_value)
 
 
-    def resolve_stored_entity(self, user_store, address, update=False):
-        if update:
-            return None
-        address = tx_normalize.wallet_address(address)
+    def resolve_entity(self, user_store, address, update=False, lookup=True):
         try:
-            v = user_store.get(address)
-            return v
-        except FileNotFoundError:
-            return None
-        except ExpiredRecordError:
-            return None
+            r = user_store.by_address(address, update=update, lookup=lookup)
+        except MetadataNotFoundError:
+            return address_checksummer.sum(address)
+        return str(r)
 
 
-    def resolve_sender_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.sender, update=update)
-        if v != None:
-            return v
+    def resolve_sender_entity(self, user_store, update=False, lookup=True):
         if self.tx_type == TokenTxType.faucet_giveto.value:
             return 'FAUCET'
-        r = user_store.by_address(self.sender)
-        return str(r)
+        return self.resolve_entity(user_store, self.sender, update=update, lookup=lookup)
+        
+
+    def resolve_recipient_entity(self, user_store, update=False, lookup=True):
+        return self.resolve_entity(user_store, self.recipient, update=update, lookup=lookup)
 
 
-    def resolve_recipient_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.recipient, update=update)
-        if v != None:
-            return v
-        r = user_store.by_address(self.recipient, update=update)
-        return str(r)
+    def resolve_entities(self, user_store, update=False, lookup=True):
+        self.sender_label = self.resolve_sender_entity(user_store, update=update, lookup=lookup)
+        self.recipient_label = self.resolve_recipient_entity(user_store, update=update, lookup=lookup)
 
 
-    def resolve_entities(self, user_store, update=False):
-        self.sender_label = self.resolve_sender_entity(user_store, update=update)
-        self.recipient_label = self.resolve_recipient_entity(user_store, update=update)
-
-
-    def resolve(self, token_store, user_store, show_decimals=False, update=False):
-        self.resolve_tokens(token_store, show_decimals, update=update)
+    def resolve(self, token_store, user_store, show_decimals=False, update=False, lookup=True):
+        self.resolve_tokens(token_store, show_decimals, update=update, lookup=lookup)
         self.resolve_entities(user_store, update=update)
 
 
-    def __str__(self):
-        if self.symmetric:
-            return '{}\t{} => {}\t{} {}'.format(
-                    self.date_block_label,
-                    self.sender_label,
-                    self.recipient_label,
-                    self.destination_token_label,
-                    self.to_value_label,
-            )
-
-
 class TxGetter:
 
     def __init__(self, cache_url, limit=0):

clicada/user/file.py

@@ -18,7 +18,10 @@ import phonenumbers
 # local imports
 from clicada.encode import tx_normalize
 from clicada.store.mem import MemDictStore
-from clicada.error import ExpiredRecordError
+from clicada.error import (
+        ExpiredRecordError,
+        MetadataNotFoundError,
+        )
 
 logg = logging.getLogger(__name__)
 
@@ -62,7 +65,7 @@ class Account(Person):
 
 class FileUserStore:
 
-    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl):
+    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None, notifier=None):
         invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
         self.invalidate_before = int(invalidate_before.timestamp())
         self.have_xattr = False
@@ -78,6 +81,9 @@ class FileUserStore:
         os.makedirs(self.store_path, exist_ok=True)
         self.__validate_dir()
         self.metadata_opener = metadata_opener
+        self.failed_entities = {}
+        self.encrypter = encrypter
+        self.notifier = notifier
 
 
     def __validate_dir(self):
@@ -86,6 +92,10 @@ class FileUserStore:
             logg.debug('using existing file store {} for {}'.format(self.store_path, self.label))
 
 
+    def is_dud(self, address):
+        return bool(self.failed_entities.get(address))
+
+
     def put(self, k, v, force=False):
         have_file = False
         p = os.path.join(self.store_path, k)
@@ -100,8 +110,14 @@ class FileUserStore:
         if have_file and not its_time and not force:
             raise FileExistsError('user resolution already exists for {}'.format(k))
 
-        f = open(p, 'w')
-        f.write(v)
+        ve = v
+        f = None
+        if self.encrypter != None:
+            ve = self.encrypter.encrypt(k, ve.encode('utf-8'))
+            f = open(p, 'wb')
+        else:
+            f = open(p, 'w')
+        f.write(ve)
         f.close()
 
         logg.info('added user store {} record {} -> {}'.format(self.label, k, v))
@@ -166,12 +182,20 @@ class FileUserStore:
                 self.__unstick(p)
         self.check_expiry(p)
 
-        f = open(p, 'r')
-        r = f.read()
+        f = None
+        if self.encrypter != None:
+            f = open(p, 'rb')
+        else:
+            f = open(p, 'r')
+        v = f.read()
         f.close()
 
+        if self.encrypter != None:
+            v = self.encrypter.decrypt(k, v)
+            v = v.decode('utf-8')
+
         logg.debug('retrieved {} from {}'.format(k, p))
-        return r.strip()
+        return v.strip()
 
 
     def by_phone(self, phone, update=False):
@@ -191,6 +215,8 @@ class FileUserStore:
                 logg.info(e)
                 pass
 
+        self.notifier.notify('wallet address for phone {} not found locally, retrieve from metadata service'.format(phone))
+
         getter = self.metadata_opener
         ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
         r = None
@@ -205,28 +231,38 @@ class FileUserStore:
         self.put(phone_file, user_address, force=update)
         return user_address
 
-    
-    def by_address(self, address, update=False):
-        add = tx_normalize.wallet_address(address)
+
+    def metadata_to_person(self, v):
+        person = Account()
+        try:
+            person_data = person.deserialize(person_data=v)
+        except Exception as e:
+            person_data = v
+        return person_data
+
+
+    def by_address(self, address, update=False, lookup=True):
+        address = tx_normalize.wallet_address(address)
+        address = strip_0x(address)
+        if self.is_dud(address):
+            logg.debug('already tried and failed {}, skipping'.format(address))
+            raise MetadataNotFoundError()
+
         ignore_expired = self.sticky(address)
 
         if not update:
             try:
                 v = self.get(address, ignore_expired=ignore_expired)
                 v = json.loads(v)
-                person = Account()
-                try:
-                    person_data = person.deserialize(person_data=v)
-                except Exception as e:
-                    person_data = v
-                return person_data
+                return self.metadata_to_person(v)
             except FileNotFoundError:
                 pass
             except ExpiredRecordError as e:
                 logg.info(e)
                 pass
 
-        address = strip_0x(address)
+        self.notifier.notify('metadata for wallet {} not found locally, retrieve from metadata service'.format(address))
+
         getter = self.metadata_opener
 
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@@ -235,12 +271,17 @@ class FileUserStore:
             r = getter.open(ptr)
         except Exception as e:
             logg.debug('no metadata found for {}: {}'.format(address, e))
-            return address
+        
+        if r == None:
+            self.failed_entities[address] = True
+            raise MetadataNotFoundError()
 
         data = json.loads(r)
         person = Account()
         person_data = person.deserialize(person_data=data)
 
+        logg.debug('wallet {} resolved to {}'.format(address, str(person)))
+        self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
         r = None
         try:

requirements.txt

@@ -1,7 +1,11 @@
-usumbufu~=0.3.3a3
-confini~=0.5.1
+usumbufu~=0.3.5
+confini~=0.5.3
 cic-eth-registry~=0.6.1
-cic-types~=0.2.1a5
+cic-types~=0.2.1a8
 phonenumbers==8.12.12
 eth-erc20~=0.1.2
 hexathon~=0.1.0
+pycryptodome~=3.10.1
+chainlib-eth~=0.0.21
+chainlib~=0.0.17
+leveldir~=0.1.0

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = clicada
-version = 0.0.2a2
+version = 0.0.7rc1
 description = CLI CRM tool for the cic-stack custodial wallet system
 author = Louis Holbrook
 author_email = dev@holbrook.no
@@ -34,3 +34,4 @@ packages =
 	clicada.cli
 	clicada.tx
 	clicada.user
+	clicada.crypt