Add raw tx filestore, optional retrieve and cache of raw tx

clicada/cli/arg.py

@@ -73,19 +73,8 @@ class CmdCtrl:
 
         self.remote_openers = {}
         if self.get('META_URL') != None:
-            sctx = None 
-            if self.cmd_args.cafile != None:
-                import ssl
-                sctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-                sctx.load_verify_locations(self.cmd_args.cafile)
-
             auth_client_session = PGPClientSession(self.__auth)
-            self.remote_openers['meta'] = HTTPSession(
-                    self.get('META_URL'),
-                    auth=auth_client_session,
-                    origin=self.config.get('META_HTTP_ORIGIN'),
-                    ssl_context=sctx,
-                    )
+            self.remote_openers['meta'] = HTTPSession(self.get('META_URL'), auth=auth_client_session, origin=self.config.get('META_HTTP_ORIGIN'))
 
 
     def blockchain(self):
@@ -169,6 +158,8 @@ class CmdCtrl:
         r = self.config.get(k, default)
         if k in [
                 '_FORCE',
+                '_FORCE_ALL',
+                '_RAW_TX',
                 ]:
             if r == None:
                 return False

clicada/cli/http.py

@@ -12,7 +12,6 @@ from usumbufu.client.base import (
         )
 from usumbufu.client.bearer import BearerClientSession
 from usumbufu.client.hoba import HobaClientSession
-from urlybird.host import url_apply_port_string
 
 logg = logging.getLogger(__name__)
 
@@ -22,7 +21,6 @@ class PGPClientSession(HobaClientSession):
     alg = '969'
 
     def __init__(self, auth):
-        super(PGPClientSession, self).__init__()
         self.auth = auth
         self.origin = None
         self.fingerprint = self.auth.fingerprint()
@@ -48,12 +46,23 @@ class HTTPSession:
     
     token_dir = '/run/user/{}/clicada/usumbufu/.token'.format(os.getuid())
 
-    def __init__(self, url, auth=None, origin=None, ssl_context=None):
+    def __init__(self, url, auth=None, origin=None):
         self.base_url = url
-
-        if origin == None:
-            origin = url_apply_port_string(url, as_origin=True)
+        url_parts = urllib.parse.urlsplit(self.base_url)
+        url_parts_origin_host = url_parts[1].split(":")
+        host = url_parts_origin_host[0]
+        try:
+            host = host + ':' + url_parts_origin_host[1]
+        except IndexError:
+            host = host + ':' + str(getservbyname(url_parts[0]))
+            logg.info('changed origin with missing port number from {} to {}'.format(url_parts[1], host))
+        url_parts_origin = (url_parts[0], host, '', '', '',)
+            
         self.origin = origin
+        if self.origin == None:
+            self.origin = urllib.parse.urlunsplit(url_parts_origin)
+        else:
+            logg.debug('overriding http origin for {} with {}'.format(url, self.origin))
 
         h = hashlib.sha256()
         h.update(self.base_url.encode('utf-8'))
@@ -63,7 +72,7 @@ class HTTPSession:
         os.makedirs(token_store_dir, exist_ok=True)
         self.token_store = BaseTokenStore(path=token_store_dir)
 
-        self.session = ClientSession(self.origin, token_store=self.token_store, ssl_context=ssl_context)
+        self.session = ClientSession(self.origin, token_store=self.token_store)
 
         bearer_handler = BearerClientSession(self.origin, token_store=self.token_store)
         self.session.add_subhandler(bearer_handler)
@@ -79,9 +88,6 @@ class HTTPSession:
         url = urllib.parse.urljoin(self.base_url, endpoint)
         logg.debug('open {} with opener {}'.format(url, self))
         r = self.opener.open(url)
-        logg.debug('response code {} for {}'.format(r.code, endpoint))
-        if r.code == 404:
-            raise FileNotFoundError()
         return r.read().decode('utf-8')
 
 

clicada/cli/user.py

@@ -9,7 +9,10 @@ from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
 from cic_types.models.person import Person
 from chainlib.eth.address import to_checksum_address
 from chainlib.encode import TxHexNormalizer
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
 from clicada.tx import TxGetter
@@ -19,6 +22,7 @@ from clicada.token import (
         token_balance,
         )
 from clicada.tx import ResolvedTokenTx
+from clicada.tx.file import FileTxStore
 from clicada.error import MetadataNotFoundError
 
 logg = logging.getLogger(__name__)
@@ -29,14 +33,17 @@ tx_normalizer = TxHexNormalizer()
 def process_args(argparser):
     argparser.add_argument('-m', '--method', type=str, help='lookup method')
     argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
-    argparser.add_argument('--cafile', type=str, help='CA certificate chain file to use for verifying SSL session')
     argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries')
+    argparser.add_argument('-ff', '--force-update-all', dest='force_update_all', action='store_true', help='Update records of mutable entries and immutable entries')
+    argparser.add_argument('--raw-tx', dest='raw_tx', action='store_true', help='Also cache raw transaction data')
     argparser.add_argument('identifier', type=str, help='user identifier')
 
 
 def extra_args():
     return {
+        'raw_tx': '_RAW_TX',
         'force_update': '_FORCE',
+        'force_update_all': '_FORCE_ALL',
         'method': 'META_LOOKUP_METHOD',
         'meta_url': 'META_URL',
         'identifier': '_IDENTIFIER',
@@ -57,7 +64,7 @@ def execute(ctrl):
 
     store_path = '.clicada'
     user_phone_file_label = 'phone'
-    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter)
+    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
 
     ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
     user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
@@ -79,7 +86,7 @@ def execute(ctrl):
     token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
 
     user_address_file_label = 'address'
-    user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter)
+    user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
 
     ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
     try:
@@ -109,16 +116,24 @@ Tags: {}""".format(
     )
 )
 
+    raw_rpc = None
+    if ctrl.get('_RAW_TX'):
+        raw_rpc = ctrl.rpc 
+
+    tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
     tx_lines = []
     seen_tokens = {}
     for tx_src in txs['data']:
-        ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash']))
+        tx_hash = strip_0x(tx_src['tx_hash'])
+        ctrl.notify('resolve details for tx {}'.format(tx_hash))
         tx = ResolvedTokenTx.from_dict(tx_src)
         tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
         tx_lines.append(tx)
         seen_tokens[tx.source_token_label] = tx.source_token
         seen_tokens[tx.destination_token_label] = tx.destination_token
+        tx_store.put(tx_hash, str(tx_src), overwrite=ctrl.get('_FORCE_ALL'))
 
+    ctrl.write("Balances:")
     for k in seen_tokens.keys():
         ctrl.notify('resolve token {}'.format(seen_tokens[k]))
         (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
@@ -126,7 +141,7 @@ Tags: {}""".format(
         balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
         fmt =  '{:.' + str(token_decimals) + 'f}'
         decimal_balance = fmt.format(balance / (10 ** token_decimals))
-        ctrl.write("Balances:\n    {} {}".format(token_symbol, decimal_balance))
+        ctrl.write("    {} {}".format(token_symbol, decimal_balance))
 
     print()
     for l in tx_lines:

clicada/tx/file.py

@@ -0,0 +1,45 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.tx import transaction
+from leveldir.numeric import NumDir
+from leveldir.hex import HexDir
+from hexathon import strip_0x
+
+logg = logging.getLogger(__name__)
+
+
+class FileTxStore:
+
+    subdivision = 100000
+
+    def __init__(self, store_base_path, rpc=None, notifier=None):
+        tx_base_path = os.path.join(store_base_path, 'tx')
+        num_base_path = os.path.join(tx_base_path, 'blocks')
+        hash_base_path = os.path.join(tx_base_path, 'hash')
+        raw_base_path = os.path.join(tx_base_path, 'raw')
+        self.block_index_dir = NumDir(num_base_path)
+        self.hash_index_dir = HexDir(hash_base_path, 32)
+        self.raw_index_dir = HexDir(raw_base_path, 32)
+        self.rpc = rpc
+        self.notifier = notifier
+
+
+    def put(self, k, v, overwrite=False):
+        if self.notifier != None:
+            self.notifier.notify('caching tx data for {}'.format(k))
+        hsh = bytes.fromhex(k)
+        if not overwrite and self.hash_index_dir.have(k):
+            logg.debug('tx store already has {}'.format(k))
+            return
+
+        self.hash_index_dir.add(hsh, v.encode('utf-8'))
+
+        if self.rpc != None:
+            self.notifier.notify('retrieve and cache raw tx data for {}'.format(k))
+            o = transaction(k) 
+            r = self.rpc.conn.do(o)
+            raw = bytes.fromhex(strip_0x(r['raw']))
+            self.raw_index_dir.add(hsh, raw)

clicada/user/file.py

@@ -65,7 +65,7 @@ class Account(Person):
 
 class FileUserStore:
 
-    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None):
+    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None, notifier=None):
         invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
         self.invalidate_before = int(invalidate_before.timestamp())
         self.have_xattr = False
@@ -83,6 +83,7 @@ class FileUserStore:
         self.metadata_opener = metadata_opener
         self.failed_entities = {}
         self.encrypter = encrypter
+        self.notifier = notifier
 
 
     def __validate_dir(self):
@@ -191,7 +192,6 @@ class FileUserStore:
 
         if self.encrypter != None:
             v = self.encrypter.decrypt(k, v)
-            logg.debug('>>>>>>>>>>>>< v decoded {}'.format(v))
             v = v.decode('utf-8')
 
         logg.debug('retrieved {} from {}'.format(k, p))
@@ -215,6 +215,8 @@ class FileUserStore:
                 logg.info(e)
                 pass
 
+        self.notifier.notify('wallet address for phone {} not found locally, retrieve from metadata service'.format(phone))
+
         getter = self.metadata_opener
         ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
         r = None
@@ -260,6 +262,8 @@ class FileUserStore:
                 logg.info(e)
                 pass
 
+        self.notifier.notify('metadata for wallet {} not found locally, retrieve from metadata service'.format(address))
+
         getter = self.metadata_opener
 
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@@ -268,7 +272,7 @@ class FileUserStore:
             r = getter.open(ptr)
         except Exception as e:
             logg.debug('no metadata found for {}: {}'.format(address, e))
-       
+        
         if r == None:
             self.failed_entities[address] = True
             raise MetadataNotFoundError()
@@ -277,6 +281,7 @@ class FileUserStore:
         person = Account()
         person_data = person.deserialize(person_data=data)
 
+        self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
         r = None
         try:

requirements.txt

@@ -1,5 +1,5 @@
-usumbufu~=0.3.6
-confini~=0.5.4
+usumbufu~=0.3.5
+confini~=0.5.3
 cic-eth-registry~=0.6.1
 cic-types~=0.2.1a8
 phonenumbers==8.12.12
@@ -8,4 +8,4 @@ hexathon~=0.1.0
 pycryptodome~=3.10.1
 chainlib-eth~=0.0.21
 chainlib~=0.0.17
-urlybird~=0.0.2
+leveldir~=0.1.0