Add raw tx filestore, optional retrieve and cache of raw tx

This MR adds colorized progress statements when resolving and retrieving data for user. It disables logging if loglevel is set to default (logging.WARNING at this time).

It also skips metadata lookups that have failed in the same session, speeding up retrievals when same address repeatedly occurs in transaction list.

closes #6
closes #5

Co-authored-by: lash <dev@holbrook.no>
Reviewed-on: #4
Co-authored-by: lash <accounts-grassrootseconomics@holbrook.no>
Co-committed-by: lash <accounts-grassrootseconomics@holbrook.no>

CHANGELOG

@@ -1,3 +1,5 @@
+- 0.0.6
+	* Add cache encryption, with AES-CTR-128
 - 0.0.5
 	* Replace logs with colorized progress output on default loglevel
 	* Do not repeat already failed metadata lookups

clicada/cli/arg.py

@@ -1,7 +1,7 @@
 # import notifier
 from clicada.cli.notify import NotifyWriter
 notifier = NotifyWriter()
-notifier.notify('loading script')
+#notifier.notify('loading script')
 
 # standard imports
 import os
@@ -22,6 +22,7 @@ from clicada.cli.http import (
         HTTPSession,
         PGPClientSession,
         )
+from clicada.crypt.aes import AESCTREncrypt
 
 logg = logging.getLogger()
 
@@ -150,12 +151,15 @@ class CmdCtrl:
         auth_db_path = self.get('AUTH_DB_PATH', default_auth_db_path)
         self.__auth = PGPAuthCrypt(auth_db_path, self.get('AUTH_KEY'), self.get('AUTH_KEYRING_PATH'))
         self.__auth.get_secret(self.get('AUTH_PASSPHRASE'))
+        self.encrypter = AESCTREncrypt(auth_db_path, self.__auth.secret)
 
 
     def get(self, k, default=None):
         r = self.config.get(k, default)
         if k in [
                 '_FORCE',
+                '_FORCE_ALL',
+                '_RAW_TX',
                 ]:
             if r == None:
                 return False

clicada/cli/auth.py

@@ -26,6 +26,7 @@ class PGPAuthCrypt:
             raise AuthError('invalid key {}'.format(auth_key))
         self.auth_key = auth_key
         self.gpg = gnupg.GPG(gnupghome=pgp_dir)
+        self.secret = None
 
 
     def get_secret(self, passphrase=''):
@@ -49,10 +50,11 @@ class PGPAuthCrypt:
             f.write(secret.data)
             f.close()
             f = open(p, 'rb')
-        self.secret = self.gpg.decrypt_file(f, passphrase=passphrase)
+        secret = self.gpg.decrypt_file(f, passphrase=passphrase)
-        if not self.secret.ok:
+        if not secret.ok:
             raise AuthError('could not decrypt encryption secret. wrong password?')
         f.close()
+        self.secret = secret.data
         self.__passphrase = passphrase
 
 

clicada/cli/http.py

@@ -3,6 +3,7 @@ import hashlib
 import urllib.parse
 import os
 import logging
+from socket import getservbyname
 
 # external imports
 from usumbufu.client.base import (
@@ -48,7 +49,15 @@ class HTTPSession:
     def __init__(self, url, auth=None, origin=None):
         self.base_url = url
         url_parts = urllib.parse.urlsplit(self.base_url)
-        url_parts_origin = (url_parts[0], url_parts[1], '', '', '',)
+        url_parts_origin_host = url_parts[1].split(":")
+        host = url_parts_origin_host[0]
+        try:
+            host = host + ':' + url_parts_origin_host[1]
+        except IndexError:
+            host = host + ':' + str(getservbyname(url_parts[0]))
+            logg.info('changed origin with missing port number from {} to {}'.format(url_parts[1], host))
+        url_parts_origin = (url_parts[0], host, '', '', '',)
+            
         self.origin = origin
         if self.origin == None:
             self.origin = urllib.parse.urlunsplit(url_parts_origin)

clicada/cli/notify.py

@@ -1,12 +1,13 @@
 # standard imports
 import os
 import sys
+import shutil
 
 
 class NotifyWriter:
 
     def __init__(self, writer=sys.stdout):
-        (c, r) = os.get_terminal_size()
+        (c, r) = shutil.get_terminal_size()
         self.cols = c
         self.fmt = "\r{:" + "<{}".format(c) + "}"
         self.w = writer

clicada/cli/user.py

@@ -9,7 +9,10 @@ from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
 from cic_types.models.person import Person
 from chainlib.eth.address import to_checksum_address
 from chainlib.encode import TxHexNormalizer
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
 from clicada.tx import TxGetter
@@ -19,6 +22,7 @@ from clicada.token import (
         token_balance,
         )
 from clicada.tx import ResolvedTokenTx
+from clicada.tx.file import FileTxStore
 from clicada.error import MetadataNotFoundError
 
 logg = logging.getLogger(__name__)
@@ -30,12 +34,16 @@ def process_args(argparser):
     argparser.add_argument('-m', '--method', type=str, help='lookup method')
     argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
     argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries')
+    argparser.add_argument('-ff', '--force-update-all', dest='force_update_all', action='store_true', help='Update records of mutable entries and immutable entries')
+    argparser.add_argument('--raw-tx', dest='raw_tx', action='store_true', help='Also cache raw transaction data')
     argparser.add_argument('identifier', type=str, help='user identifier')
 
 
 def extra_args():
     return {
+        'raw_tx': '_RAW_TX',
         'force_update': '_FORCE',
+        'force_update_all': '_FORCE_ALL',
         'method': 'META_LOOKUP_METHOD',
         'meta_url': 'META_URL',
         'identifier': '_IDENTIFIER',
@@ -56,7 +64,7 @@ def execute(ctrl):
 
     store_path = '.clicada'
     user_phone_file_label = 'phone'
-    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
+    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
 
     ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
     user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
@@ -78,7 +86,7 @@ def execute(ctrl):
     token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
 
     user_address_file_label = 'address'
-    user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
+    user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
 
     ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
     try:
@@ -108,16 +116,24 @@ Tags: {}""".format(
     )
 )
 
+    raw_rpc = None
+    if ctrl.get('_RAW_TX'):
+        raw_rpc = ctrl.rpc 
+
+    tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
     tx_lines = []
     seen_tokens = {}
     for tx_src in txs['data']:
-        ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash']))
+        tx_hash = strip_0x(tx_src['tx_hash'])
+        ctrl.notify('resolve details for tx {}'.format(tx_hash))
         tx = ResolvedTokenTx.from_dict(tx_src)
         tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
         tx_lines.append(tx)
         seen_tokens[tx.source_token_label] = tx.source_token
         seen_tokens[tx.destination_token_label] = tx.destination_token
+        tx_store.put(tx_hash, str(tx_src), overwrite=ctrl.get('_FORCE_ALL'))
 
+    ctrl.write("Balances:")
     for k in seen_tokens.keys():
         ctrl.notify('resolve token {}'.format(seen_tokens[k]))
         (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
@@ -125,7 +141,7 @@ Tags: {}""".format(
         balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
         fmt =  '{:.' + str(token_decimals) + 'f}'
         decimal_balance = fmt.format(balance / (10 ** token_decimals))
-        ctrl.write("Balances:\n    {} {}".format(token_symbol, decimal_balance))
+        ctrl.write("    {} {}".format(token_symbol, decimal_balance))
 
     print()
     for l in tx_lines:

clicada/crypt/aes.py

@@ -0,0 +1,42 @@
+# standard imports
+import os
+import logging
+import hashlib
+
+from Crypto.Cipher import AES
+from Crypto.Util import Counter
+
+from .base import Encrypter
+
+logg = logging.getLogger(__name__)
+
+
+class AESCTREncrypt(Encrypter):
+
+    aes_block_size = 1 << 7
+    counter_bytes = int(128 / 8)
+
+    def __init__(self, db_dir, secret):
+        self.secret = secret
+
+
+    def key_to_iv(self, k):
+        h = hashlib.sha256()
+        h.update(k.encode('utf-8'))
+        h.update(self.secret)
+        z = h.digest()
+        return int.from_bytes(z[:self.counter_bytes], 'big') 
+
+
+    def encrypt(self, k, v):
+        iv = self.key_to_iv(k)
+        ctr = Counter.new(self.aes_block_size, initial_value=iv)
+        cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr)
+        return cipher.encrypt(v)
+       
+
+    def decrypt(self, k, v):
+        iv = self.key_to_iv(k)
+        ctr = Counter.new(self.aes_block_size, initial_value=iv)
+        cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr)
+        return cipher.decrypt(v)

clicada/crypt/base.py

@@ -0,0 +1,8 @@
+class Encrypter:
+    
+    def encrypt(self, v):
+        raise NotImplementedError()
+
+
+    def decrypt(self, v):
+        raise NotImplementedError()

clicada/tx/file.py

@@ -0,0 +1,45 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.tx import transaction
+from leveldir.numeric import NumDir
+from leveldir.hex import HexDir
+from hexathon import strip_0x
+
+logg = logging.getLogger(__name__)
+
+
+class FileTxStore:
+
+    subdivision = 100000
+
+    def __init__(self, store_base_path, rpc=None, notifier=None):
+        tx_base_path = os.path.join(store_base_path, 'tx')
+        num_base_path = os.path.join(tx_base_path, 'blocks')
+        hash_base_path = os.path.join(tx_base_path, 'hash')
+        raw_base_path = os.path.join(tx_base_path, 'raw')
+        self.block_index_dir = NumDir(num_base_path)
+        self.hash_index_dir = HexDir(hash_base_path, 32)
+        self.raw_index_dir = HexDir(raw_base_path, 32)
+        self.rpc = rpc
+        self.notifier = notifier
+
+
+    def put(self, k, v, overwrite=False):
+        if self.notifier != None:
+            self.notifier.notify('caching tx data for {}'.format(k))
+        hsh = bytes.fromhex(k)
+        if not overwrite and self.hash_index_dir.have(k):
+            logg.debug('tx store already has {}'.format(k))
+            return
+
+        self.hash_index_dir.add(hsh, v.encode('utf-8'))
+
+        if self.rpc != None:
+            self.notifier.notify('retrieve and cache raw tx data for {}'.format(k))
+            o = transaction(k) 
+            r = self.rpc.conn.do(o)
+            raw = bytes.fromhex(strip_0x(r['raw']))
+            self.raw_index_dir.add(hsh, raw)

clicada/user/file.py

@@ -65,7 +65,7 @@ class Account(Person):
 
 class FileUserStore:
 
-    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl):
+    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None, notifier=None):
         invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
         self.invalidate_before = int(invalidate_before.timestamp())
         self.have_xattr = False
@@ -82,6 +82,8 @@ class FileUserStore:
         self.__validate_dir()
         self.metadata_opener = metadata_opener
         self.failed_entities = {}
+        self.encrypter = encrypter
+        self.notifier = notifier
 
 
     def __validate_dir(self):
@@ -108,8 +110,14 @@ class FileUserStore:
         if have_file and not its_time and not force:
             raise FileExistsError('user resolution already exists for {}'.format(k))
 
-        f = open(p, 'w')
+        ve = v
-        f.write(v)
+        f = None
+        if self.encrypter != None:
+            ve = self.encrypter.encrypt(k, ve.encode('utf-8'))
+            f = open(p, 'wb')
+        else:
+            f = open(p, 'w')
+        f.write(ve)
         f.close()
 
         logg.info('added user store {} record {} -> {}'.format(self.label, k, v))
@@ -174,12 +182,20 @@ class FileUserStore:
                 self.__unstick(p)
         self.check_expiry(p)
 
-        f = open(p, 'r')
+        f = None
-        r = f.read()
+        if self.encrypter != None:
+            f = open(p, 'rb')
+        else:
+            f = open(p, 'r')
+        v = f.read()
         f.close()
 
+        if self.encrypter != None:
+            v = self.encrypter.decrypt(k, v)
+            v = v.decode('utf-8')
+
         logg.debug('retrieved {} from {}'.format(k, p))
-        return r.strip()
+        return v.strip()
 
 
     def by_phone(self, phone, update=False):
@@ -199,6 +215,8 @@ class FileUserStore:
                 logg.info(e)
                 pass
 
+        self.notifier.notify('wallet address for phone {} not found locally, retrieve from metadata service'.format(phone))
+
         getter = self.metadata_opener
         ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
         r = None
@@ -244,6 +262,8 @@ class FileUserStore:
                 logg.info(e)
                 pass
 
+        self.notifier.notify('metadata for wallet {} not found locally, retrieve from metadata service'.format(address))
+
         getter = self.metadata_opener
 
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@@ -261,6 +281,7 @@ class FileUserStore:
         person = Account()
         person_data = person.deserialize(person_data=data)
 
+        self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
         r = None
         try:

requirements.txt

@@ -1,7 +1,11 @@
-usumbufu~=0.3.4
+usumbufu~=0.3.5
-confini~=0.5.1
+confini~=0.5.3
 cic-eth-registry~=0.6.1
 cic-types~=0.2.1a8
 phonenumbers==8.12.12
 eth-erc20~=0.1.2
 hexathon~=0.1.0
+pycryptodome~=3.10.1
+chainlib-eth~=0.0.21
+chainlib~=0.0.17
+leveldir~=0.1.0

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = clicada
-version = 0.0.5a1
+version = 0.0.6a2
 description = CLI CRM tool for the cic-stack custodial wallet system
 author = Louis Holbrook
 author_email = dev@holbrook.no
@@ -34,3 +34,4 @@ packages =
 	clicada.cli
 	clicada.tx
 	clicada.user
+	clicada.crypt