WIP add AES to local cache

CHANGELOG

@@ -1,3 +1,10 @@
+- 0.0.5
+	* Replace logs with colorized progress output on default loglevel
+	* Do not repeat already failed metadata lookups
+- 0.0.4
+	* Resolve metadata to labels when loading from cache
+- 0.0.3
+	* Upgrade usumbufu to prevent missing bearer auth on https
 - 0.0.2
 	* Use ~/.config for default config override
 - 0.0.1-unreleased

clicada/cli/arg.py

@@ -1,8 +1,13 @@
+# import notifier
+from clicada.cli.notify import NotifyWriter
+notifier = NotifyWriter()
+notifier.notify('loading script')
+
 # standard imports
 import os
-#import argparse
 import logging
 import importlib
+import sys
 
 # external imports
 import confini
@@ -25,6 +30,20 @@ data_dir = os.path.join(script_dir, '..', 'data')
 base_config_dir = os.path.join(data_dir, 'config')
 
 
+class NullWriter:
+
+    def notify(self, v):
+        pass
+
+
+    def ouch(self, v):
+        pass
+
+
+    def write(self, v):
+        sys.stdout.write(str(v))
+
+
 class CmdCtrl:
 
     __cmd_alias = {
@@ -45,6 +64,8 @@ class CmdCtrl:
        
         self.config()
 
+        self.notifier()
+
         self.auth()
 
         self.blockchain()
@@ -156,3 +177,27 @@ class CmdCtrl:
 
     def opener(self, k):
         return self.remote_openers[k]
+
+
+    def notifier(self):
+        if logg.root.level >= logging.WARNING:
+            logging.disable()
+            self.writer = notifier
+        else:
+            self.writer = NullWriter()
+
+
+    def notify(self, v):
+        self.writer.notify(v)
+
+
+    def ouch(self, v):
+        self.writer.croak(v)
+        print()
+
+
+    def write(self, v):
+        self.writer.write("")
+        self.writer.write(v)
+        print()
+

clicada/cli/user.py

@@ -19,6 +19,7 @@ from clicada.token import (
         token_balance,
         )
 from clicada.tx import ResolvedTokenTx
+from clicada.error import MetadataNotFoundError
 
 logg = logging.getLogger(__name__)
 
@@ -51,24 +52,27 @@ def validate(config, args):
 
 
 def execute(ctrl):
-    tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'))
+    tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'), 10)
 
     store_path = '.clicada'
     user_phone_file_label = 'phone'
     user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
 
+    ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
     user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
     if user_address == None:
-        sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
+        ctrl.ouch('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
         sys.exit(1)
     try:
         user_address = to_checksum_address(user_address)
     except ValueError:
-        sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
+        ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
         sys.exit(1)
 
     logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER')))
 
+    user_address_normal = tx_normalizer.wallet_address(user_address)
+    ctrl.notify('retrieving txs for address {}'.format(user_address_normal))
     txs = tx_getter.get(user_address)
 
     token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
@@ -76,12 +80,14 @@ def execute(ctrl):
     user_address_file_label = 'address'
     user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
 
-    user_address_normal = tx_normalizer.wallet_address(user_address)
+    ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
-
+    try:
         r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
-    print('r {}'.format(r))
+    except MetadataNotFoundError as e:
+        ctrl.ouch('could not resolve metadata for user: {}'.format(e))
+        sys.exit(1)
 
-    print("""Phone: {}
+    ctrl.write("""Phone: {}
 Network address: {}
 Chain: {}
 Name: {}
@@ -89,8 +95,7 @@ Registered: {}
 Gender: {}
 Location: {}
 Products: {}
-Tags: {}
+Tags: {}""".format(
-Balances:""".format(
     ctrl.get('_IDENTIFIER'),
     add_0x(user_address),
     ctrl.chain().common_name(),
@@ -106,6 +111,7 @@ Balances:""".format(
     tx_lines = []
     seen_tokens = {}
     for tx_src in txs['data']:
+        ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash']))
         tx = ResolvedTokenTx.from_dict(tx_src)
         tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
         tx_lines.append(tx)
@@ -113,12 +119,14 @@ Balances:""".format(
         seen_tokens[tx.destination_token_label] = tx.destination_token
 
     for k in seen_tokens.keys():
+        ctrl.notify('resolve token {}'.format(seen_tokens[k]))
         (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
+        ctrl.notify('get token balance for {} => {}'.format(token_symbol, seen_tokens[k]))
         balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
         fmt =  '{:.' + str(token_decimals) + 'f}'
         decimal_balance = fmt.format(balance / (10 ** token_decimals))
-        print("\t{} {}".format(token_symbol, decimal_balance))
+        ctrl.write("Balances:\n    {} {}".format(token_symbol, decimal_balance))
 
     print()
     for l in tx_lines:
-        print(l)
+        ctrl.write(l)

clicada/crypt.py

@@ -0,0 +1,38 @@
+# standard imports
+import os
+import logging
+
+from Crypto.Cipher import AES
+from Crypto.Util import Counter
+
+logg = logging.getLogger(__name__)
+
+
+class Encrypt:
+
+    aesBlockSize = 1 << 7
+
+    def __init__(self, secret, db_dir):
+        fp = os.path.join(db_dir, '.aes_ctr_iv')
+        try:
+            f = open(fp, 'rb')
+            self.iv = f.read()
+        except FileNotFoundError:
+            logg.debug('generating new iv for aes-ctr')
+            self.iv = os.urandom(8)
+            f = open(fp, 'wb')
+            f.write(self.iv)
+
+        f.close()
+
+        iv_num = int.from_bytes(self.iv, 'big')
+        self.ctr = Counter.new(aesBlockSize, initial_value=iv_num)
+        self.cipher = AES.new(secret, AES.MODE_CTR, counter=self.ctr)
+
+
+    def encrypt(self, v):
+        return self.cipher.encrypt(v)
+       
+
+    def decrypt(self, v):
+        return self.cipher.decrypt(v)

clicada/error.py

@@ -4,3 +4,7 @@ class ExpiredRecordError(Exception):
 
 class AuthError(Exception):
     pass
+
+
+class MetadataNotFoundError(Exception):
+    pass

clicada/tx/tx.py

@@ -12,7 +12,10 @@ from cic_types.models.tx import (
 
 # local imports
 from clicada.encode import tx_normalize
-from clicada.error import ExpiredRecordError
+from clicada.error import (
+        ExpiredRecordError,
+        MetadataNotFoundError,
+        )
 
 logg = logging.getLogger(__name__)
 
@@ -56,35 +59,22 @@ class ResolvedTokenTx(TokenTx):
         self.to_value_label = fmt.format(token_value)
 
 
-    def resolve_stored_entity(self, user_store, address, update=False):
+    def resolve_entity(self, user_store, address):
-        if update:
-            return None
-        address = tx_normalize.wallet_address(address)
         try:
-            v = user_store.get(address)
+            r = user_store.by_address(address)
-            return v
+        except MetadataNotFoundError:
-        except FileNotFoundError:
+            return address
-            return None
+        return str(r)
-        except ExpiredRecordError:
-            return None
 
 
     def resolve_sender_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.sender, update=update)
-        if v != None:
-            return v
         if self.tx_type == TokenTxType.faucet_giveto.value:
             return 'FAUCET'
-        r = user_store.by_address(self.sender)
+        return self.resolve_entity(user_store, self.sender)
-        return str(r)
         
 
     def resolve_recipient_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.recipient, update=update)
+        return self.resolve_entity(user_store, self.recipient)
-        if v != None:
-            return v
-        r = user_store.by_address(self.recipient, update=update)
-        return str(r)
 
 
     def resolve_entities(self, user_store, update=False):
@@ -99,7 +89,7 @@ class ResolvedTokenTx(TokenTx):
 
     def __str__(self):
         if self.symmetric:
-            return '{}\t{} => {}\t{} {}'.format(
+            return '{}    {} => {}    {} {}'.format(
                     self.date_block_label,
                     self.sender_label,
                     self.recipient_label,

clicada/user/file.py

@@ -18,7 +18,10 @@ import phonenumbers
 # local imports
 from clicada.encode import tx_normalize
 from clicada.store.mem import MemDictStore
-from clicada.error import ExpiredRecordError
+from clicada.error import (
+        ExpiredRecordError,
+        MetadataNotFoundError,
+        )
 
 logg = logging.getLogger(__name__)
 
@@ -62,7 +65,7 @@ class Account(Person):
 
 class FileUserStore:
 
-    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl):
+    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None):
         invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
         self.invalidate_before = int(invalidate_before.timestamp())
         self.have_xattr = False
@@ -78,6 +81,8 @@ class FileUserStore:
         os.makedirs(self.store_path, exist_ok=True)
         self.__validate_dir()
         self.metadata_opener = metadata_opener
+        self.failed_entities = {}
+        self.encrypter = encrypter
 
 
     def __validate_dir(self):
@@ -86,6 +91,10 @@ class FileUserStore:
             logg.debug('using existing file store {} for {}'.format(self.store_path, self.label))
 
 
+    def is_dud(self, address):
+        return bool(self.failed_entities.get(address))
+
+
     def put(self, k, v, force=False):
         have_file = False
         p = os.path.join(self.store_path, k)
@@ -206,27 +215,36 @@ class FileUserStore:
         return user_address
 
 
-    def by_address(self, address, update=False):
+    def metadata_to_person(self, v):
-        add = tx_normalize.wallet_address(address)
-        ignore_expired = self.sticky(address)
-
-        if not update:
-            try:
-                v = self.get(address, ignore_expired=ignore_expired)
-                v = json.loads(v)
         person = Account()
         try:
             person_data = person.deserialize(person_data=v)
         except Exception as e:
             person_data = v
         return person_data
+
+
+    def by_address(self, address, update=False):
+        address = tx_normalize.wallet_address(address)
+        address = strip_0x(address)
+        #if self.failed_entities.get(address):
+        if self.is_dud(address):
+            logg.debug('already tried and failed {}, skipping'.format(address))
+            return None
+
+        ignore_expired = self.sticky(address)
+
+        if not update:
+            try:
+                v = self.get(address, ignore_expired=ignore_expired)
+                v = json.loads(v)
+                return self.metadata_to_person(v)
             except FileNotFoundError:
                 pass
             except ExpiredRecordError as e:
                 logg.info(e)
                 pass
 
-        address = strip_0x(address)
         getter = self.metadata_opener
 
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@@ -235,7 +253,10 @@ class FileUserStore:
             r = getter.open(ptr)
         except Exception as e:
             logg.debug('no metadata found for {}: {}'.format(address, e))
-            return address
+        
+        if r == None:
+            self.failed_entities[address] = True
+            raise MetadataNotFoundError()
 
         data = json.loads(r)
         person = Account()

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = clicada
-version = 0.0.3
+version = 0.0.5a1
 description = CLI CRM tool for the cic-stack custodial wallet system
 author = Louis Holbrook
 author_email = dev@holbrook.no