Add missing notify

2022-01-21 10:15:59 +00:00
12 changed files with 19 additions and 174 deletions
--- a/2
+++ b/2
@@ -1,5 +1,3 @@
- 0.0.6
-	* Add cache encryption, with AES-CTR-128
 - 0.0.5
 	* Replace logs with colorized progress output on default loglevel
 	* Do not repeat already failed metadata lookups
--- a/clicada/cli/arg.py
+++ b/clicada/cli/arg.py
@@ -1,7 +1,7 @@
 # import notifier
 from clicada.cli.notify import NotifyWriter
 notifier = NotifyWriter()
-#notifier.notify('loading script')
+notifier.notify('loading script')

 # standard imports
 import os
@@ -22,7 +22,6 @@ from clicada.cli.http import (
        HTTPSession,
        PGPClientSession,
        )
-from clicada.crypt.aes import AESCTREncrypt

 logg = logging.getLogger()

@@ -151,15 +150,12 @@ class CmdCtrl:
        auth_db_path = self.get('AUTH_DB_PATH', default_auth_db_path)
        self.__auth = PGPAuthCrypt(auth_db_path, self.get('AUTH_KEY'), self.get('AUTH_KEYRING_PATH'))
        self.__auth.get_secret(self.get('AUTH_PASSPHRASE'))
-        self.encrypter = AESCTREncrypt(auth_db_path, self.__auth.secret)


    def get(self, k, default=None):
        r = self.config.get(k, default)
        if k in [
                '_FORCE',
-                '_FORCE_ALL',
-                '_RAW_TX',
                ]:
            if r == None:
                return False
--- a/clicada/cli/auth.py
+++ b/clicada/cli/auth.py
@@ -26,7 +26,6 @@ class PGPAuthCrypt:
            raise AuthError('invalid key {}'.format(auth_key))
        self.auth_key = auth_key
        self.gpg = gnupg.GPG(gnupghome=pgp_dir)
-        self.secret = None


    def get_secret(self, passphrase=''):
@@ -50,11 +49,10 @@ class PGPAuthCrypt:
            f.write(secret.data)
            f.close()
            f = open(p, 'rb')
-        secret = self.gpg.decrypt_file(f, passphrase=passphrase)
-        if not secret.ok:
+        self.secret = self.gpg.decrypt_file(f, passphrase=passphrase)
+        if not self.secret.ok:
            raise AuthError('could not decrypt encryption secret. wrong password?')
        f.close()
-        self.secret = secret.data
        self.__passphrase = passphrase


--- a/clicada/cli/http.py
+++ b/clicada/cli/http.py
@@ -3,7 +3,6 @@ import hashlib
 import urllib.parse
 import os
 import logging
-from socket import getservbyname

 # external imports
 from usumbufu.client.base import (
@@ -49,15 +48,7 @@ class HTTPSession:
    def __init__(self, url, auth=None, origin=None):
        self.base_url = url
        url_parts = urllib.parse.urlsplit(self.base_url)
-        url_parts_origin_host = url_parts[1].split(":")
-        host = url_parts_origin_host[0]
-        try:
-            host = host + ':' + url_parts_origin_host[1]
-        except IndexError:
-            host = host + ':' + str(getservbyname(url_parts[0]))
-            logg.info('changed origin with missing port number from {} to {}'.format(url_parts[1], host))
-        url_parts_origin = (url_parts[0], host, '', '', '',)
-            
+        url_parts_origin = (url_parts[0], url_parts[1], '', '', '',)
        self.origin = origin
        if self.origin == None:
            self.origin = urllib.parse.urlunsplit(url_parts_origin)
--- a/clicada/cli/notify.py
+++ b/clicada/cli/notify.py
@@ -1,13 +1,12 @@
 # standard imports
 import os
 import sys
-import shutil


 class NotifyWriter:

    def __init__(self, writer=sys.stdout):
-        (c, r) = shutil.get_terminal_size()
+        (c, r) = os.get_terminal_size()
        self.cols = c
        self.fmt = "\r{:" + "<{}".format(c) + "}"
        self.w = writer
--- a/clicada/cli/user.py
+++ b/clicada/cli/user.py
@@ -9,10 +9,7 @@ from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
 from cic_types.models.person import Person
 from chainlib.eth.address import to_checksum_address
 from chainlib.encode import TxHexNormalizer
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
+from hexathon import add_0x

 # local imports
 from clicada.tx import TxGetter
@@ -22,7 +19,6 @@ from clicada.token import (
        token_balance,
        )
 from clicada.tx import ResolvedTokenTx
-from clicada.tx.file import FileTxStore
 from clicada.error import MetadataNotFoundError

 logg = logging.getLogger(__name__)
@@ -34,16 +30,12 @@ def process_args(argparser):
    argparser.add_argument('-m', '--method', type=str, help='lookup method')
    argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
    argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries')
-    argparser.add_argument('-ff', '--force-update-all', dest='force_update_all', action='store_true', help='Update records of mutable entries and immutable entries')
-    argparser.add_argument('--raw-tx', dest='raw_tx', action='store_true', help='Also cache raw transaction data')
    argparser.add_argument('identifier', type=str, help='user identifier')


 def extra_args():
    return {
-        'raw_tx': '_RAW_TX',
        'force_update': '_FORCE',
-        'force_update_all': '_FORCE_ALL',
        'method': 'META_LOOKUP_METHOD',
        'meta_url': 'META_URL',
        'identifier': '_IDENTIFIER',
@@ -64,7 +56,7 @@ def execute(ctrl):

    store_path = '.clicada'
    user_phone_file_label = 'phone'
-    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
+    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))

    ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
    user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
@@ -86,7 +78,7 @@ def execute(ctrl):
    token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)

    user_address_file_label = 'address'
-    user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
+    user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))

    ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
    try:
@@ -116,24 +108,16 @@ Tags: {}""".format(
    )
 )

-    raw_rpc = None
-    if ctrl.get('_RAW_TX'):
-        raw_rpc = ctrl.rpc 
-
-    tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
    tx_lines = []
    seen_tokens = {}
    for tx_src in txs['data']:
-        tx_hash = strip_0x(tx_src['tx_hash'])
-        ctrl.notify('resolve details for tx {}'.format(tx_hash))
+        ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash']))
        tx = ResolvedTokenTx.from_dict(tx_src)
        tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
        tx_lines.append(tx)
        seen_tokens[tx.source_token_label] = tx.source_token
        seen_tokens[tx.destination_token_label] = tx.destination_token
-        tx_store.put(tx_hash, str(tx_src), overwrite=ctrl.get('_FORCE_ALL'))

-    ctrl.write("Balances:")
    for k in seen_tokens.keys():
        ctrl.notify('resolve token {}'.format(seen_tokens[k]))
        (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
@@ -141,7 +125,7 @@ Tags: {}""".format(
        balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
        fmt =  '{:.' + str(token_decimals) + 'f}'
        decimal_balance = fmt.format(balance / (10 ** token_decimals))
-        ctrl.write("    {} {}".format(token_symbol, decimal_balance))
+        ctrl.write("Balances:\n    {} {}".format(token_symbol, decimal_balance))

    print()
    for l in tx_lines:
--- a/clicada/crypt/aes.py
+++ b/clicada/crypt/aes.py
@@ -1,42 +0,0 @@
-# standard imports
-import os
-import logging
-import hashlib
-
-from Crypto.Cipher import AES
-from Crypto.Util import Counter
-
-from .base import Encrypter
-
-logg = logging.getLogger(__name__)
-
-
-class AESCTREncrypt(Encrypter):
-
-    aes_block_size = 1 << 7
-    counter_bytes = int(128 / 8)
-
-    def __init__(self, db_dir, secret):
-        self.secret = secret
-
-
-    def key_to_iv(self, k):
-        h = hashlib.sha256()
-        h.update(k.encode('utf-8'))
-        h.update(self.secret)
-        z = h.digest()
-        return int.from_bytes(z[:self.counter_bytes], 'big') 
-
-
-    def encrypt(self, k, v):
-        iv = self.key_to_iv(k)
-        ctr = Counter.new(self.aes_block_size, initial_value=iv)
-        cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr)
-        return cipher.encrypt(v)
-       
-
-    def decrypt(self, k, v):
-        iv = self.key_to_iv(k)
-        ctr = Counter.new(self.aes_block_size, initial_value=iv)
-        cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr)
-        return cipher.decrypt(v)
--- a/clicada/crypt/base.py
+++ b/clicada/crypt/base.py
@@ -1,8 +0,0 @@
-class Encrypter:
-    
-    def encrypt(self, v):
-        raise NotImplementedError()
-
-
-    def decrypt(self, v):
-        raise NotImplementedError()
--- a/clicada/tx/file.py
+++ b/clicada/tx/file.py
@@ -1,45 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-from chainlib.eth.tx import transaction
-from leveldir.numeric import NumDir
-from leveldir.hex import HexDir
-from hexathon import strip_0x
-
-logg = logging.getLogger(__name__)
-
-
-class FileTxStore:
-
-    subdivision = 100000
-
-    def __init__(self, store_base_path, rpc=None, notifier=None):
-        tx_base_path = os.path.join(store_base_path, 'tx')
-        num_base_path = os.path.join(tx_base_path, 'blocks')
-        hash_base_path = os.path.join(tx_base_path, 'hash')
-        raw_base_path = os.path.join(tx_base_path, 'raw')
-        self.block_index_dir = NumDir(num_base_path)
-        self.hash_index_dir = HexDir(hash_base_path, 32)
-        self.raw_index_dir = HexDir(raw_base_path, 32)
-        self.rpc = rpc
-        self.notifier = notifier
-
-
-    def put(self, k, v, overwrite=False):
-        if self.notifier != None:
-            self.notifier.notify('caching tx data for {}'.format(k))
-        hsh = bytes.fromhex(k)
-        if not overwrite and self.hash_index_dir.have(k):
-            logg.debug('tx store already has {}'.format(k))
-            return
-
-        self.hash_index_dir.add(hsh, v.encode('utf-8'))
-
-        if self.rpc != None:
-            self.notifier.notify('retrieve and cache raw tx data for {}'.format(k))
-            o = transaction(k) 
-            r = self.rpc.conn.do(o)
-            raw = bytes.fromhex(strip_0x(r['raw']))
-            self.raw_index_dir.add(hsh, raw)
--- a/clicada/user/file.py
+++ b/clicada/user/file.py
@@ -65,7 +65,7 @@ class Account(Person):

 class FileUserStore:

-    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None, notifier=None):
+    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl):
        invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
        self.invalidate_before = int(invalidate_before.timestamp())
        self.have_xattr = False
@@ -82,8 +82,6 @@ class FileUserStore:
        self.__validate_dir()
        self.metadata_opener = metadata_opener
        self.failed_entities = {}
-        self.encrypter = encrypter
-        self.notifier = notifier


    def __validate_dir(self):
@@ -110,14 +108,8 @@ class FileUserStore:
        if have_file and not its_time and not force:
            raise FileExistsError('user resolution already exists for {}'.format(k))

-        ve = v
-        f = None
-        if self.encrypter != None:
-            ve = self.encrypter.encrypt(k, ve.encode('utf-8'))
-            f = open(p, 'wb')
-        else:
-            f = open(p, 'w')
-        f.write(ve)
+        f = open(p, 'w')
+        f.write(v)
        f.close()

        logg.info('added user store {} record {} -> {}'.format(self.label, k, v))
@@ -182,20 +174,12 @@ class FileUserStore:
                self.__unstick(p)
        self.check_expiry(p)

-        f = None
-        if self.encrypter != None:
-            f = open(p, 'rb')
-        else:
-            f = open(p, 'r')
-        v = f.read()
+        f = open(p, 'r')
+        r = f.read()
        f.close()

-        if self.encrypter != None:
-            v = self.encrypter.decrypt(k, v)
-            v = v.decode('utf-8')
-
        logg.debug('retrieved {} from {}'.format(k, p))
-        return v.strip()
+        return r.strip()


    def by_phone(self, phone, update=False):
@@ -215,8 +199,6 @@ class FileUserStore:
                logg.info(e)
                pass

-        self.notifier.notify('wallet address for phone {} not found locally, retrieve from metadata service'.format(phone))
-
        getter = self.metadata_opener
        ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
        r = None
@@ -262,8 +244,6 @@ class FileUserStore:
                logg.info(e)
                pass

-        self.notifier.notify('metadata for wallet {} not found locally, retrieve from metadata service'.format(address))
-
        getter = self.metadata_opener

        ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@@ -281,7 +261,6 @@ class FileUserStore:
        person = Account()
        person_data = person.deserialize(person_data=data)

-        self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
        ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
        r = None
        try:
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,7 @@
-usumbufu~=0.3.5
-confini~=0.5.3
+usumbufu~=0.3.4
+confini~=0.5.1
 cic-eth-registry~=0.6.1
 cic-types~=0.2.1a8
 phonenumbers==8.12.12
 eth-erc20~=0.1.2
 hexathon~=0.1.0
-pycryptodome~=3.10.1
-chainlib-eth~=0.0.21
-chainlib~=0.0.17
-leveldir~=0.1.0
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = clicada
-version = 0.0.6a2
+version = 0.0.5a1
 description = CLI CRM tool for the cic-stack custodial wallet system
 author = Louis Holbrook
 author_email = dev@holbrook.no
@@ -34,4 +34,3 @@ packages =
 	clicada.cli
 	clicada.tx
 	clicada.user
-	clicada.crypt