From b057cb65ff792578fe3adb7a8a33c88038d3b4ed Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 10:03:01 +0000 Subject: [PATCH 1/9] WIP add AES to local cache --- CHANGELOG | 7 +++++++ clicada/cli/arg.py | 49 ++++++++++++++++++++++++++++++++++++++++++-- clicada/cli/user.py | 32 ++++++++++++++++++----------- clicada/crypt.py | 38 ++++++++++++++++++++++++++++++++++ clicada/error.py | 4 ++++ clicada/tx/tx.py | 36 ++++++++++++-------------------- clicada/user/file.py | 45 +++++++++++++++++++++++++++++----------- setup.cfg | 2 +- 8 files changed, 163 insertions(+), 50 deletions(-) create mode 100644 clicada/crypt.py diff --git a/CHANGELOG b/CHANGELOG index 41a8d68..b77a6f2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +- 0.0.5 + * Replace logs with colorized progress output on default loglevel + * Do not repeat already failed metadata lookups +- 0.0.4 + * Resolve metadata to labels when loading from cache +- 0.0.3 + * Upgrade usumbufu to prevent missing bearer auth on https - 0.0.2 * Use ~/.config for default config override - 0.0.1-unreleased diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index 5de057c..ff7da17 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -1,8 +1,13 @@ +# import notifier +from clicada.cli.notify import NotifyWriter +notifier = NotifyWriter() +notifier.notify('loading script') + # standard imports import os -#import argparse import logging import importlib +import sys # external imports import confini @@ -25,6 +30,20 @@ data_dir = os.path.join(script_dir, '..', 'data') base_config_dir = os.path.join(data_dir, 'config') +class NullWriter: + + def notify(self, v): + pass + + + def ouch(self, v): + pass + + + def write(self, v): + sys.stdout.write(str(v)) + + class CmdCtrl: __cmd_alias = { @@ -45,10 +64,12 @@ class CmdCtrl: self.config() + self.notifier() + self.auth() self.blockchain() - + self.remote_openers = {} if self.get('META_URL') != None: auth_client_session = PGPClientSession(self.__auth) @@ -156,3 +177,27 @@ class CmdCtrl: def opener(self, k): return self.remote_openers[k] + + + def notifier(self): + if logg.root.level >= logging.WARNING: + logging.disable() + self.writer = notifier + else: + self.writer = NullWriter() + + + def notify(self, v): + self.writer.notify(v) + + + def ouch(self, v): + self.writer.croak(v) + print() + + + def write(self, v): + self.writer.write("") + self.writer.write(v) + print() + diff --git a/clicada/cli/user.py b/clicada/cli/user.py index c035baf..c4a4794 100644 --- a/clicada/cli/user.py +++ b/clicada/cli/user.py @@ -19,6 +19,7 @@ from clicada.token import ( token_balance, ) from clicada.tx import ResolvedTokenTx +from clicada.error import MetadataNotFoundError logg = logging.getLogger(__name__) @@ -51,24 +52,27 @@ def validate(config, args): def execute(ctrl): - tx_getter = TxGetter(ctrl.get('TX_CACHE_URL')) + tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'), 10) store_path = '.clicada' user_phone_file_label = 'phone' user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) + ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER'))) user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE')) if user_address == None: - sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER'))) + ctrl.ouch('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER'))) sys.exit(1) try: user_address = to_checksum_address(user_address) except ValueError: - sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER'))) + ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER'))) sys.exit(1) logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER'))) + user_address_normal = tx_normalizer.wallet_address(user_address) + ctrl.notify('retrieving txs for address {}'.format(user_address_normal)) txs = tx_getter.get(user_address) token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) @@ -76,12 +80,14 @@ def execute(ctrl): user_address_file_label = 'address' user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) - user_address_normal = tx_normalizer.wallet_address(user_address) + ctrl.notify('resolving metadata for address {}'.format(user_address_normal)) + try: + r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE')) + except MetadataNotFoundError as e: + ctrl.ouch('could not resolve metadata for user: {}'.format(e)) + sys.exit(1) - r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE')) - print('r {}'.format(r)) - - print("""Phone: {} + ctrl.write("""Phone: {} Network address: {} Chain: {} Name: {} @@ -89,8 +95,7 @@ Registered: {} Gender: {} Location: {} Products: {} -Tags: {} -Balances:""".format( +Tags: {}""".format( ctrl.get('_IDENTIFIER'), add_0x(user_address), ctrl.chain().common_name(), @@ -106,6 +111,7 @@ Balances:""".format( tx_lines = [] seen_tokens = {} for tx_src in txs['data']: + ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash'])) tx = ResolvedTokenTx.from_dict(tx_src) tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE')) tx_lines.append(tx) @@ -113,12 +119,14 @@ Balances:""".format( seen_tokens[tx.destination_token_label] = tx.destination_token for k in seen_tokens.keys(): + ctrl.notify('resolve token {}'.format(seen_tokens[k])) (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k]) + ctrl.notify('get token balance for {} => {}'.format(token_symbol, seen_tokens[k])) balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address) fmt = '{:.' + str(token_decimals) + 'f}' decimal_balance = fmt.format(balance / (10 ** token_decimals)) - print("\t{} {}".format(token_symbol, decimal_balance)) + ctrl.write("Balances:\n {} {}".format(token_symbol, decimal_balance)) print() for l in tx_lines: - print(l) + ctrl.write(l) diff --git a/clicada/crypt.py b/clicada/crypt.py new file mode 100644 index 0000000..b917098 --- /dev/null +++ b/clicada/crypt.py @@ -0,0 +1,38 @@ +# standard imports +import os +import logging + +from Crypto.Cipher import AES +from Crypto.Util import Counter + +logg = logging.getLogger(__name__) + + +class Encrypt: + + aesBlockSize = 1 << 7 + + def __init__(self, secret, db_dir): + fp = os.path.join(db_dir, '.aes_ctr_iv') + try: + f = open(fp, 'rb') + self.iv = f.read() + except FileNotFoundError: + logg.debug('generating new iv for aes-ctr') + self.iv = os.urandom(8) + f = open(fp, 'wb') + f.write(self.iv) + + f.close() + + iv_num = int.from_bytes(self.iv, 'big') + self.ctr = Counter.new(aesBlockSize, initial_value=iv_num) + self.cipher = AES.new(secret, AES.MODE_CTR, counter=self.ctr) + + + def encrypt(self, v): + return self.cipher.encrypt(v) + + + def decrypt(self, v): + return self.cipher.decrypt(v) diff --git a/clicada/error.py b/clicada/error.py index e405133..f1ac2de 100644 --- a/clicada/error.py +++ b/clicada/error.py @@ -4,3 +4,7 @@ class ExpiredRecordError(Exception): class AuthError(Exception): pass + + +class MetadataNotFoundError(Exception): + pass diff --git a/clicada/tx/tx.py b/clicada/tx/tx.py index dad971b..2cbea44 100644 --- a/clicada/tx/tx.py +++ b/clicada/tx/tx.py @@ -12,7 +12,10 @@ from cic_types.models.tx import ( # local imports from clicada.encode import tx_normalize -from clicada.error import ExpiredRecordError +from clicada.error import ( + ExpiredRecordError, + MetadataNotFoundError, + ) logg = logging.getLogger(__name__) @@ -56,35 +59,22 @@ class ResolvedTokenTx(TokenTx): self.to_value_label = fmt.format(token_value) - def resolve_stored_entity(self, user_store, address, update=False): - if update: - return None - address = tx_normalize.wallet_address(address) + def resolve_entity(self, user_store, address): try: - v = user_store.get(address) - return v - except FileNotFoundError: - return None - except ExpiredRecordError: - return None + r = user_store.by_address(address) + except MetadataNotFoundError: + return address + return str(r) def resolve_sender_entity(self, user_store, update=False): - v = self.resolve_stored_entity(user_store, self.sender, update=update) - if v != None: - return v if self.tx_type == TokenTxType.faucet_giveto.value: return 'FAUCET' - r = user_store.by_address(self.sender) - return str(r) - + return self.resolve_entity(user_store, self.sender) + def resolve_recipient_entity(self, user_store, update=False): - v = self.resolve_stored_entity(user_store, self.recipient, update=update) - if v != None: - return v - r = user_store.by_address(self.recipient, update=update) - return str(r) + return self.resolve_entity(user_store, self.recipient) def resolve_entities(self, user_store, update=False): @@ -99,7 +89,7 @@ class ResolvedTokenTx(TokenTx): def __str__(self): if self.symmetric: - return '{}\t{} => {}\t{} {}'.format( + return '{} {} => {} {} {}'.format( self.date_block_label, self.sender_label, self.recipient_label, diff --git a/clicada/user/file.py b/clicada/user/file.py index 3eeda13..846ffc2 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -18,7 +18,10 @@ import phonenumbers # local imports from clicada.encode import tx_normalize from clicada.store.mem import MemDictStore -from clicada.error import ExpiredRecordError +from clicada.error import ( + ExpiredRecordError, + MetadataNotFoundError, + ) logg = logging.getLogger(__name__) @@ -62,7 +65,7 @@ class Account(Person): class FileUserStore: - def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl): + def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None): invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl) self.invalidate_before = int(invalidate_before.timestamp()) self.have_xattr = False @@ -78,6 +81,8 @@ class FileUserStore: os.makedirs(self.store_path, exist_ok=True) self.__validate_dir() self.metadata_opener = metadata_opener + self.failed_entities = {} + self.encrypter = encrypter def __validate_dir(self): @@ -86,6 +91,10 @@ class FileUserStore: logg.debug('using existing file store {} for {}'.format(self.store_path, self.label)) + def is_dud(self, address): + return bool(self.failed_entities.get(address)) + + def put(self, k, v, force=False): have_file = False p = os.path.join(self.store_path, k) @@ -205,28 +214,37 @@ class FileUserStore: self.put(phone_file, user_address, force=update) return user_address - + + def metadata_to_person(self, v): + person = Account() + try: + person_data = person.deserialize(person_data=v) + except Exception as e: + person_data = v + return person_data + + def by_address(self, address, update=False): - add = tx_normalize.wallet_address(address) + address = tx_normalize.wallet_address(address) + address = strip_0x(address) + #if self.failed_entities.get(address): + if self.is_dud(address): + logg.debug('already tried and failed {}, skipping'.format(address)) + return None + ignore_expired = self.sticky(address) if not update: try: v = self.get(address, ignore_expired=ignore_expired) v = json.loads(v) - person = Account() - try: - person_data = person.deserialize(person_data=v) - except Exception as e: - person_data = v - return person_data + return self.metadata_to_person(v) except FileNotFoundError: pass except ExpiredRecordError as e: logg.info(e) pass - address = strip_0x(address) getter = self.metadata_opener ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON) @@ -235,7 +253,10 @@ class FileUserStore: r = getter.open(ptr) except Exception as e: logg.debug('no metadata found for {}: {}'.format(address, e)) - return address + + if r == None: + self.failed_entities[address] = True + raise MetadataNotFoundError() data = json.loads(r) person = Account() diff --git a/setup.cfg b/setup.cfg index 0f325fb..ecf7664 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = clicada -version = 0.0.3 +version = 0.0.5a1 description = CLI CRM tool for the cic-stack custodial wallet system author = Louis Holbrook author_email = dev@holbrook.no From 64c7fa950cf1699748d813489e6acd7b039570b6 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 10:06:29 +0000 Subject: [PATCH 2/9] Clean pollution from encrypt branch --- clicada/crypt.py | 38 -------------------------------------- clicada/user/file.py | 3 +-- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 clicada/crypt.py diff --git a/clicada/crypt.py b/clicada/crypt.py deleted file mode 100644 index b917098..0000000 --- a/clicada/crypt.py +++ /dev/null @@ -1,38 +0,0 @@ -# standard imports -import os -import logging - -from Crypto.Cipher import AES -from Crypto.Util import Counter - -logg = logging.getLogger(__name__) - - -class Encrypt: - - aesBlockSize = 1 << 7 - - def __init__(self, secret, db_dir): - fp = os.path.join(db_dir, '.aes_ctr_iv') - try: - f = open(fp, 'rb') - self.iv = f.read() - except FileNotFoundError: - logg.debug('generating new iv for aes-ctr') - self.iv = os.urandom(8) - f = open(fp, 'wb') - f.write(self.iv) - - f.close() - - iv_num = int.from_bytes(self.iv, 'big') - self.ctr = Counter.new(aesBlockSize, initial_value=iv_num) - self.cipher = AES.new(secret, AES.MODE_CTR, counter=self.ctr) - - - def encrypt(self, v): - return self.cipher.encrypt(v) - - - def decrypt(self, v): - return self.cipher.decrypt(v) diff --git a/clicada/user/file.py b/clicada/user/file.py index 846ffc2..8ca23d4 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -65,7 +65,7 @@ class Account(Person): class FileUserStore: - def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None): + def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl): invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl) self.invalidate_before = int(invalidate_before.timestamp()) self.have_xattr = False @@ -82,7 +82,6 @@ class FileUserStore: self.__validate_dir() self.metadata_opener = metadata_opener self.failed_entities = {} - self.encrypter = encrypter def __validate_dir(self): From f17e31d801c9862fb82fa5cbb5e6a47537adc1c0 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 10:13:11 +0000 Subject: [PATCH 3/9] Return address on shortcircuited dud lookup --- clicada/cli/arg.py | 2 +- clicada/user/file.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index ff7da17..4c7564a 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -192,7 +192,7 @@ class CmdCtrl: def ouch(self, v): - self.writer.croak(v) + self.writer.ouch(v) print() diff --git a/clicada/user/file.py b/clicada/user/file.py index 8ca23d4..a84a4f9 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -229,7 +229,7 @@ class FileUserStore: #if self.failed_entities.get(address): if self.is_dud(address): logg.debug('already tried and failed {}, skipping'.format(address)) - return None + return address ignore_expired = self.sticky(address) From 36b4fcab93ea22b076d4d75dc633fdfbdf6671c1 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 10:15:43 +0000 Subject: [PATCH 4/9] Add back crypt module --- clicada/crypt.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 clicada/crypt.py diff --git a/clicada/crypt.py b/clicada/crypt.py new file mode 100644 index 0000000..b917098 --- /dev/null +++ b/clicada/crypt.py @@ -0,0 +1,38 @@ +# standard imports +import os +import logging + +from Crypto.Cipher import AES +from Crypto.Util import Counter + +logg = logging.getLogger(__name__) + + +class Encrypt: + + aesBlockSize = 1 << 7 + + def __init__(self, secret, db_dir): + fp = os.path.join(db_dir, '.aes_ctr_iv') + try: + f = open(fp, 'rb') + self.iv = f.read() + except FileNotFoundError: + logg.debug('generating new iv for aes-ctr') + self.iv = os.urandom(8) + f = open(fp, 'wb') + f.write(self.iv) + + f.close() + + iv_num = int.from_bytes(self.iv, 'big') + self.ctr = Counter.new(aesBlockSize, initial_value=iv_num) + self.cipher = AES.new(secret, AES.MODE_CTR, counter=self.ctr) + + + def encrypt(self, v): + return self.cipher.encrypt(v) + + + def decrypt(self, v): + return self.cipher.decrypt(v) From d302c5754c003230f3c469dfafb73c5647da2d49 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 10:59:27 +0000 Subject: [PATCH 5/9] Add AES CTR with key as iv --- clicada/cli/arg.py | 2 ++ clicada/cli/auth.py | 6 ++++-- clicada/cli/notify.py | 32 ++++++++++++++++++++++++++++++++ clicada/cli/user.py | 4 ++-- clicada/crypt.py | 38 -------------------------------------- clicada/crypt/aes.py | 42 ++++++++++++++++++++++++++++++++++++++++++ clicada/crypt/base.py | 8 ++++++++ clicada/user/file.py | 28 ++++++++++++++++++++++------ 8 files changed, 112 insertions(+), 48 deletions(-) create mode 100644 clicada/cli/notify.py delete mode 100644 clicada/crypt.py create mode 100644 clicada/crypt/aes.py create mode 100644 clicada/crypt/base.py diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index 4c7564a..441db69 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -22,6 +22,7 @@ from clicada.cli.http import ( HTTPSession, PGPClientSession, ) +from clicada.crypt.aes import AESCTREncrypt logg = logging.getLogger() @@ -150,6 +151,7 @@ class CmdCtrl: auth_db_path = self.get('AUTH_DB_PATH', default_auth_db_path) self.__auth = PGPAuthCrypt(auth_db_path, self.get('AUTH_KEY'), self.get('AUTH_KEYRING_PATH')) self.__auth.get_secret(self.get('AUTH_PASSPHRASE')) + self.encrypter = AESCTREncrypt(auth_db_path, self.__auth.secret) def get(self, k, default=None): diff --git a/clicada/cli/auth.py b/clicada/cli/auth.py index 5954eb9..d5dcb78 100644 --- a/clicada/cli/auth.py +++ b/clicada/cli/auth.py @@ -26,6 +26,7 @@ class PGPAuthCrypt: raise AuthError('invalid key {}'.format(auth_key)) self.auth_key = auth_key self.gpg = gnupg.GPG(gnupghome=pgp_dir) + self.secret = None def get_secret(self, passphrase=''): @@ -49,10 +50,11 @@ class PGPAuthCrypt: f.write(secret.data) f.close() f = open(p, 'rb') - self.secret = self.gpg.decrypt_file(f, passphrase=passphrase) - if not self.secret.ok: + secret = self.gpg.decrypt_file(f, passphrase=passphrase) + if not secret.ok: raise AuthError('could not decrypt encryption secret. wrong password?') f.close() + self.secret = secret.data self.__passphrase = passphrase diff --git a/clicada/cli/notify.py b/clicada/cli/notify.py new file mode 100644 index 0000000..6b8d8ef --- /dev/null +++ b/clicada/cli/notify.py @@ -0,0 +1,32 @@ +# standard imports +import os +import sys + + +class NotifyWriter: + + def __init__(self, writer=sys.stdout): + (c, r) = os.get_terminal_size() + self.cols = c + self.fmt = "\r{:" + "<{}".format(c) + "}" + self.w = writer + self.notify_max = self.cols - 4 + + + def notify(self, v): + if len(v) > self.notify_max: + v = v[:self.notify_max] + self.write('\x1b[0;36m... ' + v + '\x1b[0;39m') + + + def ouch(self, v): + if len(v) > self.notify_max: + v = v[:self.notify_max] + self.write('\x1b[0;91m!!! ' + v + '\x1b[0;39m') + + + def write(self, v): + s = str(v) + if len(s) > self.cols: + s = s[:self.cols] + self.w.write(self.fmt.format(s)) diff --git a/clicada/cli/user.py b/clicada/cli/user.py index c4a4794..92a0aa0 100644 --- a/clicada/cli/user.py +++ b/clicada/cli/user.py @@ -56,7 +56,7 @@ def execute(ctrl): store_path = '.clicada' user_phone_file_label = 'phone' - user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) + user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter) ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER'))) user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE')) @@ -78,7 +78,7 @@ def execute(ctrl): token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) user_address_file_label = 'address' - user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) + user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter) ctrl.notify('resolving metadata for address {}'.format(user_address_normal)) try: diff --git a/clicada/crypt.py b/clicada/crypt.py deleted file mode 100644 index b917098..0000000 --- a/clicada/crypt.py +++ /dev/null @@ -1,38 +0,0 @@ -# standard imports -import os -import logging - -from Crypto.Cipher import AES -from Crypto.Util import Counter - -logg = logging.getLogger(__name__) - - -class Encrypt: - - aesBlockSize = 1 << 7 - - def __init__(self, secret, db_dir): - fp = os.path.join(db_dir, '.aes_ctr_iv') - try: - f = open(fp, 'rb') - self.iv = f.read() - except FileNotFoundError: - logg.debug('generating new iv for aes-ctr') - self.iv = os.urandom(8) - f = open(fp, 'wb') - f.write(self.iv) - - f.close() - - iv_num = int.from_bytes(self.iv, 'big') - self.ctr = Counter.new(aesBlockSize, initial_value=iv_num) - self.cipher = AES.new(secret, AES.MODE_CTR, counter=self.ctr) - - - def encrypt(self, v): - return self.cipher.encrypt(v) - - - def decrypt(self, v): - return self.cipher.decrypt(v) diff --git a/clicada/crypt/aes.py b/clicada/crypt/aes.py new file mode 100644 index 0000000..8d8afb4 --- /dev/null +++ b/clicada/crypt/aes.py @@ -0,0 +1,42 @@ +# standard imports +import os +import logging +import hashlib + +from Crypto.Cipher import AES +from Crypto.Util import Counter + +from .base import Encrypter + +logg = logging.getLogger(__name__) + + +class AESCTREncrypt(Encrypter): + + aes_block_size = 1 << 7 + counter_bytes = int(128 / 8) + + def __init__(self, db_dir, secret): + self.secret = secret + + + def key_to_iv(self, k): + h = hashlib.sha256() + h.update(k.encode('utf-8')) + h.update(self.secret) + z = h.digest() + return int.from_bytes(z[:self.counter_bytes], 'big') + + + def encrypt(self, k, v): + iv = self.key_to_iv(k) + ctr = Counter.new(self.aes_block_size, initial_value=iv) + cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr) + return cipher.encrypt(v) + + + def decrypt(self, k, v): + iv = self.key_to_iv(k) + ctr = Counter.new(self.aes_block_size, initial_value=iv) + cipher = AES.new(self.secret, AES.MODE_CTR, counter=ctr) + return cipher.decrypt(v) diff --git a/clicada/crypt/base.py b/clicada/crypt/base.py new file mode 100644 index 0000000..19bc150 --- /dev/null +++ b/clicada/crypt/base.py @@ -0,0 +1,8 @@ +class Encrypter: + + def encrypt(self, v): + raise NotImplementedError() + + + def decrypt(self, v): + raise NotImplementedError() diff --git a/clicada/user/file.py b/clicada/user/file.py index a84a4f9..02bf037 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -65,7 +65,7 @@ class Account(Person): class FileUserStore: - def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl): + def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None): invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl) self.invalidate_before = int(invalidate_before.timestamp()) self.have_xattr = False @@ -82,6 +82,7 @@ class FileUserStore: self.__validate_dir() self.metadata_opener = metadata_opener self.failed_entities = {} + self.encrypter = encrypter def __validate_dir(self): @@ -108,8 +109,14 @@ class FileUserStore: if have_file and not its_time and not force: raise FileExistsError('user resolution already exists for {}'.format(k)) - f = open(p, 'w') - f.write(v) + ve = v + f = None + if self.encrypter != None: + ve = self.encrypter.encrypt(k, ve.encode('utf-8')) + f = open(p, 'wb') + else: + f = open(p, 'w') + f.write(ve) f.close() logg.info('added user store {} record {} -> {}'.format(self.label, k, v)) @@ -174,12 +181,21 @@ class FileUserStore: self.__unstick(p) self.check_expiry(p) - f = open(p, 'r') - r = f.read() + f = None + if self.encrypter != None: + f = open(p, 'rb') + else: + f = open(p, 'r') + v = f.read() f.close() + if self.encrypter != None: + v = self.encrypter.decrypt(k, v) + logg.debug('>>>>>>>>>>>>< v decoded {}'.format(v)) + v = v.decode('utf-8') + logg.debug('retrieved {} from {}'.format(k, p)) - return r.strip() + return v.strip() def by_phone(self, phone, update=False): From c5b4c41db01de3304d1f3b6ca2a4e0b0bc6202b2 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 11:04:17 +0000 Subject: [PATCH 6/9] Bump version --- CHANGELOG | 2 ++ setup.cfg | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index b77a6f2..0df2409 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +- 0.0.6 + * Add cache encryption, with AES-CTR-128 - 0.0.5 * Replace logs with colorized progress output on default loglevel * Do not repeat already failed metadata lookups diff --git a/setup.cfg b/setup.cfg index ecf7664..a0f589c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = clicada -version = 0.0.5a1 +version = 0.0.6a1 description = CLI CRM tool for the cic-stack custodial wallet system author = Louis Holbrook author_email = dev@holbrook.no From 140df0d1c66878c7d8fb9388c9944fcbe73e67a4 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 11:09:17 +0000 Subject: [PATCH 7/9] Add pycryptodome dep --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f210ea9..5826fa7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ cic-types~=0.2.1a8 phonenumbers==8.12.12 eth-erc20~=0.1.2 hexathon~=0.1.0 +pycryptodome~=3.10.1 From 3663665d91b040b7f69aa7965cde5cf6bb547891 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 11:28:22 +0000 Subject: [PATCH 8/9] Update packaging and deps --- requirements.txt | 6 ++++-- setup.cfg | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5826fa7..2003eba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ -usumbufu~=0.3.4 -confini~=0.5.1 +usumbufu~=0.3.5 +confini~=0.5.3 cic-eth-registry~=0.6.1 cic-types~=0.2.1a8 phonenumbers==8.12.12 eth-erc20~=0.1.2 hexathon~=0.1.0 pycryptodome~=3.10.1 +chainlib-eth~=0.0.21 +chainlib~=0.0.17 diff --git a/setup.cfg b/setup.cfg index a0f589c..907da2d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,3 +34,4 @@ packages = clicada.cli clicada.tx clicada.user + clicada.crypt From 0bfe054b906f21770609f16c289141da7dd816d9 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 21 Jan 2022 14:38:06 +0000 Subject: [PATCH 9/9] Auto-complete origin on missing port, magic shutil terminal size --- clicada/cli/arg.py | 2 +- clicada/cli/http.py | 11 ++++++++++- clicada/cli/notify.py | 3 ++- setup.cfg | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index 441db69..2ea2cb2 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -1,7 +1,7 @@ # import notifier from clicada.cli.notify import NotifyWriter notifier = NotifyWriter() -notifier.notify('loading script') +#notifier.notify('loading script') # standard imports import os diff --git a/clicada/cli/http.py b/clicada/cli/http.py index 0ae7dda..f49ceee 100644 --- a/clicada/cli/http.py +++ b/clicada/cli/http.py @@ -3,6 +3,7 @@ import hashlib import urllib.parse import os import logging +from socket import getservbyname # external imports from usumbufu.client.base import ( @@ -48,7 +49,15 @@ class HTTPSession: def __init__(self, url, auth=None, origin=None): self.base_url = url url_parts = urllib.parse.urlsplit(self.base_url) - url_parts_origin = (url_parts[0], url_parts[1], '', '', '',) + url_parts_origin_host = url_parts[1].split(":") + host = url_parts_origin_host[0] + try: + host = host + ':' + url_parts_origin_host[1] + except IndexError: + host = host + ':' + str(getservbyname(url_parts[0])) + logg.info('changed origin with missing port number from {} to {}'.format(url_parts[1], host)) + url_parts_origin = (url_parts[0], host, '', '', '',) + self.origin = origin if self.origin == None: self.origin = urllib.parse.urlunsplit(url_parts_origin) diff --git a/clicada/cli/notify.py b/clicada/cli/notify.py index 6b8d8ef..8ee25b9 100644 --- a/clicada/cli/notify.py +++ b/clicada/cli/notify.py @@ -1,12 +1,13 @@ # standard imports import os import sys +import shutil class NotifyWriter: def __init__(self, writer=sys.stdout): - (c, r) = os.get_terminal_size() + (c, r) = shutil.get_terminal_size() self.cols = c self.fmt = "\r{:" + "<{}".format(c) + "}" self.w = writer diff --git a/setup.cfg b/setup.cfg index 907da2d..17328ab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = clicada -version = 0.0.6a1 +version = 0.0.6a2 description = CLI CRM tool for the cic-stack custodial wallet system author = Louis Holbrook author_email = dev@holbrook.no