feat: Add cache encryption #9
@ -1,3 +1,10 @@
|
|||||||
|
- 0.0.5
|
||||||
|
* Replace logs with colorized progress output on default loglevel
|
||||||
|
* Do not repeat already failed metadata lookups
|
||||||
|
- 0.0.4
|
||||||
|
* Resolve metadata to labels when loading from cache
|
||||||
|
- 0.0.3
|
||||||
|
* Upgrade usumbufu to prevent missing bearer auth on https
|
||||||
- 0.0.2
|
- 0.0.2
|
||||||
* Use ~/.config for default config override
|
* Use ~/.config for default config override
|
||||||
- 0.0.1-unreleased
|
- 0.0.1-unreleased
|
||||||
|
@ -1,8 +1,13 @@
|
|||||||
|
# import notifier
|
||||||
|
from clicada.cli.notify import NotifyWriter
|
||||||
|
notifier = NotifyWriter()
|
||||||
|
notifier.notify('loading script')
|
||||||
|
|
||||||
# standard imports
|
# standard imports
|
||||||
import os
|
import os
|
||||||
#import argparse
|
|
||||||
import logging
|
import logging
|
||||||
import importlib
|
import importlib
|
||||||
|
import sys
|
||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
import confini
|
import confini
|
||||||
@ -25,6 +30,20 @@ data_dir = os.path.join(script_dir, '..', 'data')
|
|||||||
base_config_dir = os.path.join(data_dir, 'config')
|
base_config_dir = os.path.join(data_dir, 'config')
|
||||||
|
|
||||||
|
|
||||||
|
class NullWriter:
|
||||||
|
|
||||||
|
def notify(self, v):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def ouch(self, v):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def write(self, v):
|
||||||
|
sys.stdout.write(str(v))
|
||||||
|
|
||||||
|
|
||||||
class CmdCtrl:
|
class CmdCtrl:
|
||||||
|
|
||||||
__cmd_alias = {
|
__cmd_alias = {
|
||||||
@ -45,6 +64,8 @@ class CmdCtrl:
|
|||||||
|
|
||||||
self.config()
|
self.config()
|
||||||
|
|
||||||
|
self.notifier()
|
||||||
|
|
||||||
self.auth()
|
self.auth()
|
||||||
|
|
||||||
self.blockchain()
|
self.blockchain()
|
||||||
@ -156,3 +177,27 @@ class CmdCtrl:
|
|||||||
|
|
||||||
def opener(self, k):
|
def opener(self, k):
|
||||||
return self.remote_openers[k]
|
return self.remote_openers[k]
|
||||||
|
|
||||||
|
|
||||||
|
def notifier(self):
|
||||||
|
if logg.root.level >= logging.WARNING:
|
||||||
|
logging.disable()
|
||||||
|
self.writer = notifier
|
||||||
|
else:
|
||||||
|
self.writer = NullWriter()
|
||||||
|
|
||||||
|
|
||||||
|
def notify(self, v):
|
||||||
|
self.writer.notify(v)
|
||||||
|
|
||||||
|
|
||||||
|
def ouch(self, v):
|
||||||
|
self.writer.croak(v)
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def write(self, v):
|
||||||
|
self.writer.write("")
|
||||||
|
self.writer.write(v)
|
||||||
|
print()
|
||||||
|
|
||||||
|
@ -19,6 +19,7 @@ from clicada.token import (
|
|||||||
token_balance,
|
token_balance,
|
||||||
)
|
)
|
||||||
from clicada.tx import ResolvedTokenTx
|
from clicada.tx import ResolvedTokenTx
|
||||||
|
from clicada.error import MetadataNotFoundError
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -51,24 +52,27 @@ def validate(config, args):
|
|||||||
|
|
||||||
|
|
||||||
def execute(ctrl):
|
def execute(ctrl):
|
||||||
tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'))
|
tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'), 10)
|
||||||
|
|
||||||
store_path = '.clicada'
|
store_path = '.clicada'
|
||||||
user_phone_file_label = 'phone'
|
user_phone_file_label = 'phone'
|
||||||
user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
|
user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
|
||||||
|
|
||||||
|
ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
|
||||||
user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
|
user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
|
||||||
if user_address == None:
|
if user_address == None:
|
||||||
sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
|
ctrl.ouch('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
try:
|
try:
|
||||||
user_address = to_checksum_address(user_address)
|
user_address = to_checksum_address(user_address)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
|
ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER')))
|
logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER')))
|
||||||
|
|
||||||
|
user_address_normal = tx_normalizer.wallet_address(user_address)
|
||||||
|
ctrl.notify('retrieving txs for address {}'.format(user_address_normal))
|
||||||
txs = tx_getter.get(user_address)
|
txs = tx_getter.get(user_address)
|
||||||
|
|
||||||
token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
|
token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
|
||||||
@ -76,12 +80,14 @@ def execute(ctrl):
|
|||||||
user_address_file_label = 'address'
|
user_address_file_label = 'address'
|
||||||
user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
|
user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
|
||||||
|
|
||||||
user_address_normal = tx_normalizer.wallet_address(user_address)
|
ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
|
||||||
|
try:
|
||||||
|
r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
|
||||||
|
except MetadataNotFoundError as e:
|
||||||
|
ctrl.ouch('could not resolve metadata for user: {}'.format(e))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
|
ctrl.write("""Phone: {}
|
||||||
print('r {}'.format(r))
|
|
||||||
|
|
||||||
print("""Phone: {}
|
|
||||||
Network address: {}
|
Network address: {}
|
||||||
Chain: {}
|
Chain: {}
|
||||||
Name: {}
|
Name: {}
|
||||||
@ -89,8 +95,7 @@ Registered: {}
|
|||||||
Gender: {}
|
Gender: {}
|
||||||
Location: {}
|
Location: {}
|
||||||
Products: {}
|
Products: {}
|
||||||
Tags: {}
|
Tags: {}""".format(
|
||||||
Balances:""".format(
|
|
||||||
ctrl.get('_IDENTIFIER'),
|
ctrl.get('_IDENTIFIER'),
|
||||||
add_0x(user_address),
|
add_0x(user_address),
|
||||||
ctrl.chain().common_name(),
|
ctrl.chain().common_name(),
|
||||||
@ -106,6 +111,7 @@ Balances:""".format(
|
|||||||
tx_lines = []
|
tx_lines = []
|
||||||
seen_tokens = {}
|
seen_tokens = {}
|
||||||
for tx_src in txs['data']:
|
for tx_src in txs['data']:
|
||||||
|
ctrl.notify('resolve details for tx {}'.format(tx_src['tx_hash']))
|
||||||
tx = ResolvedTokenTx.from_dict(tx_src)
|
tx = ResolvedTokenTx.from_dict(tx_src)
|
||||||
tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
|
tx.resolve(token_store, user_address_store, show_decimals=True, update=ctrl.get('_FORCE'))
|
||||||
tx_lines.append(tx)
|
tx_lines.append(tx)
|
||||||
@ -113,12 +119,14 @@ Balances:""".format(
|
|||||||
seen_tokens[tx.destination_token_label] = tx.destination_token
|
seen_tokens[tx.destination_token_label] = tx.destination_token
|
||||||
|
|
||||||
for k in seen_tokens.keys():
|
for k in seen_tokens.keys():
|
||||||
|
ctrl.notify('resolve token {}'.format(seen_tokens[k]))
|
||||||
(token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
|
(token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
|
||||||
|
ctrl.notify('get token balance for {} => {}'.format(token_symbol, seen_tokens[k]))
|
||||||
balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
|
balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
|
||||||
fmt = '{:.' + str(token_decimals) + 'f}'
|
fmt = '{:.' + str(token_decimals) + 'f}'
|
||||||
decimal_balance = fmt.format(balance / (10 ** token_decimals))
|
decimal_balance = fmt.format(balance / (10 ** token_decimals))
|
||||||
print("\t{} {}".format(token_symbol, decimal_balance))
|
ctrl.write("Balances:\n {} {}".format(token_symbol, decimal_balance))
|
||||||
|
|
||||||
print()
|
print()
|
||||||
for l in tx_lines:
|
for l in tx_lines:
|
||||||
print(l)
|
ctrl.write(l)
|
||||||
|
38
clicada/crypt.py
Normal file
38
clicada/crypt.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
# standard imports
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from Crypto.Cipher import AES
|
||||||
|
from Crypto.Util import Counter
|
||||||
|
|
||||||
|
logg = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class Encrypt:
|
||||||
|
|
||||||
|
aesBlockSize = 1 << 7
|
||||||
|
|
||||||
|
def __init__(self, secret, db_dir):
|
||||||
|
fp = os.path.join(db_dir, '.aes_ctr_iv')
|
||||||
|
try:
|
||||||
|
f = open(fp, 'rb')
|
||||||
|
self.iv = f.read()
|
||||||
|
except FileNotFoundError:
|
||||||
|
logg.debug('generating new iv for aes-ctr')
|
||||||
|
self.iv = os.urandom(8)
|
||||||
|
f = open(fp, 'wb')
|
||||||
|
f.write(self.iv)
|
||||||
|
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
iv_num = int.from_bytes(self.iv, 'big')
|
||||||
|
self.ctr = Counter.new(aesBlockSize, initial_value=iv_num)
|
||||||
|
self.cipher = AES.new(secret, AES.MODE_CTR, counter=self.ctr)
|
||||||
|
|
||||||
|
|
||||||
|
def encrypt(self, v):
|
||||||
|
return self.cipher.encrypt(v)
|
||||||
|
|
||||||
|
|
||||||
|
def decrypt(self, v):
|
||||||
|
return self.cipher.decrypt(v)
|
@ -4,3 +4,7 @@ class ExpiredRecordError(Exception):
|
|||||||
|
|
||||||
class AuthError(Exception):
|
class AuthError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class MetadataNotFoundError(Exception):
|
||||||
|
pass
|
||||||
|
@ -12,7 +12,10 @@ from cic_types.models.tx import (
|
|||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from clicada.encode import tx_normalize
|
from clicada.encode import tx_normalize
|
||||||
from clicada.error import ExpiredRecordError
|
from clicada.error import (
|
||||||
|
ExpiredRecordError,
|
||||||
|
MetadataNotFoundError,
|
||||||
|
)
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -56,35 +59,22 @@ class ResolvedTokenTx(TokenTx):
|
|||||||
self.to_value_label = fmt.format(token_value)
|
self.to_value_label = fmt.format(token_value)
|
||||||
|
|
||||||
|
|
||||||
def resolve_stored_entity(self, user_store, address, update=False):
|
def resolve_entity(self, user_store, address):
|
||||||
if update:
|
|
||||||
return None
|
|
||||||
address = tx_normalize.wallet_address(address)
|
|
||||||
try:
|
try:
|
||||||
v = user_store.get(address)
|
r = user_store.by_address(address)
|
||||||
return v
|
except MetadataNotFoundError:
|
||||||
except FileNotFoundError:
|
return address
|
||||||
return None
|
return str(r)
|
||||||
except ExpiredRecordError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_sender_entity(self, user_store, update=False):
|
def resolve_sender_entity(self, user_store, update=False):
|
||||||
v = self.resolve_stored_entity(user_store, self.sender, update=update)
|
|
||||||
if v != None:
|
|
||||||
return v
|
|
||||||
if self.tx_type == TokenTxType.faucet_giveto.value:
|
if self.tx_type == TokenTxType.faucet_giveto.value:
|
||||||
return 'FAUCET'
|
return 'FAUCET'
|
||||||
r = user_store.by_address(self.sender)
|
return self.resolve_entity(user_store, self.sender)
|
||||||
return str(r)
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_recipient_entity(self, user_store, update=False):
|
def resolve_recipient_entity(self, user_store, update=False):
|
||||||
v = self.resolve_stored_entity(user_store, self.recipient, update=update)
|
return self.resolve_entity(user_store, self.recipient)
|
||||||
if v != None:
|
|
||||||
return v
|
|
||||||
r = user_store.by_address(self.recipient, update=update)
|
|
||||||
return str(r)
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_entities(self, user_store, update=False):
|
def resolve_entities(self, user_store, update=False):
|
||||||
@ -99,7 +89,7 @@ class ResolvedTokenTx(TokenTx):
|
|||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
if self.symmetric:
|
if self.symmetric:
|
||||||
return '{}\t{} => {}\t{} {}'.format(
|
return '{} {} => {} {} {}'.format(
|
||||||
self.date_block_label,
|
self.date_block_label,
|
||||||
self.sender_label,
|
self.sender_label,
|
||||||
self.recipient_label,
|
self.recipient_label,
|
||||||
|
@ -18,7 +18,10 @@ import phonenumbers
|
|||||||
# local imports
|
# local imports
|
||||||
from clicada.encode import tx_normalize
|
from clicada.encode import tx_normalize
|
||||||
from clicada.store.mem import MemDictStore
|
from clicada.store.mem import MemDictStore
|
||||||
from clicada.error import ExpiredRecordError
|
from clicada.error import (
|
||||||
|
ExpiredRecordError,
|
||||||
|
MetadataNotFoundError,
|
||||||
|
)
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -62,7 +65,7 @@ class Account(Person):
|
|||||||
|
|
||||||
class FileUserStore:
|
class FileUserStore:
|
||||||
|
|
||||||
def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl):
|
def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None):
|
||||||
invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
|
invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
|
||||||
self.invalidate_before = int(invalidate_before.timestamp())
|
self.invalidate_before = int(invalidate_before.timestamp())
|
||||||
self.have_xattr = False
|
self.have_xattr = False
|
||||||
@ -78,6 +81,8 @@ class FileUserStore:
|
|||||||
os.makedirs(self.store_path, exist_ok=True)
|
os.makedirs(self.store_path, exist_ok=True)
|
||||||
self.__validate_dir()
|
self.__validate_dir()
|
||||||
self.metadata_opener = metadata_opener
|
self.metadata_opener = metadata_opener
|
||||||
|
self.failed_entities = {}
|
||||||
|
self.encrypter = encrypter
|
||||||
|
|
||||||
|
|
||||||
def __validate_dir(self):
|
def __validate_dir(self):
|
||||||
@ -86,6 +91,10 @@ class FileUserStore:
|
|||||||
logg.debug('using existing file store {} for {}'.format(self.store_path, self.label))
|
logg.debug('using existing file store {} for {}'.format(self.store_path, self.label))
|
||||||
|
|
||||||
|
|
||||||
|
def is_dud(self, address):
|
||||||
|
return bool(self.failed_entities.get(address))
|
||||||
|
|
||||||
|
|
||||||
def put(self, k, v, force=False):
|
def put(self, k, v, force=False):
|
||||||
have_file = False
|
have_file = False
|
||||||
p = os.path.join(self.store_path, k)
|
p = os.path.join(self.store_path, k)
|
||||||
@ -206,27 +215,36 @@ class FileUserStore:
|
|||||||
return user_address
|
return user_address
|
||||||
|
|
||||||
|
|
||||||
|
def metadata_to_person(self, v):
|
||||||
|
person = Account()
|
||||||
|
try:
|
||||||
|
person_data = person.deserialize(person_data=v)
|
||||||
|
except Exception as e:
|
||||||
|
person_data = v
|
||||||
|
return person_data
|
||||||
|
|
||||||
|
|
||||||
def by_address(self, address, update=False):
|
def by_address(self, address, update=False):
|
||||||
add = tx_normalize.wallet_address(address)
|
address = tx_normalize.wallet_address(address)
|
||||||
|
address = strip_0x(address)
|
||||||
|
#if self.failed_entities.get(address):
|
||||||
|
if self.is_dud(address):
|
||||||
|
logg.debug('already tried and failed {}, skipping'.format(address))
|
||||||
|
return None
|
||||||
|
|
||||||
ignore_expired = self.sticky(address)
|
ignore_expired = self.sticky(address)
|
||||||
|
|
||||||
if not update:
|
if not update:
|
||||||
try:
|
try:
|
||||||
v = self.get(address, ignore_expired=ignore_expired)
|
v = self.get(address, ignore_expired=ignore_expired)
|
||||||
v = json.loads(v)
|
v = json.loads(v)
|
||||||
person = Account()
|
return self.metadata_to_person(v)
|
||||||
try:
|
|
||||||
person_data = person.deserialize(person_data=v)
|
|
||||||
except Exception as e:
|
|
||||||
person_data = v
|
|
||||||
return person_data
|
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
pass
|
pass
|
||||||
except ExpiredRecordError as e:
|
except ExpiredRecordError as e:
|
||||||
logg.info(e)
|
logg.info(e)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
address = strip_0x(address)
|
|
||||||
getter = self.metadata_opener
|
getter = self.metadata_opener
|
||||||
|
|
||||||
ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
|
ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
|
||||||
@ -235,7 +253,10 @@ class FileUserStore:
|
|||||||
r = getter.open(ptr)
|
r = getter.open(ptr)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logg.debug('no metadata found for {}: {}'.format(address, e))
|
logg.debug('no metadata found for {}: {}'.format(address, e))
|
||||||
return address
|
|
||||||
|
if r == None:
|
||||||
|
self.failed_entities[address] = True
|
||||||
|
raise MetadataNotFoundError()
|
||||||
|
|
||||||
data = json.loads(r)
|
data = json.loads(r)
|
||||||
person = Account()
|
person = Account()
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[metadata]
|
[metadata]
|
||||||
name = clicada
|
name = clicada
|
||||||
version = 0.0.3
|
version = 0.0.5a1
|
||||||
description = CLI CRM tool for the cic-stack custodial wallet system
|
description = CLI CRM tool for the cic-stack custodial wallet system
|
||||||
author = Louis Holbrook
|
author = Louis Holbrook
|
||||||
author_email = dev@holbrook.no
|
author_email = dev@holbrook.no
|
||||||
|
Loading…
Reference in New Issue
Block a user