From f64872420556e5809bb50ea1a3bde8c5c858e445 Mon Sep 17 00:00:00 2001 From: nolash Date: Sat, 6 Nov 2021 04:42:18 +0100 Subject: [PATCH] Add local record expiry --- clicada/cli/arg.py | 10 ++++-- clicada/cli/user.py | 4 ++- clicada/data/config/config.ini | 3 ++ clicada/error.py | 2 ++ clicada/runnable/view.py | 13 +++---- clicada/tx/tx.py | 31 +++++++++------- clicada/user/file.py | 65 ++++++++++++++++++++++++---------- 7 files changed, 84 insertions(+), 44 deletions(-) create mode 100644 clicada/error.py diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index d94e747..4dc9878 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -78,8 +78,14 @@ class CmdCtrl: def get(self, k): - return self.config.get(k) - + r = self.config.get(k) + if k in [ + '_FORCE', + ]: + if r == None: + return False + return self.config.true(k) + return r def chain(self): return self.chain_spec diff --git a/clicada/cli/user.py b/clicada/cli/user.py index d92f88c..f817670 100644 --- a/clicada/cli/user.py +++ b/clicada/cli/user.py @@ -1,11 +1,13 @@ def process_args(argparser): argparser.add_argument('-m', '--method', type=str, help='lookup method') - argparser.add_argument('--meta-url', dest='meta_url', type=str, help='url to retrieve metadata from') + argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from') + argparser.add_argument('-f', '--force-update', dest='force_update', type=str, help='Update records of mutable entries') argparser.add_argument('identifier', type=str, help='user identifier') def extra_args(): return { + 'force_update': '_FORCE', 'method': 'META_LOOKUP_METHOD', 'meta_url': 'META_URL', 'identifier': '_ARG_USER_IDENTIFIER', diff --git a/clicada/data/config/config.ini b/clicada/data/config/config.ini index 4edaa53..6a8a16d 100644 --- a/clicada/data/config/config.ini +++ b/clicada/data/config/config.ini @@ -2,6 +2,9 @@ lookup_method = phone url = +[filestore] +ttl = 86400 + [tx] cache_url = diff --git a/clicada/error.py b/clicada/error.py new file mode 100644 index 0000000..3b67fd8 --- /dev/null +++ b/clicada/error.py @@ -0,0 +1,2 @@ +class ExpiredRecordError(Exception): + pass diff --git a/clicada/runnable/view.py b/clicada/runnable/view.py index 1431905..1542461 100644 --- a/clicada/runnable/view.py +++ b/clicada/runnable/view.py @@ -26,14 +26,9 @@ MetadataRequestsHandler.base_url = ctrl.get('META_URL') store_path = '.clicada' user_phone_file_label = 'phone' -user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path) -#try: -# user_phone_store.put('25413241324', '5c4f9EeE1a6375d30f50ab547cD4EE21B37ac8Ab') -#except FileExistsError: -# logg.debug('already have record') +user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) -#k = '25413241324' -user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER')) +user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'), update=ctrl.get('_FORCE')) if user_address == None: sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_ARG_USER_IDENTIFIER'))) sys.exit(1) @@ -51,9 +46,9 @@ txs = tx_getter.get(user_address) token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) user_address_file_label = 'address' -user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path) +user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) for tx_src in txs['data']: tx = ResolvedTokenTx.from_dict(tx_src) - tx.resolve(token_store, user_address_store) + tx.resolve(token_store, user_address_store, update=ctrl.get('_FORCE')) print(tx) diff --git a/clicada/tx/tx.py b/clicada/tx/tx.py index 261777d..f239fad 100644 --- a/clicada/tx/tx.py +++ b/clicada/tx/tx.py @@ -12,6 +12,7 @@ from cic_types.models.tx import ( # local imports from clicada.encode import tx_normalize +from clicada.error import ExpiredRecordError logg = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class ResolvedTokenTx(TokenTx): self.recipient_entity = None - def resolve_tokens(self, token_store, show_decimals=False): + def resolve_tokens(self, token_store, show_decimals=False, update=False): (token_symbol, token_decimals) = token_store.by_address(self.source_token) self.source_token_label = token_symbol token_value = self.to_value / (10 ** token_decimals) @@ -49,17 +50,21 @@ class ResolvedTokenTx(TokenTx): self.to_value_label = fmt.format(token_value) - def resolve_stored_entity(self, user_store, address): + def resolve_stored_entity(self, user_store, address, update=False): + if update: + return None address = tx_normalize.wallet_address(address) try: v = user_store.get(address) return v except FileNotFoundError: return None + except ExpiredRecordError: + return None - def resolve_sender_entity(self, user_store): - v = self.resolve_stored_entity(user_store, self.sender) + def resolve_sender_entity(self, user_store, update=False): + v = self.resolve_stored_entity(user_store, self.sender, update=update) if v != None: return v if self.tx_type == TokenTxType.faucet_giveto.value: @@ -67,21 +72,21 @@ class ResolvedTokenTx(TokenTx): return user_store.get_label(self.sender) - def resolve_recipient_entity(self, user_store): - v = self.resolve_stored_entity(user_store, self.recipient) + def resolve_recipient_entity(self, user_store, update=False): + v = self.resolve_stored_entity(user_store, self.recipient, update=update) if v != None: return v - return user_store.get_label(self.recipient) + return user_store.get_label(self.recipient, update=update) - def resolve_entities(self, user_store): - self.sender_label = self.resolve_sender_entity(user_store) - self.recipient_label = self.resolve_recipient_entity(user_store) + def resolve_entities(self, user_store, update=False): + self.sender_label = self.resolve_sender_entity(user_store, update=update) + self.recipient_label = self.resolve_recipient_entity(user_store, update=update) - def resolve(self, token_store, user_store, show_decimals=False): - self.resolve_tokens(token_store, show_decimals) - self.resolve_entities(user_store) + def resolve(self, token_store, user_store, show_decimals=False, update=False): + self.resolve_tokens(token_store, show_decimals, update=update) + self.resolve_entities(user_store, update=update) def __str__(self): diff --git a/clicada/user/file.py b/clicada/user/file.py index c32b88d..01abf09 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -4,6 +4,7 @@ import os import logging import json import urllib.parse +import datetime # external imports from hexathon import strip_0x @@ -18,13 +19,16 @@ import phonenumbers # local imports from clicada.encode import tx_normalize from clicada.store.mem import MemDictStore +from clicada.error import ExpiredRecordError logg = logging.getLogger(__name__) class FileUserStore: - def __init__(self, chain_spec, label, store_base_path): + def __init__(self, chain_spec, label, store_base_path, ttl): + invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl) + self.invalidate_before = int(invalidate_before.timestamp()) self.label = label self.store_path = os.path.join( store_base_path, @@ -47,13 +51,15 @@ class FileUserStore: def put(self, k, v, force=False): have_file = False p = os.path.join(self.store_path, k) + its_time = True try: st = os.stat(p) have_file = True + its_time = st[stat.ST_MTIME] < self.invalidate_before except FileNotFoundError: pass - if have_file and not force: + if have_file and not its_time and not force: raise FileExistsError('user resolution already exists for {}'.format(k)) f = open(p, 'w') @@ -63,47 +69,68 @@ class FileUserStore: logg.info('added user store {} record {} -> {}'.format(self.label, k, v)) + def check_expiry(self, p): + st = os.stat(p) + if st[stat.ST_MTIME] < self.invalidate_before: + raise ExpiredRecordError('record in {} is expired'.format(p)) + + def get(self, k): p = os.path.join(self.store_path, k) + self.check_expiry(p) + f = open(p, 'r') r = f.read() f.close() return r.strip() - def by_phone(self, phone): + def by_phone(self, phone, update=False): phone = phonenumbers.parse(phone, None) phone = phonenumbers.format_number(phone, phonenumbers.PhoneNumberFormat.E164) phone_file = phone.replace('+', '') - try: - v = self.get(phone_file) - return v - except FileNotFoundError: - pass + if not update: + try: + v = self.get(phone_file) + return v + except FileNotFoundError: + pass + except ExpiredRecordError as e: + logg.info(e) + pass + #getter = MetadataRequestsHandler(MetadataPointer.PHONE, phone.encode('utf-8')) r = None + user_address = None try: ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) url = urllib.parse.urljoin(MetadataRequestsHandler.base_url, ptr) r = make_request('GET', url) - return r.json() + user_address = r.json() except requests.exceptions.HTTPError as e: logg.debug('no address found for phone {}: {}'.format(phone, e)) - return None + return None + + self.put(phone_file, user_address) + return user_address - def get_label(self, address): + def get_label(self, address, update=False): add = tx_normalize.wallet_address(address) - try: - v = self.get(address) - v = json.loads(v) - person = Person() - person_data = person.deserialize(person_data=v) - return str(person_data) - except FileNotFoundError: - pass + if not update: + try: + v = self.get(address) + v = json.loads(v) + person = Person() + person_data = person.deserialize(person_data=v) + return str(person_data) + except FileNotFoundError: + pass + except ExpiredRecordError as e: + logg.info(e) + pass address = strip_0x(address) getter = MetadataRequestsHandler(MetadataPointer.PERSON, bytes.fromhex(address))