Add local record expiry

clicada/cli/arg.py

@@ -78,8 +78,14 @@ class CmdCtrl:
 
 
     def get(self, k):
-        return self.config.get(k)
+        r = self.config.get(k)
-
+        if k in [
+                '_FORCE',
+                ]:
+            if r == None:
+                return False
+            return self.config.true(k)
+        return r
 
     def chain(self):
         return self.chain_spec

clicada/cli/user.py

@@ -1,11 +1,13 @@
 def process_args(argparser):
     argparser.add_argument('-m', '--method', type=str, help='lookup method')
-    argparser.add_argument('--meta-url', dest='meta_url', type=str, help='url to retrieve metadata from')
+    argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
+    argparser.add_argument('-f', '--force-update', dest='force_update', type=str, help='Update records of mutable entries')
     argparser.add_argument('identifier', type=str, help='user identifier')
 
 
 def extra_args():
     return {
+        'force_update': '_FORCE',
         'method': 'META_LOOKUP_METHOD',
         'meta_url': 'META_URL',
         'identifier': '_ARG_USER_IDENTIFIER',

clicada/data/config/config.ini

@@ -2,6 +2,9 @@
 lookup_method = phone
 url =
 
+[filestore]
+ttl = 86400
+
 [tx]
 cache_url = 
 

clicada/error.py

@@ -0,0 +1,2 @@
+class ExpiredRecordError(Exception):
+    pass

clicada/runnable/view.py

@@ -26,14 +26,9 @@ MetadataRequestsHandler.base_url = ctrl.get('META_URL')
 
 store_path = '.clicada'
 user_phone_file_label = 'phone'
-user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path)
+user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
-#try:
-#    user_phone_store.put('25413241324', '5c4f9EeE1a6375d30f50ab547cD4EE21B37ac8Ab')
-#except FileExistsError:
-#    logg.debug('already have record')
 
-#k = '25413241324'
+user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'), update=ctrl.get('_FORCE'))
-user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'))
 if user_address == None:
     sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_ARG_USER_IDENTIFIER')))
     sys.exit(1)
@@ -51,9 +46,9 @@ txs = tx_getter.get(user_address)
 token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
 
 user_address_file_label = 'address'
-user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path)
+user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
 
 for tx_src in txs['data']:
     tx = ResolvedTokenTx.from_dict(tx_src)
-    tx.resolve(token_store, user_address_store)
+    tx.resolve(token_store, user_address_store, update=ctrl.get('_FORCE'))
     print(tx)

clicada/tx/tx.py

@@ -12,6 +12,7 @@ from cic_types.models.tx import (
 
 # local imports
 from clicada.encode import tx_normalize
+from clicada.error import ExpiredRecordError
 
 logg = logging.getLogger(__name__)
 
@@ -27,7 +28,7 @@ class ResolvedTokenTx(TokenTx):
         self.recipient_entity = None
 
 
-    def resolve_tokens(self, token_store, show_decimals=False):
+    def resolve_tokens(self, token_store, show_decimals=False, update=False):
         (token_symbol, token_decimals) = token_store.by_address(self.source_token)
         self.source_token_label = token_symbol
         token_value = self.to_value / (10 ** token_decimals)
@@ -49,17 +50,21 @@ class ResolvedTokenTx(TokenTx):
         self.to_value_label = fmt.format(token_value)
 
 
-    def resolve_stored_entity(self, user_store, address):
+    def resolve_stored_entity(self, user_store, address, update=False):
+        if update:
+            return None
         address = tx_normalize.wallet_address(address)
         try:
             v = user_store.get(address)
             return v
         except FileNotFoundError:
             return None
+        except ExpiredRecordError:
+            return None
 
 
-    def resolve_sender_entity(self, user_store):
+    def resolve_sender_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.sender)
+        v = self.resolve_stored_entity(user_store, self.sender, update=update)
         if v != None:
             return v
         if self.tx_type == TokenTxType.faucet_giveto.value:
@@ -67,21 +72,21 @@ class ResolvedTokenTx(TokenTx):
         return user_store.get_label(self.sender)
 
 
-    def resolve_recipient_entity(self, user_store):
+    def resolve_recipient_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.recipient)
+        v = self.resolve_stored_entity(user_store, self.recipient, update=update)
         if v != None:
             return v
-        return user_store.get_label(self.recipient)
+        return user_store.get_label(self.recipient, update=update)
 
 
-    def resolve_entities(self, user_store):
+    def resolve_entities(self, user_store, update=False):
-        self.sender_label = self.resolve_sender_entity(user_store)
+        self.sender_label = self.resolve_sender_entity(user_store, update=update)
-        self.recipient_label = self.resolve_recipient_entity(user_store)
+        self.recipient_label = self.resolve_recipient_entity(user_store, update=update)
 
 
-    def resolve(self, token_store, user_store, show_decimals=False):
+    def resolve(self, token_store, user_store, show_decimals=False, update=False):
-        self.resolve_tokens(token_store, show_decimals)
+        self.resolve_tokens(token_store, show_decimals, update=update)
-        self.resolve_entities(user_store)
+        self.resolve_entities(user_store, update=update)
 
 
     def __str__(self):

clicada/user/file.py

@@ -4,6 +4,7 @@ import os
 import logging
 import json
 import urllib.parse
+import datetime
 
 # external imports
 from hexathon import strip_0x
@@ -18,13 +19,16 @@ import phonenumbers
 # local imports
 from clicada.encode import tx_normalize
 from clicada.store.mem import MemDictStore
+from clicada.error import ExpiredRecordError
 
 logg = logging.getLogger(__name__)
 
 
 class FileUserStore:
 
-    def __init__(self, chain_spec, label, store_base_path):
+    def __init__(self, chain_spec, label, store_base_path, ttl):
+        invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
+        self.invalidate_before = int(invalidate_before.timestamp())
         self.label = label
         self.store_path = os.path.join(
                 store_base_path,
@@ -47,13 +51,15 @@ class FileUserStore:
     def put(self, k, v, force=False):
         have_file = False
         p = os.path.join(self.store_path, k)
+        its_time = True
         try:
             st = os.stat(p)
             have_file = True
+            its_time = st[stat.ST_MTIME] < self.invalidate_before
         except FileNotFoundError:
             pass
 
-        if have_file and not force:
+        if have_file and not its_time and not force:
             raise FileExistsError('user resolution already exists for {}'.format(k))
 
         f = open(p, 'w')
@@ -63,47 +69,68 @@ class FileUserStore:
         logg.info('added user store {} record {} -> {}'.format(self.label, k, v))
 
 
+    def check_expiry(self, p):
+        st = os.stat(p)
+        if st[stat.ST_MTIME] < self.invalidate_before:
+            raise ExpiredRecordError('record in {} is expired'.format(p))
+
+
     def get(self, k):
         p = os.path.join(self.store_path, k)
+        self.check_expiry(p)
+
         f = open(p, 'r')
         r = f.read()
         f.close()
         return r.strip()
 
 
-    def by_phone(self, phone):
+    def by_phone(self, phone, update=False):
         phone = phonenumbers.parse(phone, None)
         phone = phonenumbers.format_number(phone, phonenumbers.PhoneNumberFormat.E164)
         phone_file = phone.replace('+', '')
-        try:
-            v = self.get(phone_file)
-            return v
-        except FileNotFoundError:
-            pass
 
+        if not update:
+            try:
+                v = self.get(phone_file)
+                return v
+            except FileNotFoundError:
+                pass
+            except ExpiredRecordError as e:
+                logg.info(e)
+                pass
+            
         #getter = MetadataRequestsHandler(MetadataPointer.PHONE, phone.encode('utf-8'))
         r = None
+        user_address = None
         try:
             ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
             url = urllib.parse.urljoin(MetadataRequestsHandler.base_url, ptr)
             r = make_request('GET', url)
-            return r.json()
+            user_address = r.json()
         except requests.exceptions.HTTPError as e:
             logg.debug('no address found for phone {}: {}'.format(phone, e))
-        return None
+            return None
+
+        self.put(phone_file, user_address)
+        return user_address
 
     
-    def get_label(self, address):
+    def get_label(self, address, update=False):
         add = tx_normalize.wallet_address(address)
 
-        try:
+        if not update:
-            v = self.get(address)
+            try:
-            v = json.loads(v)
+                v = self.get(address)
-            person = Person()
+                v = json.loads(v)
-            person_data = person.deserialize(person_data=v)
+                person = Person()
-            return str(person_data)
+                person_data = person.deserialize(person_data=v)
-        except FileNotFoundError:
+                return str(person_data)
-            pass
+            except FileNotFoundError:
+                pass
+            except ExpiredRecordError as e:
+                logg.info(e)
+                pass
 
         address = strip_0x(address)
         getter = MetadataRequestsHandler(MetadataPointer.PERSON, bytes.fromhex(address))