Add local record expiry

clicada/cli/arg.py

@@ -78,8 +78,14 @@ class CmdCtrl:
 
 
     def get(self, k):
-        return self.config.get(k)
+        r = self.config.get(k)
-
+        if k in [
+                '_FORCE',
+                ]:
+            if r == None:
+                return False
+            return self.config.true(k)
+        return r
 
     def chain(self):
         return self.chain_spec

clicada/cli/user.py

@@ -1,11 +1,13 @@
 def process_args(argparser):
     argparser.add_argument('-m', '--method', type=str, help='lookup method')
-    argparser.add_argument('--meta-url', dest='meta_url', type=str, help='url to retrieve metadata from')
+    argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
+    argparser.add_argument('-f', '--force-update', dest='force_update', type=str, help='Update records of mutable entries')
     argparser.add_argument('identifier', type=str, help='user identifier')
 
 
 def extra_args():
     return {
+        'force_update': '_FORCE',
         'method': 'META_LOOKUP_METHOD',
         'meta_url': 'META_URL',
         'identifier': '_ARG_USER_IDENTIFIER',

clicada/data/config/config.ini

@@ -2,6 +2,9 @@
 lookup_method = phone
 url =
 
+[filestore]
+ttl = 86400
+
 [tx]
 cache_url = 
 

clicada/error.py

@@ -0,0 +1,2 @@
+class ExpiredRecordError(Exception):
+    pass

clicada/runnable/view.py

@@ -6,6 +6,7 @@ import logging
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
 from cic_types.ext.metadata import MetadataRequestsHandler
+from chainlib.eth.address import to_checksum_address
 
 # local imports
 from clicada.tx import TxGetter
@@ -21,28 +22,33 @@ ctrl = CmdCtrl(argv=sys.argv[1:], logger=logg)
 
 tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'))
 
+MetadataRequestsHandler.base_url = ctrl.get('META_URL')
+
 store_path = '.clicada'
 user_phone_file_label = 'phone'
-user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path)
+user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
-try:
-    user_phone_store.put('25413241324', '5c4f9EeE1a6375d30f50ab547cD4EE21B37ac8Ab')
-except FileExistsError:
-    logg.debug('already have record')
 
-k = '25413241324'
+user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'), update=ctrl.get('_FORCE'))
-user_address = user_phone_store.get(k)
+if user_address == None:
-logg.debug('loaded user address {} for {}'.format(user_address, k))
+    sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_ARG_USER_IDENTIFIER')))
+    sys.exit(1)
+try:
+    user_address = to_checksum_address(user_address)
+except ValueError:
+    sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_ARG_USER_IDENTIFIER')))
+    sys.exit(1)
+
+
+logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_ARG_USER_IDENTIFIER')))
 
 txs = tx_getter.get(user_address)
 
 token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
 
-MetadataRequestsHandler.base_url = ctrl.get('META_URL')
-
 user_address_file_label = 'address'
-user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path)
+user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
 
 for tx_src in txs['data']:
     tx = ResolvedTokenTx.from_dict(tx_src)
-    tx.resolve(token_store, user_address_store)
+    tx.resolve(token_store, user_address_store, update=ctrl.get('_FORCE'))
     print(tx)

clicada/tx/tx.py

@@ -12,6 +12,7 @@ from cic_types.models.tx import (
 
 # local imports
 from clicada.encode import tx_normalize
+from clicada.error import ExpiredRecordError
 
 logg = logging.getLogger(__name__)
 
@@ -27,7 +28,7 @@ class ResolvedTokenTx(TokenTx):
         self.recipient_entity = None
 
 
-    def resolve_tokens(self, token_store, show_decimals=False):
+    def resolve_tokens(self, token_store, show_decimals=False, update=False):
         (token_symbol, token_decimals) = token_store.by_address(self.source_token)
         self.source_token_label = token_symbol
         token_value = self.to_value / (10 ** token_decimals)
@@ -49,39 +50,43 @@ class ResolvedTokenTx(TokenTx):
         self.to_value_label = fmt.format(token_value)
 
 
-    def resolve_stored_entity(self, user_store, address):
+    def resolve_stored_entity(self, user_store, address, update=False):
+        if update:
+            return None
         address = tx_normalize.wallet_address(address)
         try:
             v = user_store.get(address)
             return v
         except FileNotFoundError:
             return None
+        except ExpiredRecordError:
+            return None
 
 
-    def resolve_sender_entity(self, user_store):
+    def resolve_sender_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.sender)
+        v = self.resolve_stored_entity(user_store, self.sender, update=update)
         if v != None:
             return v
         if self.tx_type == TokenTxType.faucet_giveto.value:
             return 'FAUCET'
-        return user_store.get_metadata(self.sender)
+        return user_store.get_label(self.sender)
 
 
-    def resolve_recipient_entity(self, user_store):
+    def resolve_recipient_entity(self, user_store, update=False):
-        v = self.resolve_stored_entity(user_store, self.recipient)
+        v = self.resolve_stored_entity(user_store, self.recipient, update=update)
         if v != None:
             return v
-        return user_store.get_metadata(self.recipient)
+        return user_store.get_label(self.recipient, update=update)
 
 
-    def resolve_entities(self, user_store):
+    def resolve_entities(self, user_store, update=False):
-        self.sender_label = self.resolve_sender_entity(user_store)
+        self.sender_label = self.resolve_sender_entity(user_store, update=update)
-        self.recipient_label = self.resolve_recipient_entity(user_store)
+        self.recipient_label = self.resolve_recipient_entity(user_store, update=update)
 
 
-    def resolve(self, token_store, user_store, show_decimals=False):
+    def resolve(self, token_store, user_store, show_decimals=False, update=False):
-        self.resolve_tokens(token_store, show_decimals)
+        self.resolve_tokens(token_store, show_decimals, update=update)
-        self.resolve_entities(user_store)
+        self.resolve_entities(user_store, update=update)
 
 
     def __str__(self):

clicada/user/file.py

@@ -2,24 +2,33 @@
 import stat
 import os
 import logging
+import json
+import urllib.parse
+import datetime
 
 # external imports
 from hexathon import strip_0x
 from cic_types.condiments import MetadataPointer
 from cic_types.ext.metadata import MetadataRequestsHandler
 from cic_types.models.person import Person
+from cic_types.ext.requests import make_request
+from cic_types.processor import generate_metadata_pointer
 import requests.exceptions
+import phonenumbers
 
 # local imports
 from clicada.encode import tx_normalize
 from clicada.store.mem import MemDictStore
+from clicada.error import ExpiredRecordError
 
 logg = logging.getLogger(__name__)
 
 
 class FileUserStore:
 
-    def __init__(self, chain_spec, label, store_base_path):
+    def __init__(self, chain_spec, label, store_base_path, ttl):
+        invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
+        self.invalidate_before = int(invalidate_before.timestamp())
         self.label = label
         self.store_path = os.path.join(
                 store_base_path,
@@ -42,13 +51,15 @@ class FileUserStore:
     def put(self, k, v, force=False):
         have_file = False
         p = os.path.join(self.store_path, k)
+        its_time = True
         try:
             st = os.stat(p)
             have_file = True
+            its_time = st[stat.ST_MTIME] < self.invalidate_before
         except FileNotFoundError:
             pass
 
-        if have_file and not force:
+        if have_file and not its_time and not force:
             raise FileExistsError('user resolution already exists for {}'.format(k))
 
         f = open(p, 'w')
@@ -58,15 +69,69 @@ class FileUserStore:
         logg.info('added user store {} record {} -> {}'.format(self.label, k, v))
 
 
+    def check_expiry(self, p):
+        st = os.stat(p)
+        if st[stat.ST_MTIME] < self.invalidate_before:
+            raise ExpiredRecordError('record in {} is expired'.format(p))
+
+
     def get(self, k):
         p = os.path.join(self.store_path, k)
+        self.check_expiry(p)
+
         f = open(p, 'r')
         r = f.read()
         f.close()
         return r.strip()
 
 
-    def get_metadata(self, address):
+    def by_phone(self, phone, update=False):
+        phone = phonenumbers.parse(phone, None)
+        phone = phonenumbers.format_number(phone, phonenumbers.PhoneNumberFormat.E164)
+        phone_file = phone.replace('+', '')
+
+        if not update:
+            try:
+                v = self.get(phone_file)
+                return v
+            except FileNotFoundError:
+                pass
+            except ExpiredRecordError as e:
+                logg.info(e)
+                pass
+            
+        #getter = MetadataRequestsHandler(MetadataPointer.PHONE, phone.encode('utf-8'))
+        r = None
+        user_address = None
+        try:
+            ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
+            url = urllib.parse.urljoin(MetadataRequestsHandler.base_url, ptr)
+            r = make_request('GET', url)
+            user_address = r.json()
+        except requests.exceptions.HTTPError as e:
+            logg.debug('no address found for phone {}: {}'.format(phone, e))
+            return None
+
+        self.put(phone_file, user_address)
+        return user_address
+
+    
+    def get_label(self, address, update=False):
+        add = tx_normalize.wallet_address(address)
+
+        if not update:
+            try:
+                v = self.get(address)
+                v = json.loads(v)
+                person = Person()
+                person_data = person.deserialize(person_data=v)
+                return str(person_data)
+            except FileNotFoundError:
+                pass
+            except ExpiredRecordError as e:
+                logg.info(e)
+                pass
+
         address = strip_0x(address)
         getter = MetadataRequestsHandler(MetadataPointer.PERSON, bytes.fromhex(address))
         r = None
@@ -75,7 +140,9 @@ class FileUserStore:
         except requests.exceptions.HTTPError as e:
             logg.debug('no metadata found for {}: {}'.format(address, e))
             return address
+
         data = r.json()
         person = Person()
         person_data = person.deserialize(person_data=data)
+        self.put(address, json.dumps(person_data.serialize()))
         return str(person_data)

requirements.txt

@@ -2,3 +2,4 @@ usumbufu~=0.3.2
 confini~=0.5.1
 cic-eth-registry~=0.6.1
 cic-types~=0.2.1a3
+phonenumbers==8.12.12