Add local record expiry

This commit is contained in:
nolash 2021-11-06 04:42:18 +01:00
parent e8b6173ec5
commit f648724205
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
7 changed files with 84 additions and 44 deletions

View File

@ -78,8 +78,14 @@ class CmdCtrl:
def get(self, k): def get(self, k):
return self.config.get(k) r = self.config.get(k)
if k in [
'_FORCE',
]:
if r == None:
return False
return self.config.true(k)
return r
def chain(self): def chain(self):
return self.chain_spec return self.chain_spec

View File

@ -1,11 +1,13 @@
def process_args(argparser): def process_args(argparser):
argparser.add_argument('-m', '--method', type=str, help='lookup method') argparser.add_argument('-m', '--method', type=str, help='lookup method')
argparser.add_argument('--meta-url', dest='meta_url', type=str, help='url to retrieve metadata from') argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
argparser.add_argument('-f', '--force-update', dest='force_update', type=str, help='Update records of mutable entries')
argparser.add_argument('identifier', type=str, help='user identifier') argparser.add_argument('identifier', type=str, help='user identifier')
def extra_args(): def extra_args():
return { return {
'force_update': '_FORCE',
'method': 'META_LOOKUP_METHOD', 'method': 'META_LOOKUP_METHOD',
'meta_url': 'META_URL', 'meta_url': 'META_URL',
'identifier': '_ARG_USER_IDENTIFIER', 'identifier': '_ARG_USER_IDENTIFIER',

View File

@ -2,6 +2,9 @@
lookup_method = phone lookup_method = phone
url = url =
[filestore]
ttl = 86400
[tx] [tx]
cache_url = cache_url =

2
clicada/error.py Normal file
View File

@ -0,0 +1,2 @@
class ExpiredRecordError(Exception):
pass

View File

@ -26,14 +26,9 @@ MetadataRequestsHandler.base_url = ctrl.get('META_URL')
store_path = '.clicada' store_path = '.clicada'
user_phone_file_label = 'phone' user_phone_file_label = 'phone'
user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path) user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
#try:
# user_phone_store.put('25413241324', '5c4f9EeE1a6375d30f50ab547cD4EE21B37ac8Ab')
#except FileExistsError:
# logg.debug('already have record')
#k = '25413241324' user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'), update=ctrl.get('_FORCE'))
user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'))
if user_address == None: if user_address == None:
sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_ARG_USER_IDENTIFIER'))) sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_ARG_USER_IDENTIFIER')))
sys.exit(1) sys.exit(1)
@ -51,9 +46,9 @@ txs = tx_getter.get(user_address)
token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
user_address_file_label = 'address' user_address_file_label = 'address'
user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path) user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')))
for tx_src in txs['data']: for tx_src in txs['data']:
tx = ResolvedTokenTx.from_dict(tx_src) tx = ResolvedTokenTx.from_dict(tx_src)
tx.resolve(token_store, user_address_store) tx.resolve(token_store, user_address_store, update=ctrl.get('_FORCE'))
print(tx) print(tx)

View File

@ -12,6 +12,7 @@ from cic_types.models.tx import (
# local imports # local imports
from clicada.encode import tx_normalize from clicada.encode import tx_normalize
from clicada.error import ExpiredRecordError
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)
@ -27,7 +28,7 @@ class ResolvedTokenTx(TokenTx):
self.recipient_entity = None self.recipient_entity = None
def resolve_tokens(self, token_store, show_decimals=False): def resolve_tokens(self, token_store, show_decimals=False, update=False):
(token_symbol, token_decimals) = token_store.by_address(self.source_token) (token_symbol, token_decimals) = token_store.by_address(self.source_token)
self.source_token_label = token_symbol self.source_token_label = token_symbol
token_value = self.to_value / (10 ** token_decimals) token_value = self.to_value / (10 ** token_decimals)
@ -49,17 +50,21 @@ class ResolvedTokenTx(TokenTx):
self.to_value_label = fmt.format(token_value) self.to_value_label = fmt.format(token_value)
def resolve_stored_entity(self, user_store, address): def resolve_stored_entity(self, user_store, address, update=False):
if update:
return None
address = tx_normalize.wallet_address(address) address = tx_normalize.wallet_address(address)
try: try:
v = user_store.get(address) v = user_store.get(address)
return v return v
except FileNotFoundError: except FileNotFoundError:
return None return None
except ExpiredRecordError:
return None
def resolve_sender_entity(self, user_store): def resolve_sender_entity(self, user_store, update=False):
v = self.resolve_stored_entity(user_store, self.sender) v = self.resolve_stored_entity(user_store, self.sender, update=update)
if v != None: if v != None:
return v return v
if self.tx_type == TokenTxType.faucet_giveto.value: if self.tx_type == TokenTxType.faucet_giveto.value:
@ -67,21 +72,21 @@ class ResolvedTokenTx(TokenTx):
return user_store.get_label(self.sender) return user_store.get_label(self.sender)
def resolve_recipient_entity(self, user_store): def resolve_recipient_entity(self, user_store, update=False):
v = self.resolve_stored_entity(user_store, self.recipient) v = self.resolve_stored_entity(user_store, self.recipient, update=update)
if v != None: if v != None:
return v return v
return user_store.get_label(self.recipient) return user_store.get_label(self.recipient, update=update)
def resolve_entities(self, user_store): def resolve_entities(self, user_store, update=False):
self.sender_label = self.resolve_sender_entity(user_store) self.sender_label = self.resolve_sender_entity(user_store, update=update)
self.recipient_label = self.resolve_recipient_entity(user_store) self.recipient_label = self.resolve_recipient_entity(user_store, update=update)
def resolve(self, token_store, user_store, show_decimals=False): def resolve(self, token_store, user_store, show_decimals=False, update=False):
self.resolve_tokens(token_store, show_decimals) self.resolve_tokens(token_store, show_decimals, update=update)
self.resolve_entities(user_store) self.resolve_entities(user_store, update=update)
def __str__(self): def __str__(self):

View File

@ -4,6 +4,7 @@ import os
import logging import logging
import json import json
import urllib.parse import urllib.parse
import datetime
# external imports # external imports
from hexathon import strip_0x from hexathon import strip_0x
@ -18,13 +19,16 @@ import phonenumbers
# local imports # local imports
from clicada.encode import tx_normalize from clicada.encode import tx_normalize
from clicada.store.mem import MemDictStore from clicada.store.mem import MemDictStore
from clicada.error import ExpiredRecordError
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)
class FileUserStore: class FileUserStore:
def __init__(self, chain_spec, label, store_base_path): def __init__(self, chain_spec, label, store_base_path, ttl):
invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
self.invalidate_before = int(invalidate_before.timestamp())
self.label = label self.label = label
self.store_path = os.path.join( self.store_path = os.path.join(
store_base_path, store_base_path,
@ -47,13 +51,15 @@ class FileUserStore:
def put(self, k, v, force=False): def put(self, k, v, force=False):
have_file = False have_file = False
p = os.path.join(self.store_path, k) p = os.path.join(self.store_path, k)
its_time = True
try: try:
st = os.stat(p) st = os.stat(p)
have_file = True have_file = True
its_time = st[stat.ST_MTIME] < self.invalidate_before
except FileNotFoundError: except FileNotFoundError:
pass pass
if have_file and not force: if have_file and not its_time and not force:
raise FileExistsError('user resolution already exists for {}'.format(k)) raise FileExistsError('user resolution already exists for {}'.format(k))
f = open(p, 'w') f = open(p, 'w')
@ -63,47 +69,68 @@ class FileUserStore:
logg.info('added user store {} record {} -> {}'.format(self.label, k, v)) logg.info('added user store {} record {} -> {}'.format(self.label, k, v))
def check_expiry(self, p):
st = os.stat(p)
if st[stat.ST_MTIME] < self.invalidate_before:
raise ExpiredRecordError('record in {} is expired'.format(p))
def get(self, k): def get(self, k):
p = os.path.join(self.store_path, k) p = os.path.join(self.store_path, k)
self.check_expiry(p)
f = open(p, 'r') f = open(p, 'r')
r = f.read() r = f.read()
f.close() f.close()
return r.strip() return r.strip()
def by_phone(self, phone): def by_phone(self, phone, update=False):
phone = phonenumbers.parse(phone, None) phone = phonenumbers.parse(phone, None)
phone = phonenumbers.format_number(phone, phonenumbers.PhoneNumberFormat.E164) phone = phonenumbers.format_number(phone, phonenumbers.PhoneNumberFormat.E164)
phone_file = phone.replace('+', '') phone_file = phone.replace('+', '')
try:
v = self.get(phone_file) if not update:
return v try:
except FileNotFoundError: v = self.get(phone_file)
pass return v
except FileNotFoundError:
pass
except ExpiredRecordError as e:
logg.info(e)
pass
#getter = MetadataRequestsHandler(MetadataPointer.PHONE, phone.encode('utf-8')) #getter = MetadataRequestsHandler(MetadataPointer.PHONE, phone.encode('utf-8'))
r = None r = None
user_address = None
try: try:
ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
url = urllib.parse.urljoin(MetadataRequestsHandler.base_url, ptr) url = urllib.parse.urljoin(MetadataRequestsHandler.base_url, ptr)
r = make_request('GET', url) r = make_request('GET', url)
return r.json() user_address = r.json()
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
logg.debug('no address found for phone {}: {}'.format(phone, e)) logg.debug('no address found for phone {}: {}'.format(phone, e))
return None return None
self.put(phone_file, user_address)
return user_address
def get_label(self, address): def get_label(self, address, update=False):
add = tx_normalize.wallet_address(address) add = tx_normalize.wallet_address(address)
try: if not update:
v = self.get(address) try:
v = json.loads(v) v = self.get(address)
person = Person() v = json.loads(v)
person_data = person.deserialize(person_data=v) person = Person()
return str(person_data) person_data = person.deserialize(person_data=v)
except FileNotFoundError: return str(person_data)
pass except FileNotFoundError:
pass
except ExpiredRecordError as e:
logg.info(e)
pass
address = strip_0x(address) address = strip_0x(address)
getter = MetadataRequestsHandler(MetadataPointer.PERSON, bytes.fromhex(address)) getter = MetadataRequestsHandler(MetadataPointer.PERSON, bytes.fromhex(address))