From 76b331178cd40c1bf65357be8353d5fd998cb61c Mon Sep 17 00:00:00 2001 From: nolash Date: Sat, 6 Nov 2021 06:29:45 +0100 Subject: [PATCH] Add tagger --- clicada/cli/arg.py | 13 +++++--- clicada/cli/tag.py | 41 ++++++++++++++++++++++++ clicada/cli/user.py | 63 +++++++++++++++++++++++++++++++++---- clicada/runnable/view.py | 46 +++------------------------ clicada/user/file.py | 67 +++++++++++++++++++++++++++++++++++++--- 5 files changed, 174 insertions(+), 56 deletions(-) create mode 100644 clicada/cli/tag.py diff --git a/clicada/cli/arg.py b/clicada/cli/arg.py index 4dc9878..c397f7a 100644 --- a/clicada/cli/arg.py +++ b/clicada/cli/arg.py @@ -11,6 +11,7 @@ from chainlib.chain import ChainSpec # local imports import clicada.cli.user as cmd_user +import clicada.cli.tag as cmd_tag script_dir = os.path.dirname(os.path.realpath(__file__)) data_dir = os.path.join(script_dir, '..', 'data') @@ -21,6 +22,7 @@ class CmdCtrl: __cmd_alias = { 'u': 'user', + 't': 'tag', } def __init__(self, argv=None, description=None, logger=None, *args, **kwargs): @@ -35,6 +37,8 @@ class CmdCtrl: sub.dest = 'command' sub_user = sub.add_parser('user', aliases=['u'], help='retrieve transactions for a user') cmd_user.process_args(sub_user) + sub_tag = sub.add_parser('tag', aliases=['t'], help='locally assign a display value to an identifier') + cmd_tag.process_args(sub_tag) self.cmd_args = self.argparser.parse_args(argv) @@ -57,10 +61,6 @@ class CmdCtrl: logger.debug('using module {}'.format(modname)) self.cmd_mod = importlib.import_module(modname) -# if self.cmd_args.c: -# self.config = confini.Config(base_config_dir, override_dirs=self.cmd_args.c) -# else: -# self.config = confini.Config(base_config_dir) extra_args = self.cmd_mod.extra_args() logger.debug('using extra args {}'.format(extra_args)) if self.cmd_args.config: @@ -87,9 +87,14 @@ class CmdCtrl: return self.config.true(k) return r + def chain(self): return self.chain_spec def conn(self): return self.__conn + + + def execute(self): + self.cmd_mod.execute(self) diff --git a/clicada/cli/tag.py b/clicada/cli/tag.py new file mode 100644 index 0000000..0f1f5a4 --- /dev/null +++ b/clicada/cli/tag.py @@ -0,0 +1,41 @@ +# standard imports +import json + +# external imports +from clicada.user import FileUserStore + + +categories = [ + 'phone', + 'address', + ] + + +def process_args(argparser): + argparser.add_argument('--category', required=True, type=str, help='Identifier category') + argparser.add_argument('identifier', type=str, help='Identifier to store a display tag for') + argparser.add_argument('tag', type=str, help='Display tag to store for the identifier') + + +def extra_args(): + return { + 'category': None, + 'identifier': None, + 'tag': None, + } + + +def apply_args(config, args): + pass + + +def validate(config, args): + if category not in categories: + raise ValueError('Invalid category. Valid categories are: {}'.format(','.join(categories))) + + +def execute(ctrl): + store_path = '.clicada' + user_store = FileUserStore(ctrl.chain(), ctrl.get('_CATEGORY'), store_path, int(ctrl.get('FILESTORE_TTL'))) + user_store.put(ctrl.get('_IDENTIFIER'), json.dumps(ctrl.get('_TAG')), force=True) + user_store.stick(ctrl.get('_IDENTIFIER')) diff --git a/clicada/cli/user.py b/clicada/cli/user.py index f817670..1254755 100644 --- a/clicada/cli/user.py +++ b/clicada/cli/user.py @@ -1,3 +1,22 @@ +# standard imports +import sys +import logging + +# external imports +from cic_eth_registry import CICRegistry +from cic_eth_registry.lookup.tokenindex import TokenIndexLookup +from cic_types.ext.metadata import MetadataRequestsHandler +from chainlib.eth.address import to_checksum_address + +# local imports +from clicada.tx import TxGetter +from clicada.user import FileUserStore +from clicada.token import FileTokenStore +from clicada.tx import ResolvedTokenTx + +logg = logging.getLogger(__name__) + + def process_args(argparser): argparser.add_argument('-m', '--method', type=str, help='lookup method') argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from') @@ -14,13 +33,45 @@ def extra_args(): } -def validate_args(args): - pass - - def apply_args(config, args): + if config.get('META_LOOKUP_METHOD'): + raise NotImplementedError('Sorry, currently only "phone" lookup method is implemented') + + +def validate(config, args): pass -def execute(config, args): - pass +def execute(ctrl): + tx_getter = TxGetter(ctrl.get('TX_CACHE_URL')) + + MetadataRequestsHandler.base_url = ctrl.get('META_URL') + + store_path = '.clicada' + user_phone_file_label = 'phone' + user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) + + user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'), update=ctrl.get('_FORCE')) + if user_address == None: + sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_ARG_USER_IDENTIFIER'))) + sys.exit(1) + try: + user_address = to_checksum_address(user_address) + except ValueError: + sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_ARG_USER_IDENTIFIER'))) + sys.exit(1) + + + logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_ARG_USER_IDENTIFIER'))) + + txs = tx_getter.get(user_address) + + token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) + + user_address_file_label = 'address' + user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) + + for tx_src in txs['data']: + tx = ResolvedTokenTx.from_dict(tx_src) + tx.resolve(token_store, user_address_store, update=ctrl.get('_FORCE')) + print(tx) diff --git a/clicada/runnable/view.py b/clicada/runnable/view.py index 1542461..54dacef 100644 --- a/clicada/runnable/view.py +++ b/clicada/runnable/view.py @@ -2,53 +2,17 @@ import sys import logging -# external imports -from cic_eth_registry import CICRegistry -from cic_eth_registry.lookup.tokenindex import TokenIndexLookup -from cic_types.ext.metadata import MetadataRequestsHandler -from chainlib.eth.address import to_checksum_address - # local imports -from clicada.tx import TxGetter -from clicada.user import FileUserStore -from clicada.token import FileTokenStore from clicada.cli import CmdCtrl -from clicada.tx import ResolvedTokenTx logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -ctrl = CmdCtrl(argv=sys.argv[1:], logger=logg) -tx_getter = TxGetter(ctrl.get('TX_CACHE_URL')) - -MetadataRequestsHandler.base_url = ctrl.get('META_URL') - -store_path = '.clicada' -user_phone_file_label = 'phone' -user_phone_store = FileUserStore(ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) - -user_address = user_phone_store.by_phone(ctrl.get('_ARG_USER_IDENTIFIER'), update=ctrl.get('_FORCE')) -if user_address == None: - sys.stderr.write('unknown identifier: {}\n'.format(ctrl.get('_ARG_USER_IDENTIFIER'))) - sys.exit(1) -try: - user_address = to_checksum_address(user_address) -except ValueError: - sys.stderr.write('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_ARG_USER_IDENTIFIER'))) - sys.exit(1) +def main(ctrl): + ctrl.execute() -logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_ARG_USER_IDENTIFIER'))) - -txs = tx_getter.get(user_address) - -token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path) - -user_address_file_label = 'address' -user_address_store = FileUserStore(ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL'))) - -for tx_src in txs['data']: - tx = ResolvedTokenTx.from_dict(tx_src) - tx.resolve(token_store, user_address_store, update=ctrl.get('_FORCE')) - print(tx) +if __name__ == '__main__': + ctrl = CmdCtrl(argv=sys.argv[1:], logger=logg) + main(ctrl) diff --git a/clicada/user/file.py b/clicada/user/file.py index 01abf09..c94e594 100644 --- a/clicada/user/file.py +++ b/clicada/user/file.py @@ -29,6 +29,7 @@ class FileUserStore: def __init__(self, chain_spec, label, store_base_path, ttl): invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl) self.invalidate_before = int(invalidate_before.timestamp()) + self.have_xattr = False self.label = label self.store_path = os.path.join( store_base_path, @@ -69,14 +70,64 @@ class FileUserStore: logg.info('added user store {} record {} -> {}'.format(self.label, k, v)) + def stick(self, k): + p = os.path.join(self.store_path, k) + + if self.have_xattr: + os.setxattr(p, 'cic_sticky', b'1') + else: + (s_h, s_t) = os.path.split(p) + sp = os.path.join(s_h, '.stick_' + s_t) + f = open(sp, 'w') + f.close() + logg.debug('wrote {}'.format(sp)) + + + def __is_sticky(self, p): + if self.have_xattr: + if not os.getxattr(p, 'cic_sticky'): + return False + else: + (s_h, s_t) = os.path.split(p) + sp = os.path.join(s_h, '.stick_' + s_t) + try: + os.stat(sp) + except FileNotFoundError: + return False + return True + + + def __unstick(self, p): + (s_h, s_t) = os.path.split(p) + sp = os.path.join(s_h, '.stick_' + s_t) + os.unlink(sp) + + + def path(self, k): + return os.path.join(self.store_path, k) + + + def sticky(self, k): + p = self.path(k) + return self.__is_sticky(p) + + def check_expiry(self, p): st = os.stat(p) if st[stat.ST_MTIME] < self.invalidate_before: + if self.__is_sticky(p): + logg.debug('ignoring sticky record expiry for {}'.format(p)) + return raise ExpiredRecordError('record in {} is expired'.format(p)) - def get(self, k): - p = os.path.join(self.store_path, k) + def get(self, k, ignore_expired=False): + try: + p = os.path.join(self.store_path, k) + except FileNotFoundError as e: + if self.__is_sticky(p): + logg.warning('removed orphaned sticky record for ' + p) + self.__unstick(p) self.check_expiry(p) f = open(p, 'r') @@ -90,9 +141,11 @@ class FileUserStore: phone = phonenumbers.format_number(phone, phonenumbers.PhoneNumberFormat.E164) phone_file = phone.replace('+', '') + ignore_expired = self.sticky(phone_file) + if not update: try: - v = self.get(phone_file) + v = self.get(phone_file, ignore_expired=ignore_expired) return v except FileNotFoundError: pass @@ -118,13 +171,17 @@ class FileUserStore: def get_label(self, address, update=False): add = tx_normalize.wallet_address(address) + ignore_expired = self.sticky(address) if not update: try: - v = self.get(address) + v = self.get(address, ignore_expired=ignore_expired) v = json.loads(v) person = Person() - person_data = person.deserialize(person_data=v) + try: + person_data = person.deserialize(person_data=v) + except Exception as e: + person_data = v return str(person_data) except FileNotFoundError: pass