chore: small cleanup

Reviewed-on: #17

.gitignore

@@ -2,3 +2,7 @@ __pycache__
 *.egg-info
 build/
 *.pyc
+.venv
+.clicada
+dist/
+.vscode/

CHANGELOG

@@ -1,3 +1,5 @@
+- 0.0.7
+	* fix: make store_path relative to the users home 
 - 0.0.6
 	* Add cache encryption, with AES-CTR-128
 - 0.0.5

README.md

@@ -0,0 +1,87 @@
+## Clicada
+
+> Admin Command Line Interface to interact with cic-meta and cic-cache
+
+### Pre-requisites
+
+- Public key uploaded to `cic-auth-helper`
+- PGP Keyring for your key
+
+### Installation
+
+Use either of the following installation methods:
+
+1. Install from git release (recommended)
+
+```bash
+wget https://git.grassecon.net/grassrootseconomics/clicada/archive/v0.0.6.zip
+unzip clicada-v0.0.6.zip
+cd clicada
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt --extra-index-url=https://pip.grassrootseconomics.net
+```
+
+2. Install from pip to path (non sudo)
+
+```bash
+pip3 install -UI --extra-index-url=https://pip.grassrootseconomics.net clicada
+```
+
+### GPG Keyring setup
+
+PGP uses the default keyring, you can however pass in a custom keyring path.
+
+To create a keyring from a specific key and get its path for `AUTH_KEYRING_PATH`:
+
+```bash
+# In some dir
+gpg --homedir .gnupg --import private.pgp
+pwd
+```
+
+### Usage
+
+```bash
+usage: clicada [...optional arguments] [...positional arguments]
+
+positional arguments:
+  {user,u,tag,t}
+    user (u)            retrieve transactions for a user
+    tag (t)             locally assign a display value to an identifier
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --no-logs             Turn off all logging
+  -v                    Be verbose
+  -vv                   Be very verbose
+  -c CONFIG, --config CONFIG
+                        Configuration directory
+  -n NAMESPACE, --namespace NAMESPACE
+                        Configuration namespace
+  --dumpconfig {env,ini}
+                        Output configuration and quit. Use with --raw to omit values and output schema only.
+  --env-prefix ENV_PREFIX
+                        environment prefix for variables to overwrite configuration
+  -p P, --rpc-provider P
+                        RPC HTTP(S) provider url
+  --rpc-dialect RPC_DIALECT
+                        RPC HTTP(S) backend dialect
+  --height HEIGHT       Block height to execute against
+  -i I, --chain-spec I  Chain specification string
+  -u, --unsafe          Do not verify address checksums
+  --seq                 Use sequential rpc ids
+  -y Y, --key-file Y    Keystore file to use for signing or address
+  --raw                 Do not decode output
+  --fee-price FEE_PRICE
+                        override fee price
+  --fee-limit FEE_LIMIT
+                        override fee limit
+
+```
+
+### Example
+
+```bash
+AUTH_PASSPHRASE=queenmarlena AUTH_KEYRING_PATH=/home/kamikaze/grassroots/usumbufu/tests/testdata/pgp/.gnupg/ AUTH_KEY=CCE2E1D2D0E36ADE0405E2D0995BB21816313BD5 CHAIN_SPEC=evm:byzantium:8996:bloxberg CIC_REGISTRY_ADDRESS=0xcf60ebc445b636a5ab787f9e8bc465a2a3ef8299 RPC_PROVIDER=https://rpc.grassecon.net TX_CACHE_URL=https://cache.grassecon.net HTTP_CORS_ORIGIN=https://auth.grassecon.net META_HTTP_ORIGIN=https://auth.grassecon.net:443 PYTHONPATH=. python clicada/runnable/view.py u --meta-url https://auth.grassecon.net +254711000000
+```

clicada/cli/arg.py

@@ -1,46 +1,43 @@
 # import notifier
 from clicada.cli.notify import NotifyWriter
+
 notifier = NotifyWriter()
-#notifier.notify('loading script')
+# notifier.notify('loading script')
+
+import importlib
+import logging
 
 # standard imports
 import os
-import logging
-import importlib
 import sys
 
-# external imports
-import confini
 import chainlib.eth.cli
-from chainlib.chain import ChainSpec
+import clicada.cli.tag as cmd_tag
 
 # local imports
 import clicada.cli.user as cmd_user
-import clicada.cli.tag as cmd_tag
+
+# external imports
+import confini
+from chainlib.chain import ChainSpec
 from clicada.cli.auth import PGPAuthCrypt
-from clicada.cli.http import (
-        HTTPSession,
-        PGPClientSession,
-        )
+from clicada.cli.http import HTTPSession, PGPClientSession
 from clicada.crypt.aes import AESCTREncrypt
 
 logg = logging.getLogger()
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-data_dir = os.path.join(script_dir, '..', 'data')
-base_config_dir = os.path.join(data_dir, 'config')
+data_dir = os.path.join(script_dir, "..", "data")
+base_config_dir = os.path.join(data_dir, "config")
 
 
 class NullWriter:
-
     def notify(self, v):
         pass
 
-
     def ouch(self, v):
         pass
 
-
     def write(self, v):
         sys.stdout.write(str(v))
 
@@ -48,13 +45,13 @@ class NullWriter:
 class CmdCtrl:
 
     __cmd_alias = {
-            'u': 'user',
-            't': 'tag',
-            }
+        "u": "user",
+        "t": "tag",
+    }
 
     __auth_for = [
-        'user', 
-            ]
+        "user",
+    ]
 
     def __init__(self, argv=None, description=None, logger=None, *args, **kwargs):
         self.args(argv)
@@ -62,7 +59,7 @@ class CmdCtrl:
         self.logging(logger)
 
         self.module()
-       
+
         self.config()
 
         self.notifier()
@@ -72,29 +69,36 @@ class CmdCtrl:
         self.blockchain()
 
         self.remote_openers = {}
-        if self.get('META_URL') != None:
+        if self.get("META_URL") != None:
             auth_client_session = PGPClientSession(self.__auth)
-            self.remote_openers['meta'] = HTTPSession(self.get('META_URL'), auth=auth_client_session, origin=self.config.get('META_HTTP_ORIGIN'))
-
+            self.remote_openers["meta"] = HTTPSession(
+                self.get("META_URL"),
+                auth=auth_client_session,
+                origin=self.config.get("META_HTTP_ORIGIN"),
+            )
 
     def blockchain(self):
-        self.chain_spec = ChainSpec.from_chain_str(self.config.get('CHAIN_SPEC'))
+        self.chain_spec = ChainSpec.from_chain_str(self.config.get("CHAIN_SPEC"))
         self.rpc = chainlib.eth.cli.Rpc()
         self.__conn = self.rpc.connect_by_config(self.config)
 
-
     def args(self, argv):
-        self.argparser = chainlib.eth.cli.ArgumentParser(chainlib.eth.cli.argflag_std_read)
+        self.argparser = chainlib.eth.cli.ArgumentParser(
+            chainlib.eth.cli.argflag_std_read
+        )
         sub = self.argparser.add_subparsers()
-        sub.dest = 'command'
-        sub_user = sub.add_parser('user', aliases=['u'], help='retrieve transactions for a user')
+        sub.dest = "command"
+        sub_user = sub.add_parser(
+            "user", aliases=["u"], help="retrieve transactions for a user"
+        )
         cmd_user.process_args(sub_user)
-        sub_tag = sub.add_parser('tag', aliases=['t'], help='locally assign a display value to an identifier')
+        sub_tag = sub.add_parser(
+            "tag", aliases=["t"], help="locally assign a display value to an identifier"
+        )
         cmd_tag.process_args(sub_tag)
 
         self.cmd_args = self.argparser.parse_args(argv)
 
-
     def module(self):
         self.cmd_string = self.cmd_args.command
         cmd_string_translate = self.__cmd_alias.get(self.cmd_string)
@@ -102,12 +106,11 @@ class CmdCtrl:
             self.cmd_string = cmd_string_translate
 
         if self.cmd_string == None:
-            self.cmd_string = 'none'
-   
-        modname = 'clicada.cli.{}'.format(self.cmd_string)
-        self.logger.debug('using module {}'.format(modname))
-        self.cmd_mod = importlib.import_module(modname)
+            self.cmd_string = "none"
 
+        modname = "clicada.cli.{}".format(self.cmd_string)
+        self.logger.debug("using module {}".format(modname))
+        self.cmd_mod = importlib.import_module(modname)
 
     def logging(self, logger):
         self.logger = logger
@@ -117,74 +120,75 @@ class CmdCtrl:
             self.logger.setLevel(logging.DEBUG)
         elif self.cmd_args.v:
             self.logger.setLevel(logging.INFO)
-            
 
     def config(self):
         override_dir = self.cmd_args.config
         if override_dir == None:
-            p = os.environ.get('HOME')
+            p = os.environ.get("HOME")
             if p != None:
-                p = os.path.join(p, '.config', 'cic', 'clicada')
+                p = os.path.join(p, ".config", "cic", "clicada")
                 try:
                     os.stat(p)
                     override_dir = p
-                    logg.info('applying user config override from standard location: {}'.format(p))
+                    logg.info(
+                        "applying user config override from standard location: {}".format(
+                            p
+                        )
+                    )
                 except FileNotFoundError:
                     pass
         extra_args = self.cmd_mod.extra_args()
-        self.config = chainlib.eth.cli.Config.from_args(self.cmd_args, base_config_dir=base_config_dir, extra_args=extra_args, default_config_dir=override_dir)
+        self.config = chainlib.eth.cli.Config.from_args(
+            self.cmd_args,
+            base_config_dir=base_config_dir,
+            extra_args=extra_args,
+            default_config_dir=override_dir,
+        )
 
-        self.config.add(False, '_SEQ')
+        self.config.add(False, "_SEQ")
 
-        self.config.censor('AUTH_PASSPHRASE')
-
-        self.logger.debug('loaded config:\n{}'.format(self.config))
+        self.config.censor("AUTH_PASSPHRASE")
 
+        self.logger.debug("loaded config:\n{}".format(self.config))
 
     def auth(self):
-        typ = self.get('AUTH_TYPE')
-        if typ != 'gnupg':
-            raise NotImplementedError('Valid aut implementations are: gnupg')
+        typ = self.get("AUTH_TYPE")
+        if typ != "gnupg":
+            raise NotImplementedError("Valid aut implementations are: gnupg")
         default_auth_db_path = None
-        if os.environ.get('HOME') != None:
-            default_auth_db_path = os.path.join(os.environ['HOME'], '.local/share/cic/clicada')
-        auth_db_path = self.get('AUTH_DB_PATH', default_auth_db_path)
-        self.__auth = PGPAuthCrypt(auth_db_path, self.get('AUTH_KEY'), self.get('AUTH_KEYRING_PATH'))
-        self.__auth.get_secret(self.get('AUTH_PASSPHRASE'))
+        if os.environ.get("HOME") != None:
+            default_auth_db_path = os.path.join(
+                os.environ["HOME"], ".local/share/cic/clicada"
+            )
+        auth_db_path = self.get("AUTH_DB_PATH", default_auth_db_path)
+        self.__auth = PGPAuthCrypt(
+            auth_db_path, self.get("AUTH_KEY"), self.get("AUTH_KEYRING_PATH")
+        )
+        self.__auth.get_secret(self.get("AUTH_PASSPHRASE"))
         self.encrypter = AESCTREncrypt(auth_db_path, self.__auth.secret)
 
-
     def get(self, k, default=None):
         r = self.config.get(k, default)
         if k in [
-                '_FORCE',
-                '_FORCE_ALL',
-                '_RAW_TX',
-                '_NO_RESOLVE',
-                '_NO_TX',
-                ]:
+            "_FORCE",
+        ]:
             if r == None:
                 return False
             return self.config.true(k)
         return r
 
-
     def chain(self):
         return self.chain_spec
 
-
     def conn(self):
         return self.__conn
 
-
     def execute(self):
         self.cmd_mod.execute(self)
 
-
     def opener(self, k):
         return self.remote_openers[k]
 
-
     def notifier(self):
         if logg.root.level >= logging.WARNING:
             logging.disable()
@@ -192,18 +196,12 @@ class CmdCtrl:
         else:
             self.writer = NullWriter()
 
-
     def notify(self, v):
-        self.writer.notify(v)
-
+        if logg.root.level <= logging.INFO:
+            print("\033[96m" + v + "\033[0m")
 
     def ouch(self, v):
-        self.writer.ouch(v)
-        print()
-
+        print("\033[91m" + v + "\033[0m")
 
     def write(self, v):
-        self.writer.write("")
-        self.writer.write(v)
-        print()
-
+        print(v)

clicada/cli/tag.py

@@ -3,7 +3,8 @@ import json
 
 # external imports
 from clicada.user import FileUserStore
-
+from pathlib import Path
+import os
 
 categories = [
     'phone',
@@ -35,7 +36,7 @@ def validate(config, args):
 
 
 def execute(ctrl):
-    store_path = '.clicada'
+    store_path = os.path.join(str(Path.home()), '.clicada')
     user_store = FileUserStore(None, ctrl.chain(), ctrl.get('_CATEGORY'), store_path, int(ctrl.get('FILESTORE_TTL')))
     user_store.put(ctrl.get('_IDENTIFIER'), json.dumps(ctrl.get('_TAG')), force=True)
     user_store.stick(ctrl.get('_IDENTIFIER'))

clicada/cli/user.py

@@ -1,35 +1,24 @@
 # standard imports
-import sys
-import logging
 import datetime
-from queue import SimpleQueue as Queue
+import logging
+import os
+import sys
+from pathlib import Path
+
+from chainlib.encode import TxHexNormalizer
+from chainlib.eth.address import is_address, to_checksum_address
 
 # external imports
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
 from cic_types.models.person import Person
-from chainlib.eth.address import to_checksum_address
-from chainlib.encode import TxHexNormalizer
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
+from clicada.error import MetadataNotFoundError
+from clicada.token import FileTokenStore, token_balance
 
 # local imports
-from clicada.tx import (
-        TxGetter,
-        FormattedTokenTx,
-        )
+from clicada.tx import ResolvedTokenTx, TxGetter
 from clicada.user import FileUserStore
-from clicada.token import FileTokenStore
-from clicada.tx import ResolvedTokenTx
-from clicada.tx.file import FileTxStore
-from clicada.error import MetadataNotFoundError
-from clicada.cli.worker import (
-        MetadataResolverWorker,
-        TxResolverWorker,
-        TokenResolverWorker,
-        )
+from hexathon import add_0x
 
 logg = logging.getLogger(__name__)
 
@@ -37,34 +26,36 @@ tx_normalizer = TxHexNormalizer()
 
 
 def process_args(argparser):
-    argparser.add_argument('-m', '--method', type=str, help='lookup method')
-    argparser.add_argument('--meta-url', dest='meta_url', type=str, help='Url to retrieve metadata from')
-    argparser.add_argument('-f', '--force-update', dest='force_update', action='store_true', help='Update records of mutable entries')
-    argparser.add_argument('-ff', '--force-update-all', dest='force_update_all', action='store_true', help='Update records of mutable entries and immutable entries')
-    argparser.add_argument('-N', '--no-resolve', dest='no_resolve', action='store_true', help='Resolve no metadata')
-    argparser.add_argument('--no-tx', dest='no_tx', action='store_true', help='Do not fetch transactions')
-    argparser.add_argument('--raw-tx', dest='raw_tx', action='store_true', help='Also cache raw transaction data')
-    argparser.add_argument('identifier', type=str, help='user identifier')
+    argparser.add_argument("-m", "--method", type=str, help="lookup method")
+    argparser.add_argument(
+        "--meta-url", dest="meta_url", type=str, help="Url to retrieve metadata from"
+    )
+    argparser.add_argument(
+        "-f",
+        "--force-update",
+        dest="force_update",
+        action="store_true",
+        help="Update records of mutable entries",
+    )
+    argparser.add_argument(
+        "identifier", type=str, help="user identifier (phone_number or address)"
+    )
 
 
 def extra_args():
     return {
-        'raw_tx': '_RAW_TX',
-        'force_update': '_FORCE',
-        'force_update_all': '_FORCE_ALL',
-        'method': 'META_LOOKUP_METHOD',
-        'meta_url': 'META_URL',
-        'identifier': '_IDENTIFIER',
-        'no_resolve': '_NO_RESOLVE',
-        'no_tx': '_NO_TX',
-            }
+        "force_update": "_FORCE",
+        "method": "META_LOOKUP_METHOD",
+        "meta_url": "META_URL",
+        "identifier": "_IDENTIFIER",
+    }
 
 
 def apply_args(config, args):
-    if config.get('META_LOOKUP_METHOD'):
-        raise NotImplementedError('Sorry, currently only "phone" lookup method is implemented')
-    if config.true('_FORCE_ALL'):
-        config.add(True, '_FORCE', exists_ok=True)
+    if config.get("META_LOOKUP_METHOD"):
+        raise NotImplementedError(
+            'Sorry, currently only "phone" lookup method is implemented'
+        )
 
 
 def validate(config, args):
@@ -72,157 +63,103 @@ def validate(config, args):
 
 
 def execute(ctrl):
-    tx_getter = TxGetter(ctrl.get('TX_CACHE_URL'), 50)
+    tx_getter = TxGetter(ctrl.get("TX_CACHE_URL"), 10)
 
-    store_path = '.clicada'
-    user_phone_file_label = 'phone'
-    user_phone_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_phone_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
+    store_path = os.path.join(str(Path.home()), ".clicada")
+    user_phone_file_label = "phone"
+    user_phone_store = FileUserStore(
+        ctrl.opener("meta"),
+        ctrl.chain(),
+        user_phone_file_label,
+        store_path,
+        int(ctrl.get("FILESTORE_TTL")),
+        encrypter=ctrl.encrypter,
+    )
 
-    ctrl.notify('resolving identifier {} to wallet address'.format(ctrl.get('_IDENTIFIER')))
-    user_address = user_phone_store.by_phone(ctrl.get('_IDENTIFIER'), update=ctrl.get('_FORCE'))
+    identifier = ctrl.get("_IDENTIFIER")
+    ctrl.notify("resolving identifier {} to wallet address".format(identifier))
+    if is_address(identifier):
+        user_address = identifier
+    else:
+        user_address = user_phone_store.by_phone(identifier, update=ctrl.get("_FORCE"))
     if user_address == None:
-        ctrl.ouch('unknown identifier: {}\n'.format(ctrl.get('_IDENTIFIER')))
+        ctrl.ouch("unknown identifier: {}\n".format(identifier))
         sys.exit(1)
     try:
         user_address = to_checksum_address(user_address)
     except ValueError:
-        ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, ctrl.get('_IDENTIFIER')))
+        ctrl.ouch('invalid response "{}" for {}\n'.format(user_address, identifier))
         sys.exit(1)
 
-    logg.debug('loaded user address {} for {}'.format(user_address, ctrl.get('_IDENTIFIER')))
+    logg.debug("loaded user address {} for {}".format(user_address, identifier))
 
     user_address_normal = tx_normalizer.wallet_address(user_address)
-    ctrl.write("""Results for lookup by phone {}:
-
-Metadata:
-    Network address: {}""".format(
-        ctrl.get('_IDENTIFIER'),
-        add_0x(user_address),
-    )
-    )
-
-    if not ctrl.get('_NO_RESOLVE'):
-
-        token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), 'token', store_path)
-
-        user_address_file_label = 'address'
-        user_address_store = FileUserStore(ctrl.opener('meta'), ctrl.chain(), user_address_file_label, store_path, int(ctrl.get('FILESTORE_TTL')), encrypter=ctrl.encrypter, notifier=ctrl)
-
-        ctrl.notify('resolving metadata for address {}'.format(user_address_normal))
-        try:
-            r = user_address_store.by_address(user_address_normal, update=ctrl.get('_FORCE'))
-        except MetadataNotFoundError as e:
-            ctrl.ouch('could not resolve metadata for user: {}'.format(e))
-            sys.exit(1)
-
-        ctrl.write("""    Chain: {}
-    Name: {}
-    Registered: {}
-    Gender: {}
-    Location: {}
-    Products: {}
-    Tags: {}""".format(
-        ctrl.chain().common_name(),
-        str(r),
-        datetime.datetime.fromtimestamp(r.date_registered).ctime(),
-        r.gender,
-        r.location['area_name'],
-        ','.join(r.products),
-        ','.join(r.tags),
-        )
-    )
-
-    if ctrl.get('_NO_TX'):
-        sys.exit(0)
-
-    raw_rpc = None
-    if ctrl.get('_RAW_TX'):
-        raw_rpc = ctrl.rpc 
-
-    ctrl.notify('retrieving txs for address {}'.format(user_address_normal))
+    ctrl.notify("retrieving txs for address {}".format(user_address_normal))
     txs = tx_getter.get(user_address)
 
-    if ctrl.get('_NO_RESOLVE'):
-        for v in txs['data']:
-            tx = FormattedTokenTx.from_dict(v)
-            ctrl.write(tx)
-        sys.exit(0)
+    token_store = FileTokenStore(ctrl.chain(), ctrl.conn(), "token", store_path)
 
+    user_address_file_label = "address"
+    user_address_store = FileUserStore(
+        ctrl.opener("meta"),
+        ctrl.chain(),
+        user_address_file_label,
+        store_path,
+        int(ctrl.get("FILESTORE_TTL")),
+        encrypter=ctrl.encrypter,
+    )
 
-    token_resolver_queue = Queue()
-    token_result_queue = Queue()
-    token_resolver_worker = TokenResolverWorker(user_address, ctrl, token_store, token_resolver_queue, token_result_queue)
-    token_resolver_worker.start()
-
-    wallets = []
-    for tx in txs['data']:
-        token_resolver_queue.put_nowait(tx['source_token'])
-        token_resolver_queue.put_nowait(tx['destination_token'])
-        if tx['sender'] not in wallets:
-            logg.info('adding wallet {} to metadata lookup'.format(tx['sender']))
-            wallets.append(tx['sender'])
-        if tx['recipient'] not in wallets:
-            wallets.append(tx['recipient'])
-            logg.info('registered wallet {} for metadata lookup'.format(tx['recipient']))
-
-    wallet_threads = []
-    for a in wallets:
-        thread_wallet = MetadataResolverWorker(a, ctrl, user_address_store)
-        thread_wallet.start()
-        wallet_threads.append(thread_wallet)
-    
-    ctrl.notify('wait for metadata resolvers to finish work')
-    for t in wallet_threads:
-        t.join()
-
-    tx_store = FileTxStore(store_path, rpc=raw_rpc, notifier=ctrl)
-    tx_threads = []
-    tx_queue = Queue()
-
-    tx_n = 0
-    for tx_src in txs['data']:
-        tx_hash = strip_0x(tx_src['tx_hash'])
-        tx_worker = TxResolverWorker(tx_hash, tx_src, ctrl, tx_store, token_store, user_address_store, token_resolver_queue, tx_queue, show_decimals=True, update=ctrl.get('_FORCE'))
-        tx_thread = tx_worker.start()
-        tx_threads.append(tx_worker)
-        tx_n += 1
-
-    tx_buf = {}
-
-    for i in range(0, tx_n):
-        tx = tx_queue.get()
-        if tx == None:
-            break
-        # ugh, ugly
-        #k = float('{}.{}'.format(tx.block_number, tx.tx_index))
-        # tx_index is missing, this is temporary sort measure
-        k = str(tx.block_number) + '.' + tx.tx_hash
-        tx_buf[k] = tx
-
-    ctrl.notify('wait for transaction getters to finish work')
-    for tx_thread in tx_threads:
-        tx_thread.join()
-
-    token_resolver_queue.put_nowait(None)
-
-    token_buf = ''
-    while True:
-        l = token_result_queue.get()
-        if l == None:
-            break
-        token_buf += '    {} {}\n'.format(l[0], l[1])
-
-    ctrl.notify('wait for token resolver to finish work')
-    token_resolver_worker.join()
-
-    ctrl.write('')
-    ctrl.write("Balances:")
-    ctrl.write(token_buf)
-    ks = list(tx_buf.keys())
-    ks.sort()
-    ks.reverse()
-    for k in ks:
-        ctrl.write(tx_buf[k])
+    r = None
+    ctrl.write(
+        f"""
+Phone: {ctrl.get("_IDENTIFIER")}
+Network address: {add_0x(user_address)}
+Chain: {ctrl.chain().common_name()}"""
+    )
+    ctrl.notify("resolving metadata for address {}".format(user_address_normal))
+    try:
+        r = user_address_store.by_address(
+            user_address_normal, update=ctrl.get("_FORCE")
+        )
+        if r:
+            ctrl.write(
+                f"""
+Name: { str(r)}
+Registered: {datetime.datetime.fromtimestamp(r.date_registered).ctime()}
+Gender: {r.gender}
+Location: {r.location["area_name"]}
+Products: {",".join(r.products)}
+Tags: {",".join(r.tags)}"""
+            )
+    except MetadataNotFoundError as e:
+        ctrl.ouch(f"MetadataNotFoundError: Could not resolve metadata for user {e}\n")
 
+    tx_lines = []
+    seen_tokens = {}
+    for tx_src in txs["data"]:
+        ctrl.notify("resolve details for tx {}".format(tx_src["tx_hash"]))
+        tx = ResolvedTokenTx.from_dict(tx_src)
+        tx.resolve(
+            token_store,
+            user_address_store,
+            show_decimals=True,
+            update=ctrl.get("_FORCE"),
+        )
+        tx_lines.append(tx)
+        seen_tokens[tx.source_token_label] = tx.source_token
+        seen_tokens[tx.destination_token_label] = tx.destination_token
 
+    for k in seen_tokens.keys():
+        ctrl.notify("resolve token {}".format(seen_tokens[k]))
+        (token_symbol, token_decimals) = token_store.by_address(seen_tokens[k])
+        ctrl.notify(
+            "get token balance for {} => {}".format(token_symbol, seen_tokens[k])
+        )
+        balance = token_balance(ctrl.chain(), ctrl.conn(), seen_tokens[k], user_address)
+        fmt = "{:." + str(token_decimals) + "f}"
+        decimal_balance = fmt.format(balance / (10**token_decimals))
+        ctrl.write("Balances:\n    {} {}".format(token_symbol, decimal_balance))
 
+    print()
+    for l in tx_lines:
+        ctrl.write(l)

clicada/cli/worker.py

@@ -1,87 +0,0 @@
-# standard imports
-import threading
-import logging
-
-# external imports
-from hexathon import strip_0x
-
-# local imports
-from clicada.tx import ResolvedTokenTx
-from clicada.token import token_balance
-from clicada.error import MetadataNotFoundError
-
-logg = logging.getLogger(__name__)
-
-
-class MetadataResolverWorker(threading.Thread):
-    
-    def __init__(self, wallet_address, ctrl, user_address_store):
-        self.user_address_store = user_address_store
-        self.wallet_address = wallet_address
-        self.ctrl = ctrl
-        super(MetadataResolverWorker, self).__init__()
-        
-
-    def run(self):
-        self.ctrl.notify('resolve metadata for {}'.format(self.wallet_address))
-        try:
-            self.user_address_store.by_address(self.wallet_address)
-        except MetadataNotFoundError:
-            logg.info('failed metadata lookup for {}'.format(self.wallet_address))
-
-
-class TxResolverWorker(threading.Thread):
-    def __init__(self, tx_hash, tx_src, ctrl, tx_store, token_store, user_address_store, token_queue, tx_queue, show_decimals=True, update=None):
-        self.tx_hash = tx_hash
-        self.tx_src = tx_src
-        self.ctrl = ctrl
-        self.token_store = token_store
-        self.user_address_store = user_address_store
-        self.show_decimals = show_decimals
-        self.update = update
-        self.token_queue = token_queue
-        self.tx_store = tx_store
-        self.tx_queue = tx_queue
-        super(TxResolverWorker, self).__init__()
-
-
-    def run(self):
-        self.ctrl.notify('resolve details for tx {}'.format(self.tx_hash))
-        tx = ResolvedTokenTx.from_dict(self.tx_src)
-        tx.resolve(self.token_store, self.user_address_store, show_decimals=self.show_decimals, update=self.update, lookup=False)
-        self.tx_store.put(self.tx_hash, str(self.tx_src), overwrite=self.ctrl.get('_FORCE_ALL'))
-        self.tx_queue.put(tx)
-
-
-class TokenResolverWorker(threading.Thread):
-
-    def __init__(self, wallet_address, ctrl, token_store, in_queue, out_queue):
-        super(TokenResolverWorker, self).__init__()
-        self.ctrl = ctrl
-        self.token_store = token_store
-        self.in_queue = in_queue
-        self.out_queue = out_queue
-        self.seen_tokens = {}
-        self.wallet_address = strip_0x(wallet_address)
-
-
-    def run(self):
-        while True:
-            token_address = self.in_queue.get()
-            if token_address == None:
-                logg.debug('token resolver end')
-                self.out_queue.put_nowait(None)
-                return
-            token_address = strip_0x(token_address)
-            if self.seen_tokens.get(token_address) != None:
-                continue
-            logg.debug('resolve token {}'.format(token_address))
-            self.ctrl.notify('resolve token {}'.format(token_address))
-            (token_symbol, token_decimals) = self.token_store.by_address(token_address)
-            self.seen_tokens[token_address] = token_address
-            self.ctrl.notify('get token balance for {} => {}'.format(token_symbol, self.seen_tokens[token_address]))
-            balance = token_balance(self.ctrl.chain(), self.ctrl.conn(), self.seen_tokens[token_address], self.wallet_address)
-            fmt =  '{:.' + str(token_decimals) + 'f}'
-            decimal_balance = fmt.format(balance / (10 ** token_decimals))
-            logg.debug('token balance for {} ({}) is {}'.format(token_symbol, token_address, decimal_balance))
-            self.out_queue.put((token_symbol, decimal_balance,))

clicada/token/token.py

@@ -52,7 +52,7 @@ class FileTokenStore:
         return p
    
 
-    def by_address(self, address, update=False, lookup=True):
+    def by_address(self, address):
         address = tx_normalize.executable_address(address)
 
         token_symbol = self.memstore_symbol.get(address)
@@ -65,30 +65,17 @@ class FileTokenStore:
         try:
             f = open(p, 'r')
         except FileNotFoundError:
-            pass
-
-        if f == None:
-            if not lookup:
-                token_symbol = '???'
-                token_decimals = '???'
-            #self.memstore_symbol.put(address, token_symbol)
-            #self.memstore_decimals.put(address, token_decimals)
-            #logg.warning('token metadata not found and lookup deactivated. Will use 18 decimals as default')
-            #return (token_symbol, token_decimals,)
-
-        if token_symbol == None:
-            if f == None:
-                p = self.__cache_token(address)
-                f = open(p, 'r')
-           
-            token_symbol = f.read()
-            f.close()
-
-            p = os.path.join(self.store_path, token_symbol)
+            p = self.__cache_token(address)
             f = open(p, 'r')
-            r = f.read()
-            f.close()
-            token_decimals = int(r)
+       
+        token_symbol = f.read()
+        f.close()
+
+        p = os.path.join(self.store_path, token_symbol)
+        f = open(p, 'r')
+        r = f.read()
+        f.close()
+        token_decimals = int(r)
 
         self.memstore_symbol.put(address, token_symbol)
         self.memstore_decimals.put(token_symbol, token_decimals)

clicada/tx/file.py

@@ -1,45 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-from chainlib.eth.tx import transaction
-from leveldir.numeric import NumDir
-from leveldir.hex import HexDir
-from hexathon import strip_0x
-
-logg = logging.getLogger(__name__)
-
-
-class FileTxStore:
-
-    subdivision = 100000
-
-    def __init__(self, store_base_path, rpc=None, notifier=None):
-        tx_base_path = os.path.join(store_base_path, 'tx')
-        num_base_path = os.path.join(tx_base_path, 'blocks')
-        hash_base_path = os.path.join(tx_base_path, 'hash')
-        raw_base_path = os.path.join(tx_base_path, 'raw')
-        self.block_index_dir = NumDir(num_base_path)
-        self.hash_index_dir = HexDir(hash_base_path, 32)
-        self.raw_index_dir = HexDir(raw_base_path, 32)
-        self.rpc = rpc
-        self.notifier = notifier
-
-
-    def put(self, k, v, overwrite=False):
-        if self.notifier != None:
-            self.notifier.notify('caching tx data for {}'.format(k))
-        hsh = bytes.fromhex(k)
-        if not overwrite and self.hash_index_dir.have(k):
-            logg.debug('tx store already has {}'.format(k))
-            return
-
-        self.hash_index_dir.add(hsh, v.encode('utf-8'))
-
-        if self.rpc != None:
-            self.notifier.notify('retrieve and cache raw tx data for {}'.format(k))
-            o = transaction(k) 
-            r = self.rpc.conn.do(o)
-            raw = bytes.fromhex(strip_0x(r['raw']))
-            self.raw_index_dir.add(hsh, raw)

clicada/tx/tx.py

@@ -16,32 +16,11 @@ from clicada.error import (
         ExpiredRecordError,
         MetadataNotFoundError,
         )
-from chainlib.eth.address import AddressChecksum
 
 logg = logging.getLogger(__name__)
 
-address_checksummer = AddressChecksum()
-
-
-class FormattedTokenTx(TokenTx):
-
-    def __init__(self):
-        super(FormattedTokenTx, self).__init__()
-        self.symmetric = True
-
-    def __str__(self):
-        if self.symmetric:
-            return '{}    {} => {}    {} {}'.format(
-                    self.date_block_label,
-                    self.sender_label,
-                    self.recipient_label,
-                    self.destination_token_label,
-                    self.to_value_label,
-            )
-
-
-class ResolvedTokenTx(FormattedTokenTx):
 
+class ResolvedTokenTx(TokenTx):
 
     def __init__(self):
         super(ResolvedTokenTx, self).__init__()
@@ -49,12 +28,13 @@ class ResolvedTokenTx(FormattedTokenTx):
         self.destination_token_name = None
         self.source_token_decimals = None
         self.destination_token_decimals = None
+        self.symmetric = True
         self.sender_entity = None
         self.recipient_entity = None
 
 
-    def resolve_tokens(self, token_store, show_decimals=False, update=False, lookup=False):
-        (token_symbol, token_decimals) = token_store.by_address(self.source_token, lookup=False)
+    def resolve_tokens(self, token_store, show_decimals=False, update=False):
+        (token_symbol, token_decimals) = token_store.by_address(self.source_token)
         self.source_token_decimals = token_decimals
         self.source_token_label = token_symbol
         token_value = self.to_value / (10 ** token_decimals)
@@ -79,34 +59,45 @@ class ResolvedTokenTx(FormattedTokenTx):
         self.to_value_label = fmt.format(token_value)
 
 
-    def resolve_entity(self, user_store, address, update=False, lookup=True):
+    def resolve_entity(self, user_store, address):
         try:
-            r = user_store.by_address(address, update=update, lookup=lookup)
+            r = user_store.by_address(address)
         except MetadataNotFoundError:
-            return address_checksummer.sum(address)
+            return address
         return str(r)
 
 
-    def resolve_sender_entity(self, user_store, update=False, lookup=True):
+    def resolve_sender_entity(self, user_store, update=False):
         if self.tx_type == TokenTxType.faucet_giveto.value:
             return 'FAUCET'
-        return self.resolve_entity(user_store, self.sender, update=update, lookup=lookup)
+        return self.resolve_entity(user_store, self.sender)
         
 
-    def resolve_recipient_entity(self, user_store, update=False, lookup=True):
-        return self.resolve_entity(user_store, self.recipient, update=update, lookup=lookup)
+    def resolve_recipient_entity(self, user_store, update=False):
+        return self.resolve_entity(user_store, self.recipient)
 
 
-    def resolve_entities(self, user_store, update=False, lookup=True):
-        self.sender_label = self.resolve_sender_entity(user_store, update=update, lookup=lookup)
-        self.recipient_label = self.resolve_recipient_entity(user_store, update=update, lookup=lookup)
+    def resolve_entities(self, user_store, update=False):
+        self.sender_label = self.resolve_sender_entity(user_store, update=update)
+        self.recipient_label = self.resolve_recipient_entity(user_store, update=update)
 
 
-    def resolve(self, token_store, user_store, show_decimals=False, update=False, lookup=True):
-        self.resolve_tokens(token_store, show_decimals, update=update, lookup=lookup)
+    def resolve(self, token_store, user_store, show_decimals=False, update=False):
+        self.resolve_tokens(token_store, show_decimals, update=update)
         self.resolve_entities(user_store, update=update)
 
 
+    def __str__(self):
+        if self.symmetric:
+            return '{}    {} => {}    {} {}'.format(
+                    self.date_block_label,
+                    self.sender_label,
+                    self.recipient_label,
+                    self.destination_token_label,
+                    self.to_value_label,
+            )
+
+
 class TxGetter:
 
     def __init__(self, cache_url, limit=0):

clicada/user/file.py

@@ -12,7 +12,7 @@ from cic_types.condiments import MetadataPointer
 from cic_types.models.person import Person
 from cic_types.ext.requests import make_request
 from cic_types.processor import generate_metadata_pointer
-import requests.exceptions
+from requests.exceptions import HTTPError
 import phonenumbers
 
 # local imports
@@ -65,7 +65,7 @@ class Account(Person):
 
 class FileUserStore:
 
-    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None, notifier=None):
+    def __init__(self, metadata_opener, chain_spec, label, store_base_path, ttl, encrypter=None):
         invalidate_before = datetime.datetime.now() - datetime.timedelta(seconds=ttl)
         self.invalidate_before = int(invalidate_before.timestamp())
         self.have_xattr = False
@@ -83,7 +83,6 @@ class FileUserStore:
         self.metadata_opener = metadata_opener
         self.failed_entities = {}
         self.encrypter = encrypter
-        self.notifier = notifier
 
 
     def __validate_dir(self):
@@ -192,6 +191,7 @@ class FileUserStore:
 
         if self.encrypter != None:
             v = self.encrypter.decrypt(k, v)
+            logg.debug('>>>>>>>>>>>>< v decoded {}'.format(v))
             v = v.decode('utf-8')
 
         logg.debug('retrieved {} from {}'.format(k, p))
@@ -215,8 +215,6 @@ class FileUserStore:
                 logg.info(e)
                 pass
 
-        self.notifier.notify('wallet address for phone {} not found locally, retrieve from metadata service'.format(phone))
-
         getter = self.metadata_opener
         ptr = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
         r = None
@@ -224,7 +222,7 @@ class FileUserStore:
         try:
             r = getter.open(ptr)
             user_address = json.loads(r)
-        except requests.exceptions.HTTPError as e:
+        except HTTPError as e:
             logg.debug('no address found for phone {}: {}'.format(phone, e))
             return None
 
@@ -241,12 +239,13 @@ class FileUserStore:
         return person_data
 
 
-    def by_address(self, address, update=False, lookup=True):
+    def by_address(self, address, update=False):
         address = tx_normalize.wallet_address(address)
         address = strip_0x(address)
+        #if self.failed_entities.get(address):
         if self.is_dud(address):
             logg.debug('already tried and failed {}, skipping'.format(address))
-            raise MetadataNotFoundError()
+            return address
 
         ignore_expired = self.sticky(address)
 
@@ -261,8 +260,6 @@ class FileUserStore:
                 logg.info(e)
                 pass
 
-        self.notifier.notify('metadata for wallet {} not found locally, retrieve from metadata service'.format(address))
-
         getter = self.metadata_opener
 
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.PERSON)
@@ -272,7 +269,7 @@ class FileUserStore:
         except Exception as e:
             logg.debug('no metadata found for {}: {}'.format(address, e))
         
-        if r == None:
+        if not r:
             self.failed_entities[address] = True
             raise MetadataNotFoundError()
 
@@ -280,8 +277,6 @@ class FileUserStore:
         person = Account()
         person_data = person.deserialize(person_data=data)
 
-        logg.debug('wallet {} resolved to {}'.format(address, str(person)))
-        self.notifier.notify('wallet {} resolved to {}, retrieve extended metadata from metadata service'.format(address, str(person)))
         ptr = generate_metadata_pointer(bytes.fromhex(address), MetadataPointer.CUSTOM)
         r = None
         try:

requirements.txt

@@ -1,11 +1,10 @@
-usumbufu~=0.3.5
-confini~=0.5.3
-cic-eth-registry~=0.6.1
-cic-types~=0.2.1a8
+usumbufu~=0.3.8
+confini~=0.6.0
+cic-eth-registry~=0.6.9
+cic-types~=0.2.2
 phonenumbers==8.12.12
-eth-erc20~=0.1.2
+eth-erc20~=0.3.0
 hexathon~=0.1.0
 pycryptodome~=3.10.1
-chainlib-eth~=0.0.21
-chainlib~=0.0.17
-leveldir~=0.1.0
+chainlib-eth~=0.1.0
+chainlib~=0.1.0

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = clicada
-version = 0.0.7rc1
+version = 0.1.3
 description = CLI CRM tool for the cic-stack custodial wallet system
 author = Louis Holbrook
 author_email = dev@holbrook.no