refactor: Improve cic-cache

apps/data-seeding/cic_eth/import_balance.py

@@ -1,48 +1,37 @@
 # standard imports
+import argparse
+import json
+import logging
 import os
 import sys
-import logging
-import time
-import argparse
-import sys
-import re
-import hashlib
-import csv
-import json
 
 # external imports
 import confini
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainlib.eth.connection import EthHTTPConnection
-from chainlib.eth.block import (
-        block_latest,
-        )
-from chainlib.hash import keccak256_string_to_hex
+from chainlib.chain import ChainSpec
 from chainlib.eth.address import to_checksum_address
+from chainlib.eth.block import (
+    block_latest,
+)
+from chainlib.eth.connection import EthHTTPConnection
+from chainlib.eth.error import (
+    RequestMismatchException,
+)
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import JSONRPCRequest
-from chainlib.eth.error import (
-        EthException,
-        RequestMismatchException,
-        )
-from chainlib.chain import ChainSpec
-from chainlib.eth.constant import ZERO_ADDRESS
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
-from cic_types.models.person import Person
-from eth_erc20 import ERC20
+from chainlib.hash import keccak256_string_to_hex
+from chainsyncer.backend.memory import MemBackend
+from chainsyncer.driver.head import HeadSyncer
 from cic_eth.cli.chain import chain_interface
+from cic_types.models.person import Person
 from eth_accounts_index import AccountsIndex
 from eth_contract_registry import Registry
+from eth_erc20 import ERC20
 from eth_token_index import TokenUniqueSymbolIndex
-
+from funga.eth.keystore.dict import DictKeystore
+from funga.eth.signer import EIP155Signer
+from hexathon import (
+    strip_0x,
+)
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()

apps/data-seeding/cic_eth/traffic/common/signer.py

@@ -2,8 +2,8 @@
 import logging
 
 # external imports 
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
+from funga.eth.signer import EIP155Signer
+from funga.eth.keystore.dict import DictKeystore
 
 logg = logging.getLogger(__name__)
 

apps/data-seeding/cic_ussd/import_balance.py

@@ -9,8 +9,8 @@ from chainlib.chain import ChainSpec
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.connection import EthHTTPConnection
 from confini import Config
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
+from funga.eth.signer import EIP155Signer
+from funga.eth.keystore.dict import DictKeystore
 
 # local imports
 from import_util import BalanceProcessor, get_celery_worker_status

apps/data-seeding/docker/Dockerfile

@@ -35,6 +35,6 @@ RUN  pip install --index-url $PIP_INDEX_URL  \
      --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
      -r requirements.txt
 
-COPY . . 
+COPY . .
 
 ENTRYPOINT [ ]

apps/data-seeding/eth/import_balance.py

@@ -33,8 +33,8 @@ from chainlib.eth.error import (
         RequestMismatchException,
         )
 from chainlib.chain import ChainSpec
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
+from funga.eth.signer import EIP155Signer
+from funga.eth.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
 from cic_eth.cli.chain import chain_interface

apps/data-seeding/eth/import_users.py

@@ -27,9 +27,9 @@ from cic_types.processor import generate_metadata_pointer
 from cic_types import MetadataPointer
 from eth_accounts_index.registry import AccountRegistry
 from eth_contract_registry import Registry
-from crypto_dev_signer.keystore.dict import DictKeystore
-from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
+from funga.eth.keystore.dict import DictKeystore
+from funga.eth.signer.defaultsigner import EIP155Signer
+from funga.eth.keystore.keyfile import to_dict as to_keyfile_dict
 
 # local imports
 from common.dirs import initialize_dirs

apps/data-seeding/requirements.txt

@@ -1,15 +1,15 @@
 sarafu-faucet~=0.0.7a2
-cic-eth[tools]~=0.12.4a13
-cic-types~=0.2.1a2
-funga>=0.5.1a1,<=0.5.15
+cic-eth[tools]~=0.12.5a11
+cic-types~=0.2.1a8
+funga>=0.5.1
 faker==4.17.1
 chainsyncer~=0.0.7a3
-chainlib-eth~=0.0.10a18
+chainlib-eth~=0.0.15
 eth-address-index~=0.2.4a1
 eth-contract-registry~=0.6.3a3
 eth-accounts-index~=0.1.2a3
-eth-erc20~=0.1.2a3
+eth-erc20==0.1.4
 erc20-faucet~=0.3.2a2
 psycopg2==2.8.6
 liveness~=0.0.1a7
-confini>=0.4.2rc3,<0.5.0
+confini>=0.5.2

apps/data-seeding/verify.py

@@ -68,7 +68,11 @@ admin_tests = [
         'local_key',
         ]
 
-all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
+cache_tests = [
+        'cache_tx_user',
+        ]
+
+all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
@@ -77,11 +81,14 @@ argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', defa
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
 argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
+argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url')
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications')
 argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
+argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
+argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers')
 argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
@@ -115,6 +122,7 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 config.add(args.meta_provider, '_META_PROVIDER', True)
 config.add(args.ussd_provider, '_USSD_PROVIDER', True)
+config.add(args.cache_provider, '_CACHE_PROVIDER', True)
 
 token_symbol = args.token_symbol
 
@@ -351,6 +359,24 @@ class Verifier:
             raise VerifierError(o_retrieved, 'metadata (person)')
 
 
+    def verify_cache_tx_user(self, address, balance=None):
+        url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address)
+        req = urllib.request.Request(url)
+        req.add_header('X_CIC_CACHE_MODE', 'all')
+        try:
+            res = urllib.request.urlopen(req)
+        except urllib.error.HTTPError as e:
+            raise VerifierError(
+                    '({}) {}'.format(url, e),
+                    'cache (tx user)',
+                    )
+        r = json.load(res)
+        if len(r['data']) == 0:
+            raise VerifierError('empty tx list for address {}'.format(address), 'cache (tx user)')
+        for tx in r['data']:
+            logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address))
+
+
     def verify_metadata_phone(self, address, balance=None):
         upper_address = strip_0x(address).upper()
         f = open(os.path.join(
@@ -397,11 +423,13 @@ class Verifier:
         if m != 'CON Welcome':
             raise VerifierError(response_data, 'ussd')
 
+
     def verify_ussd_pins(self, address, balance):
         response_data = send_ussd_request(address, self.data_dir)
         if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
             raise VerifierError(response_data, 'pins')
 
+
     def verify(self, address, balance, debug_stem=None):
   
         for k in active_tests: