Merge remote-tracking branch 'origin/master' into lash/verify-cache

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -1,4 +1,4 @@
 [cic]
 registry_address = 
 trust_address =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
+health_modules =

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -5,7 +5,10 @@ import re
 import base64
 
 # external imports
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
 from cic_cache.cache import (
@@ -19,6 +22,7 @@ logg = logging.getLogger(__name__)
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
 re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
 re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
+re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
 
 DEFAULT_LIMIT = 100
 
@@ -28,9 +32,7 @@ def process_transactions_account_bloom(session, env):
     if not r:
         return None
 
-    address = r[1]
-    if r[2] == None:
-        address = add_0x(address)
+    address = strip_0x(r[1])
     offset = 0
     if r.lastindex > 2:
         offset = r[4]
@@ -113,3 +115,38 @@ def process_transactions_all_data(session, env):
     j = json.dumps(o)
 
     return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_account_data(session, env):
+    r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
+    if not r:
+        return None
+    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+        return None
+
+    logg.debug('got data request {}'.format(env))
+    address = strip_0x(r[1])
+    #if r[2] == None:
+    #    address = add_0x(address)
+    offset = 0
+    if r.lastindex > 2:
+        offset = r[4]
+    limit = DEFAULT_LIMIT
+    if r.lastindex > 4:
+        limit = r[6]
+
+    c = DataCache(session)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
+
+    for r in tx_cache:
+        r['date_block'] = r['date_block'].timestamp()
+
+    o = {
+        'low': lowest_block,
+        'high': highest_block,
+        'data': tx_cache,
+    }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -13,20 +13,18 @@ from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.runnable.daemons.query import (
         process_transactions_account_bloom,
+        process_transactions_account_data,
         process_transactions_all_bloom,
         process_transactions_all_data,
         )
+import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
 
-# process args
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_task
+arg_flags = cic_cache.cli.argflag_std_read
+local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task 
 argparser = cic_cache.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
@@ -50,6 +48,7 @@ def application(env, start_response):
             process_transactions_all_data,
             process_transactions_all_bloom,
             process_transactions_account_bloom,
+            process_transactions_account_data,
             ]:
         r = None
         try:

apps/cic-cache/config/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis:///
-result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -1,3 +0,0 @@
-[cic]
-registry_address = 
-trust_address = 

apps/cic-cache/config/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis://localhost:63379
-result_url = redis://localhost:63379

apps/cic-cache/config/docker/cic.ini

@@ -1,3 +0,0 @@
-[cic]
-registry_address =  
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=grassroots
-PASSWORD=
-HOST=localhost
-PORT=63432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/syncer.ini

@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0

apps/cic-cache/docker/Dockerfile

@@ -18,11 +18,6 @@ COPY . .
 
 RUN python setup.py install
 
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-cache/
-
 # for db migrations
 COPY ./aux/wait-for-it/wait-for-it.sh ./
 COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/

apps/cic-cache/requirements.txt

@@ -2,7 +2,7 @@ alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
 moolb~=0.1.1b2
-cic-eth-registry~=0.6.1a1
+cic-eth-registry~=0.6.1a5
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
@@ -12,3 +12,4 @@ chainsyncer[sql]>=0.0.6a3,<0.1.0
 erc20-faucet>=0.3.2a2, <0.4.0
 chainlib-eth>=0.0.9a14,<0.1.0
 eth-address-index>=0.2.3a4,<0.3.0
+okota>=0.2.4a6,<0.3.0

apps/cic-cache/scripts/migrate.py

@@ -1,14 +1,19 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+
+# standard imports
 import os
 import argparse
 import logging
 import re
 
+# external imports
 import alembic
 from alembic.config import Config as AlembicConfig
 import confini
 
+# local imports
 from cic_cache.db import dsn_from_config
+import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -21,28 +26,25 @@ configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
 
 #config_dir = os.path.join('/usr/local/etc/cic-cache')
 
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_sync
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
+argparser.add_argument('-f', '--force', action='store_true', help='force action')
+argparser.add_argument('--migrations-dir', dest='migrations_dir', type=str, help='migrations directory')
 args = argparser.parse_args()
 
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
+extra_args = {
+    'reset': None,
+    'force': None,
+    'migrations_dir': None,
+        }
 
-config = confini.Config(configdir, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
-migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
+migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE'))
 if not os.path.isdir(migrations_dir):
     logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
     migrations_dir = os.path.join(args.migrations_dir, 'default')

apps/contract-migration/3_deploy_token.sh

@@ -154,9 +154,6 @@ fi
 
 check_wait 3
 
->&2 echo -e "\033[;96mWriting token metadata and proofs\033[;39m"
-python scripts/proofs.py --token-symbol $TOKEN_SYMBOL -e $TOKEN_ADDRESS --address-declarator $DEV_ADDRESS_DECLARATOR --signer-address $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
-
 >&2 echo -e "\033[;96mWriting env_reset file\033[;39m"
 confini-dump --schema-dir ./config > ${DEV_DATA_DIR}/env_reset
 

apps/contract-migration/config/proofs/config.ini

@@ -1,8 +0,0 @@
-[pgp]
-export_dir = pgp/keys/
-keys_path = /tmp/cic/pgp/
-private_keys = privatekeys_meta.asc
-passphrase =
-
-[meta]
-url =

apps/contract-migration/docker/Dockerfile

@@ -11,10 +11,9 @@ RUN cat /etc/apt/sources.list.d/ethereum.list
 RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
 
 RUN mkdir -vp /usr/local/etc/cic
-# create secrets directory
-RUN mkdir -vp pgp/keys
 
-COPY requirements.txt .
+COPY config_template/ /usr/local/etc/cic/
+COPY requirements.txt . 
 
 #RUN apt-get install libffi-dev
 

apps/contract-migration/requirements.txt

@@ -10,4 +10,3 @@ sarafu-faucet>=0.0.7a2,<0.1.0
 confini>=0.4.2rc3,<1.0.0
 eth-token-index>=0.2.4a1,<=0.3.0
 okota>=0.2.4a15,<0.3.0
-cic-types~=0.2.1a2

apps/contract-migration/scripts/proofs.py

@@ -1,114 +0,0 @@
-# standard imports
-import hashlib
-import json
-import logging
-import os
-from typing import Union
-
-# external imports
-import cic_eth.cli
-from chainlib.chain import ChainSpec
-from chainlib.eth.connection import EthHTTPConnection
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.nonce import RPCNonceOracle
-from cic_types.condiments import MetadataPointer
-from cic_types.ext.metadata import MetadataRequestsHandler, Signer
-from eth_address_declarator import Declarator
-from eth_address_declarator.declarator import AddressDeclarator
-from funga.eth.signer import EIP155Signer
-from funga.eth.keystore.dict import DictKeystore
-from hexathon import add_0x, strip_0x
-
-# local imports
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.join(script_dir, '..')
-base_config_dir = os.path.join(root_dir, 'config', 'proofs')
-token_data_dir = os.path.join(root_dir, 'token_data')
-
-arg_flags = cic_eth.cli.argflag_std_base
-arg_parser = cic_eth.cli.ArgumentParser(arg_flags)
-arg_parser.add_argument('--token-symbol', type=str, help='Token symbol whose metadata is being processed.')
-arg_parser.add_argument('--address-declarator', type=str, help='Address to address declarator contract')
-arg_parser.add_argument('--signer-address', type=str, help='Wallet keyfile address')
-arg_parser.add_argument('-e', type=str, help='Token address.')
-args = arg_parser.parse_args()
-config = cic_eth.cli.Config.from_args(args, arg_flags, 0, base_config_dir=base_config_dir)
-
-token_metadata = os.path.join(token_data_dir, 'meta.json')
-token_proof = os.path.join(token_data_dir, 'proof.json')
-
-
-def hash_proof(data: bytes) -> hex:
-    hash_object = hashlib.sha256()
-    hash_object.update(data)
-    return hash_object.digest().hex()
-
-
-def init_meta():
-    MetadataRequestsHandler.base_url = config.get('META_URL')
-    Signer.gpg_path = config.get('PGP_EXPORT_DIR')
-    Signer.key_file_path = f"{config.get('PGP_KEYS_PATH')}{config.get('PGP_PRIVATE_KEYS')}"
-    Signer.gpg_passphrase = config.get('PGP_PASSPHRASE')
-
-
-def wrapper(chain_spec: ChainSpec, rpc: EthHTTPConnection) -> Declarator:
-    gas_oracle = OverrideGasOracle(limit=AddressDeclarator.gas(), conn=rpc)
-    nonce_oracle = RPCNonceOracle(address=add_0x(args.signer_address.lower()), conn=rpc)
-    keystore = DictKeystore()
-    keystore.import_keystore_file(keystore_file=config.get('WALLET_KEY_FILE'))
-    signer = EIP155Signer(keystore)
-    return Declarator(chain_spec, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, signer=signer)
-
-
-def write_to_declarator(contract_address: hex, contract_wrapper: Declarator, proof: any, rpc: EthHTTPConnection, signer_address: hex, token_address: hex):
-    operation = contract_wrapper.add_declaration(contract_address, signer_address, token_address, proof)
-    results = rpc.do(operation[1])
-    rpc.wait(results)
-
-
-def write_metadata(writer: MetadataRequestsHandler, data: Union[dict, str]):
-    writer.create(data)
-
-
-if __name__ == '__main__':
-    init_meta()
-    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-
-    token_address_bytes = bytes.fromhex(strip_0x(args.e))
-    token_symbol_bytes = args.token_symbol.encode('utf-8')
-    token_meta_file = open(token_metadata, 'r')
-    token_proof_file = open(token_proof, 'r')
-    token_meta_data = json.load(token_meta_file)
-    token_proof_data = json.load(token_proof_file)
-    token_meta_file.close()
-    token_proof_file.close()
-
-    token_meta_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_META, identifier=token_address_bytes)
-    write_metadata(token_meta_writer, token_meta_data)
-
-    token_meta_symbol_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_META_SYMBOL, identifier=token_symbol_bytes)
-    write_metadata(token_meta_symbol_writer, token_meta_data)
-
-    token_proof_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_PROOF, identifier=token_address_bytes)
-    write_metadata(token_proof_writer, token_proof_data)
-
-    token_proof_symbol_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_PROOF_SYMBOL, identifier=token_symbol_bytes)
-    write_metadata(token_proof_symbol_writer, token_proof_data)
-
-    rpc = EthHTTPConnection(url=config.get('RPC_PROVIDER'), chain_spec=chain_spec)
-    contract_wrapper = wrapper(chain_spec, rpc)
-
-    hashed_token_proof = hash_proof(data=json.dumps(token_proof_data).encode('utf-8'))
-    identifier = bytes.fromhex(hashed_token_proof)
-    token_immutable_proof_writer = MetadataRequestsHandler(cic_type=MetadataPointer.NONE, identifier=identifier)
-    write_metadata(token_immutable_proof_writer, token_proof_data)
-    write_to_declarator(contract_address=args.address_declarator,
-                        contract_wrapper=contract_wrapper,
-                        proof=hashed_token_proof,
-                        rpc=rpc,
-                        signer_address=args.signer_address,
-                        token_address=args.e)

apps/contract-migration/token_data/meta.json

@@ -1,6 +0,0 @@
-{
-	"contact": {"phone":  "+254757628885", "email":  "info@grassrootseconomics.org"},
-	"country_code": "KE",
-	"location": "Kilifi",
-	"name": "GRASSROOTS ECONOMICS"
-}

apps/contract-migration/token_data/proof.json

@@ -1,7 +0,0 @@
-{
-	"description": "Community support",
-	"issuer": "Grassroots Economics",
-	"namespace": "ge",
-	"proofs": [],
-	"version": 0
-}

apps/data-seeding/verify.py

@@ -68,7 +68,11 @@ admin_tests = [
         'local_key',
         ]
 
-all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
+cache_tests = [
+        'cache_tx_user',
+        ]
+
+all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
@@ -77,11 +81,14 @@ argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', defa
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
 argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
+argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url')
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications')
 argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
+argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
+argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers')
 argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
@@ -115,6 +122,7 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 config.add(args.meta_provider, '_META_PROVIDER', True)
 config.add(args.ussd_provider, '_USSD_PROVIDER', True)
+config.add(args.cache_provider, '_CACHE_PROVIDER', True)
 
 token_symbol = args.token_symbol
 
@@ -351,6 +359,24 @@ class Verifier:
             raise VerifierError(o_retrieved, 'metadata (person)')
 
 
+    def verify_cache_tx_user(self, address, balance=None):
+        url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address)
+        req = urllib.request.Request(url)
+        req.add_header('X_CIC_CACHE_MODE', 'all')
+        try:
+            res = urllib.request.urlopen(req)
+        except urllib.error.HTTPError as e:
+            raise VerifierError(
+                    '({}) {}'.format(url, e),
+                    'cache (tx user)',
+                    )
+        r = json.load(res)
+        if len(r['data']) == 0:
+            raise VerifierError('empty tx list for address {}'.format(address), 'cache (tx user)')
+        for tx in r['data']:
+            logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address))
+
+
     def verify_metadata_phone(self, address, balance=None):
         upper_address = strip_0x(address).upper()
         f = open(os.path.join(
@@ -397,11 +423,13 @@ class Verifier:
         if m != 'CON Welcome':
             raise VerifierError(response_data, 'ussd')
 
+
     def verify_ussd_pins(self, address, balance):
         response_data = send_ussd_request(address, self.data_dir)
         if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
             raise VerifierError(response_data, 'pins')
 
+
     def verify(self, address, balance, debug_stem=None):
   
         for k in active_tests:

docker-compose.yml

@@ -78,18 +78,14 @@ services:
       REDIS_PORT_CALLBACK: ${REDIS_PORT_CALLBACK:-6379}
       FAUCET_AMOUNT: ${FAUCET_AMOUNT:-0}
       WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c}
-      PGP_PASSPHRASE: merman
-      META_URL: http://meta:8000
     command: ["./run_job.sh"]
     depends_on:
       - evm
       - postgres
       - redis
-      - cic-meta-server
-      - cic-eth-tasker
+        #- cic-eth-tasker
     volumes:
       - contract-config:/tmp/cic/config
-      - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp
 
 
 
@@ -163,14 +159,6 @@ services:
     volumes:
       - signer-data:/run/crypto-dev-signer
       - contract-config:/tmp/cic/config/:ro
-    command:
-      - /bin/bash
-      - -c
-      - |
-        set -a
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        set +a
-        ./start_tasker.sh --aux-all -q cic-eth  -vv
 
 
   cic-eth-tracker: