diff --git a/apps/cic-cache/cic_cache/data/config/cic.ini b/apps/cic-cache/cic_cache/data/config/cic.ini index b4275352..d2382746 100644 --- a/apps/cic-cache/cic_cache/data/config/cic.ini +++ b/apps/cic-cache/cic_cache/data/config/cic.ini @@ -1,4 +1,4 @@ [cic] registry_address = trust_address = -health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas +health_modules = diff --git a/apps/cic-cache/cic_cache/runnable/daemons/query.py b/apps/cic-cache/cic_cache/runnable/daemons/query.py index d7edf6ab..bd67d962 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/query.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py @@ -5,7 +5,10 @@ import re import base64 # external imports -from hexathon import add_0x +from hexathon import ( + add_0x, + strip_0x, + ) # local imports from cic_cache.cache import ( @@ -19,6 +22,7 @@ logg = logging.getLogger(__name__) re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?' re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?' re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?' +re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?' DEFAULT_LIMIT = 100 @@ -28,9 +32,7 @@ def process_transactions_account_bloom(session, env): if not r: return None - address = r[1] - if r[2] == None: - address = add_0x(address) + address = strip_0x(r[1]) offset = 0 if r.lastindex > 2: offset = r[4] @@ -113,3 +115,38 @@ def process_transactions_all_data(session, env): j = json.dumps(o) return ('application/json', j.encode('utf-8'),) + + +def process_transactions_account_data(session, env): + r = re.match(re_transactions_account_data, env.get('PATH_INFO')) + if not r: + return None + if env.get('HTTP_X_CIC_CACHE_MODE') != 'all': + return None + + logg.debug('got data request {}'.format(env)) + address = strip_0x(r[1]) + #if r[2] == None: + # address = add_0x(address) + offset = 0 + if r.lastindex > 2: + offset = r[4] + limit = DEFAULT_LIMIT + if r.lastindex > 4: + limit = r[6] + + c = DataCache(session) + (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit) + + for r in tx_cache: + r['date_block'] = r['date_block'].timestamp() + + o = { + 'low': lowest_block, + 'high': highest_block, + 'data': tx_cache, + } + + j = json.dumps(o) + + return ('application/json', j.encode('utf-8'),) diff --git a/apps/cic-cache/cic_cache/runnable/daemons/server.py b/apps/cic-cache/cic_cache/runnable/daemons/server.py index 58542d03..a83cbbc0 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/server.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/server.py @@ -13,20 +13,18 @@ from cic_cache.db import dsn_from_config from cic_cache.db.models.base import SessionBase from cic_cache.runnable.daemons.query import ( process_transactions_account_bloom, + process_transactions_account_data, process_transactions_all_bloom, process_transactions_all_data, ) +import cic_cache.cli logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -dbdir = os.path.join(rootdir, 'cic_cache', 'db') -migrationsdir = os.path.join(dbdir, 'migrations') -# process args -arg_flags = cic_cache.cli.argflag_std_base -local_arg_flags = cic_cache.cli.argflag_local_task +arg_flags = cic_cache.cli.argflag_std_read +local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task argparser = cic_cache.cli.ArgumentParser(arg_flags) argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() @@ -50,6 +48,7 @@ def application(env, start_response): process_transactions_all_data, process_transactions_all_bloom, process_transactions_account_bloom, + process_transactions_account_data, ]: r = None try: diff --git a/apps/cic-cache/config/celery.ini b/apps/cic-cache/config/celery.ini deleted file mode 100644 index 6136b166..00000000 --- a/apps/cic-cache/config/celery.ini +++ /dev/null @@ -1,3 +0,0 @@ -[celery] -broker_url = redis:/// -result_url = redis:/// diff --git a/apps/cic-cache/config/cic.ini b/apps/cic-cache/config/cic.ini deleted file mode 100644 index 48c32fad..00000000 --- a/apps/cic-cache/config/cic.ini +++ /dev/null @@ -1,3 +0,0 @@ -[cic] -registry_address = -trust_address = diff --git a/apps/cic-cache/config/config.ini b/apps/cic-cache/config/config.ini new file mode 100644 index 00000000..e69de29b diff --git a/apps/cic-cache/config/database.ini b/apps/cic-cache/config/database.ini deleted file mode 100644 index 7c5836a2..00000000 --- a/apps/cic-cache/config/database.ini +++ /dev/null @@ -1,9 +0,0 @@ -[database] -NAME=cic_cache -USER=postgres -PASSWORD= -HOST=localhost -PORT=5432 -ENGINE=postgresql -DRIVER=psycopg2 -DEBUG=0 diff --git a/apps/cic-cache/config/docker/celery.ini b/apps/cic-cache/config/docker/celery.ini deleted file mode 100644 index 98c5012f..00000000 --- a/apps/cic-cache/config/docker/celery.ini +++ /dev/null @@ -1,3 +0,0 @@ -[celery] -broker_url = redis://localhost:63379 -result_url = redis://localhost:63379 diff --git a/apps/cic-cache/config/docker/cic.ini b/apps/cic-cache/config/docker/cic.ini deleted file mode 100644 index af112724..00000000 --- a/apps/cic-cache/config/docker/cic.ini +++ /dev/null @@ -1,3 +0,0 @@ -[cic] -registry_address = -trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C diff --git a/apps/cic-cache/config/docker/database.ini b/apps/cic-cache/config/docker/database.ini deleted file mode 100644 index db1f5086..00000000 --- a/apps/cic-cache/config/docker/database.ini +++ /dev/null @@ -1,9 +0,0 @@ -[database] -NAME=cic_cache -USER=grassroots -PASSWORD= -HOST=localhost -PORT=63432 -ENGINE=postgresql -DRIVER=psycopg2 -DEBUG=0 diff --git a/apps/cic-cache/config/docker/syncer.ini b/apps/cic-cache/config/docker/syncer.ini deleted file mode 100644 index 255dc00f..00000000 --- a/apps/cic-cache/config/docker/syncer.ini +++ /dev/null @@ -1,4 +0,0 @@ -[syncer] -loop_interval = 1 -offset = 0 -no_history = 0 diff --git a/apps/cic-cache/docker/Dockerfile b/apps/cic-cache/docker/Dockerfile index dc4bf6db..6a49edea 100644 --- a/apps/cic-cache/docker/Dockerfile +++ b/apps/cic-cache/docker/Dockerfile @@ -18,11 +18,6 @@ COPY . . RUN python setup.py install -# ini files in config directory defines the configurable parameters for the application -# they can all be overridden by environment variables -# to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) -#COPY config/ /usr/local/etc/cic-cache/ - # for db migrations COPY ./aux/wait-for-it/wait-for-it.sh ./ COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/ diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 8f47f9b5..4cabe432 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -2,7 +2,7 @@ alembic==1.4.2 confini>=0.3.6rc4,<0.5.0 uwsgi==2.0.19.1 moolb~=0.1.1b2 -cic-eth-registry~=0.6.1a1 +cic-eth-registry~=0.6.1a5 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 @@ -12,3 +12,4 @@ chainsyncer[sql]>=0.0.6a3,<0.1.0 erc20-faucet>=0.3.2a2, <0.4.0 chainlib-eth>=0.0.9a14,<0.1.0 eth-address-index>=0.2.3a4,<0.3.0 +okota>=0.2.4a6,<0.3.0 diff --git a/apps/cic-cache/scripts/migrate.py b/apps/cic-cache/scripts/migrate.py index 458b1ff2..581fd432 100644 --- a/apps/cic-cache/scripts/migrate.py +++ b/apps/cic-cache/scripts/migrate.py @@ -1,14 +1,19 @@ -#!/usr/bin/python +#!/usr/bin/python3 + +# standard imports import os import argparse import logging import re +# external imports import alembic from alembic.config import Config as AlembicConfig import confini +# local imports from cic_cache.db import dsn_from_config +import cic_cache.cli logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -21,28 +26,25 @@ configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config') #config_dir = os.path.join('/usr/local/etc/cic-cache') -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, help='config file') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') +arg_flags = cic_cache.cli.argflag_std_base +local_arg_flags = cic_cache.cli.argflag_local_sync +argparser = cic_cache.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') -argparser.add_argument('-f', action='store_true', help='force action') -argparser.add_argument('-v', action='store_true', help='be verbose') -argparser.add_argument('-vv', action='store_true', help='be more verbose') +argparser.add_argument('-f', '--force', action='store_true', help='force action') +argparser.add_argument('--migrations-dir', dest='migrations_dir', type=str, help='migrations directory') args = argparser.parse_args() -if args.vv: - logging.getLogger().setLevel(logging.DEBUG) -elif args.v: - logging.getLogger().setLevel(logging.INFO) +extra_args = { + 'reset': None, + 'force': None, + 'migrations_dir': None, + } -config = confini.Config(configdir, args.env_prefix) -config.process() -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config:\n{}'.format(config)) +# process config +config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) -migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE')) +migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE')) if not os.path.isdir(migrations_dir): logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE'))) migrations_dir = os.path.join(args.migrations_dir, 'default') diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index b01a5e8a..ca4a14b7 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -68,7 +68,11 @@ admin_tests = [ 'local_key', ] -all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests +cache_tests = [ + 'cache_tx_user', + ] + +all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') @@ -77,11 +81,14 @@ argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', defa argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url') +argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url') argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications') argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications') +argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification') +argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers') argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') @@ -115,6 +122,7 @@ config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') config.add(args.meta_provider, '_META_PROVIDER', True) config.add(args.ussd_provider, '_USSD_PROVIDER', True) +config.add(args.cache_provider, '_CACHE_PROVIDER', True) token_symbol = args.token_symbol @@ -351,6 +359,24 @@ class Verifier: raise VerifierError(o_retrieved, 'metadata (person)') + def verify_cache_tx_user(self, address, balance=None): + url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address) + req = urllib.request.Request(url) + req.add_header('X_CIC_CACHE_MODE', 'all') + try: + res = urllib.request.urlopen(req) + except urllib.error.HTTPError as e: + raise VerifierError( + '({}) {}'.format(url, e), + 'cache (tx user)', + ) + r = json.load(res) + if len(r['data']) == 0: + raise VerifierError('empty tx list for address {}'.format(address), 'cache (tx user)') + for tx in r['data']: + logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address)) + + def verify_metadata_phone(self, address, balance=None): upper_address = strip_0x(address).upper() f = open(os.path.join( @@ -397,11 +423,13 @@ class Verifier: if m != 'CON Welcome': raise VerifierError(response_data, 'ussd') + def verify_ussd_pins(self, address, balance): response_data = send_ussd_request(address, self.data_dir) if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio': raise VerifierError(response_data, 'pins') + def verify(self, address, balance, debug_stem=None): for k in active_tests: diff --git a/docker-compose.yml b/docker-compose.yml index 85c808d1..87b741bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -160,14 +160,6 @@ services: volumes: - signer-data:/run/crypto-dev-signer - contract-config:/tmp/cic/config/:ro - command: - - /bin/bash - - -c - - | - set -a - if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - set +a - ./start_tasker.sh --aux-all -q cic-eth -vv cic-eth-tracker: