From 6b3699471bd2e18aa3a028503d267ab0657ef658 Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 4 Nov 2021 06:08:16 +0100 Subject: [PATCH] Revert "Merge branch 'lash/verify-cache' into lash/bloxberg-seeding" This reverts commit 99b0fb5aeda19c2016db2f199ba1492428aa7273, reversing changes made to 58e766aa5804d5e9de355edad1a4a37be2693ddc. --- apps/cic-cache/cic_cache/data/config/cic.ini | 2 +- .../cic_cache/runnable/daemons/query.py | 45 ++----------------- .../cic_cache/runnable/daemons/server.py | 11 ++--- apps/cic-cache/config/celery.ini | 3 ++ apps/cic-cache/config/cic.ini | 3 ++ apps/cic-cache/config/config.ini | 0 apps/cic-cache/config/database.ini | 9 ++++ apps/cic-cache/config/docker/celery.ini | 3 ++ apps/cic-cache/config/docker/cic.ini | 3 ++ apps/cic-cache/config/docker/database.ini | 9 ++++ apps/cic-cache/config/docker/syncer.ini | 4 ++ apps/cic-cache/docker/Dockerfile | 5 +++ apps/cic-cache/requirements.txt | 3 +- apps/cic-cache/scripts/migrate.py | 38 ++++++++-------- apps/data-seeding/verify.py | 30 +------------ docker-compose.yml | 8 ++++ 16 files changed, 78 insertions(+), 98 deletions(-) create mode 100644 apps/cic-cache/config/celery.ini create mode 100644 apps/cic-cache/config/cic.ini delete mode 100644 apps/cic-cache/config/config.ini create mode 100644 apps/cic-cache/config/database.ini create mode 100644 apps/cic-cache/config/docker/celery.ini create mode 100644 apps/cic-cache/config/docker/cic.ini create mode 100644 apps/cic-cache/config/docker/database.ini create mode 100644 apps/cic-cache/config/docker/syncer.ini diff --git a/apps/cic-cache/cic_cache/data/config/cic.ini b/apps/cic-cache/cic_cache/data/config/cic.ini index d2382746..b4275352 100644 --- a/apps/cic-cache/cic_cache/data/config/cic.ini +++ b/apps/cic-cache/cic_cache/data/config/cic.ini @@ -1,4 +1,4 @@ [cic] registry_address = trust_address = -health_modules = +health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas diff --git a/apps/cic-cache/cic_cache/runnable/daemons/query.py b/apps/cic-cache/cic_cache/runnable/daemons/query.py index bd67d962..d7edf6ab 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/query.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py @@ -5,10 +5,7 @@ import re import base64 # external imports -from hexathon import ( - add_0x, - strip_0x, - ) +from hexathon import add_0x # local imports from cic_cache.cache import ( @@ -22,7 +19,6 @@ logg = logging.getLogger(__name__) re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?' re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?' re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?' -re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?' DEFAULT_LIMIT = 100 @@ -32,7 +28,9 @@ def process_transactions_account_bloom(session, env): if not r: return None - address = strip_0x(r[1]) + address = r[1] + if r[2] == None: + address = add_0x(address) offset = 0 if r.lastindex > 2: offset = r[4] @@ -115,38 +113,3 @@ def process_transactions_all_data(session, env): j = json.dumps(o) return ('application/json', j.encode('utf-8'),) - - -def process_transactions_account_data(session, env): - r = re.match(re_transactions_account_data, env.get('PATH_INFO')) - if not r: - return None - if env.get('HTTP_X_CIC_CACHE_MODE') != 'all': - return None - - logg.debug('got data request {}'.format(env)) - address = strip_0x(r[1]) - #if r[2] == None: - # address = add_0x(address) - offset = 0 - if r.lastindex > 2: - offset = r[4] - limit = DEFAULT_LIMIT - if r.lastindex > 4: - limit = r[6] - - c = DataCache(session) - (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit) - - for r in tx_cache: - r['date_block'] = r['date_block'].timestamp() - - o = { - 'low': lowest_block, - 'high': highest_block, - 'data': tx_cache, - } - - j = json.dumps(o) - - return ('application/json', j.encode('utf-8'),) diff --git a/apps/cic-cache/cic_cache/runnable/daemons/server.py b/apps/cic-cache/cic_cache/runnable/daemons/server.py index a83cbbc0..58542d03 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/server.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/server.py @@ -13,18 +13,20 @@ from cic_cache.db import dsn_from_config from cic_cache.db.models.base import SessionBase from cic_cache.runnable.daemons.query import ( process_transactions_account_bloom, - process_transactions_account_data, process_transactions_all_bloom, process_transactions_all_data, ) -import cic_cache.cli logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() +rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +dbdir = os.path.join(rootdir, 'cic_cache', 'db') +migrationsdir = os.path.join(dbdir, 'migrations') -arg_flags = cic_cache.cli.argflag_std_read -local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task +# process args +arg_flags = cic_cache.cli.argflag_std_base +local_arg_flags = cic_cache.cli.argflag_local_task argparser = cic_cache.cli.ArgumentParser(arg_flags) argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() @@ -48,7 +50,6 @@ def application(env, start_response): process_transactions_all_data, process_transactions_all_bloom, process_transactions_account_bloom, - process_transactions_account_data, ]: r = None try: diff --git a/apps/cic-cache/config/celery.ini b/apps/cic-cache/config/celery.ini new file mode 100644 index 00000000..6136b166 --- /dev/null +++ b/apps/cic-cache/config/celery.ini @@ -0,0 +1,3 @@ +[celery] +broker_url = redis:/// +result_url = redis:/// diff --git a/apps/cic-cache/config/cic.ini b/apps/cic-cache/config/cic.ini new file mode 100644 index 00000000..48c32fad --- /dev/null +++ b/apps/cic-cache/config/cic.ini @@ -0,0 +1,3 @@ +[cic] +registry_address = +trust_address = diff --git a/apps/cic-cache/config/config.ini b/apps/cic-cache/config/config.ini deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/cic-cache/config/database.ini b/apps/cic-cache/config/database.ini new file mode 100644 index 00000000..7c5836a2 --- /dev/null +++ b/apps/cic-cache/config/database.ini @@ -0,0 +1,9 @@ +[database] +NAME=cic_cache +USER=postgres +PASSWORD= +HOST=localhost +PORT=5432 +ENGINE=postgresql +DRIVER=psycopg2 +DEBUG=0 diff --git a/apps/cic-cache/config/docker/celery.ini b/apps/cic-cache/config/docker/celery.ini new file mode 100644 index 00000000..98c5012f --- /dev/null +++ b/apps/cic-cache/config/docker/celery.ini @@ -0,0 +1,3 @@ +[celery] +broker_url = redis://localhost:63379 +result_url = redis://localhost:63379 diff --git a/apps/cic-cache/config/docker/cic.ini b/apps/cic-cache/config/docker/cic.ini new file mode 100644 index 00000000..af112724 --- /dev/null +++ b/apps/cic-cache/config/docker/cic.ini @@ -0,0 +1,3 @@ +[cic] +registry_address = +trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C diff --git a/apps/cic-cache/config/docker/database.ini b/apps/cic-cache/config/docker/database.ini new file mode 100644 index 00000000..db1f5086 --- /dev/null +++ b/apps/cic-cache/config/docker/database.ini @@ -0,0 +1,9 @@ +[database] +NAME=cic_cache +USER=grassroots +PASSWORD= +HOST=localhost +PORT=63432 +ENGINE=postgresql +DRIVER=psycopg2 +DEBUG=0 diff --git a/apps/cic-cache/config/docker/syncer.ini b/apps/cic-cache/config/docker/syncer.ini new file mode 100644 index 00000000..255dc00f --- /dev/null +++ b/apps/cic-cache/config/docker/syncer.ini @@ -0,0 +1,4 @@ +[syncer] +loop_interval = 1 +offset = 0 +no_history = 0 diff --git a/apps/cic-cache/docker/Dockerfile b/apps/cic-cache/docker/Dockerfile index 6a49edea..dc4bf6db 100644 --- a/apps/cic-cache/docker/Dockerfile +++ b/apps/cic-cache/docker/Dockerfile @@ -18,6 +18,11 @@ COPY . . RUN python setup.py install +# ini files in config directory defines the configurable parameters for the application +# they can all be overridden by environment variables +# to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) +#COPY config/ /usr/local/etc/cic-cache/ + # for db migrations COPY ./aux/wait-for-it/wait-for-it.sh ./ COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/ diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 4cabe432..8f47f9b5 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -2,7 +2,7 @@ alembic==1.4.2 confini>=0.3.6rc4,<0.5.0 uwsgi==2.0.19.1 moolb~=0.1.1b2 -cic-eth-registry~=0.6.1a5 +cic-eth-registry~=0.6.1a1 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 @@ -12,4 +12,3 @@ chainsyncer[sql]>=0.0.6a3,<0.1.0 erc20-faucet>=0.3.2a2, <0.4.0 chainlib-eth>=0.0.9a14,<0.1.0 eth-address-index>=0.2.3a4,<0.3.0 -okota>=0.2.4a6,<0.3.0 diff --git a/apps/cic-cache/scripts/migrate.py b/apps/cic-cache/scripts/migrate.py index 581fd432..458b1ff2 100644 --- a/apps/cic-cache/scripts/migrate.py +++ b/apps/cic-cache/scripts/migrate.py @@ -1,19 +1,14 @@ -#!/usr/bin/python3 - -# standard imports +#!/usr/bin/python import os import argparse import logging import re -# external imports import alembic from alembic.config import Config as AlembicConfig import confini -# local imports from cic_cache.db import dsn_from_config -import cic_cache.cli logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -26,25 +21,28 @@ configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config') #config_dir = os.path.join('/usr/local/etc/cic-cache') -arg_flags = cic_cache.cli.argflag_std_base -local_arg_flags = cic_cache.cli.argflag_local_sync -argparser = cic_cache.cli.ArgumentParser(arg_flags) -argparser.process_local_flags(local_arg_flags) +argparser = argparse.ArgumentParser() +argparser.add_argument('-c', type=str, help='config file') +argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') +argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') -argparser.add_argument('-f', '--force', action='store_true', help='force action') -argparser.add_argument('--migrations-dir', dest='migrations_dir', type=str, help='migrations directory') +argparser.add_argument('-f', action='store_true', help='force action') +argparser.add_argument('-v', action='store_true', help='be verbose') +argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() -extra_args = { - 'reset': None, - 'force': None, - 'migrations_dir': None, - } +if args.vv: + logging.getLogger().setLevel(logging.DEBUG) +elif args.v: + logging.getLogger().setLevel(logging.INFO) -# process config -config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +config = confini.Config(configdir, args.env_prefix) +config.process() +config.censor('PASSWORD', 'DATABASE') +config.censor('PASSWORD', 'SSL') +logg.debug('config:\n{}'.format(config)) -migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE')) +migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE')) if not os.path.isdir(migrations_dir): logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE'))) migrations_dir = os.path.join(args.migrations_dir, 'default') diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index ca4a14b7..b01a5e8a 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -68,11 +68,7 @@ admin_tests = [ 'local_key', ] -cache_tests = [ - 'cache_tx_user', - ] - -all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests +all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') @@ -81,14 +77,11 @@ argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', defa argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url') -argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url') argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications') argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications') -argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification') -argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers') argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') @@ -122,7 +115,6 @@ config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') config.add(args.meta_provider, '_META_PROVIDER', True) config.add(args.ussd_provider, '_USSD_PROVIDER', True) -config.add(args.cache_provider, '_CACHE_PROVIDER', True) token_symbol = args.token_symbol @@ -359,24 +351,6 @@ class Verifier: raise VerifierError(o_retrieved, 'metadata (person)') - def verify_cache_tx_user(self, address, balance=None): - url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address) - req = urllib.request.Request(url) - req.add_header('X_CIC_CACHE_MODE', 'all') - try: - res = urllib.request.urlopen(req) - except urllib.error.HTTPError as e: - raise VerifierError( - '({}) {}'.format(url, e), - 'cache (tx user)', - ) - r = json.load(res) - if len(r['data']) == 0: - raise VerifierError('empty tx list for address {}'.format(address), 'cache (tx user)') - for tx in r['data']: - logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address)) - - def verify_metadata_phone(self, address, balance=None): upper_address = strip_0x(address).upper() f = open(os.path.join( @@ -423,13 +397,11 @@ class Verifier: if m != 'CON Welcome': raise VerifierError(response_data, 'ussd') - def verify_ussd_pins(self, address, balance): response_data = send_ussd_request(address, self.data_dir) if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio': raise VerifierError(response_data, 'pins') - def verify(self, address, balance, debug_stem=None): for k in active_tests: diff --git a/docker-compose.yml b/docker-compose.yml index 87b741bc..85c808d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -160,6 +160,14 @@ services: volumes: - signer-data:/run/crypto-dev-signer - contract-config:/tmp/cic/config/:ro + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./start_tasker.sh --aux-all -q cic-eth -vv cic-eth-tracker: