Add verify check for cache, use chainlib cli for cic-cache

This commit is contained in:
nolash 2021-10-08 11:19:21 +02:00
parent 6b59c87f68
commit 980191be4f
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
19 changed files with 102 additions and 102 deletions

View File

@ -1,4 +1,4 @@
[cic] [cic]
registry_address = registry_address =
trust_address = trust_address =
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas health_modules =

View File

@ -5,7 +5,10 @@ import re
import base64 import base64
# external imports # external imports
from hexathon import add_0x from hexathon import (
add_0x,
strip_0x,
)
# local imports # local imports
from cic_cache.cache import ( from cic_cache.cache import (
@ -19,6 +22,7 @@ logg = logging.getLogger(__name__)
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?' re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?' re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?' re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
DEFAULT_LIMIT = 100 DEFAULT_LIMIT = 100
@ -28,9 +32,7 @@ def process_transactions_account_bloom(session, env):
if not r: if not r:
return None return None
address = r[1] address = strip_0x(r[1])
if r[2] == None:
address = add_0x(address)
offset = 0 offset = 0
if r.lastindex > 2: if r.lastindex > 2:
offset = r[4] offset = r[4]
@ -113,3 +115,38 @@ def process_transactions_all_data(session, env):
j = json.dumps(o) j = json.dumps(o)
return ('application/json', j.encode('utf-8'),) return ('application/json', j.encode('utf-8'),)
def process_transactions_account_data(session, env):
r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
if not r:
return None
if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
return None
logg.debug('got data request {}'.format(env))
address = strip_0x(r[1])
#if r[2] == None:
# address = add_0x(address)
offset = 0
if r.lastindex > 2:
offset = r[4]
limit = DEFAULT_LIMIT
if r.lastindex > 4:
limit = r[6]
c = DataCache(session)
(lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
for r in tx_cache:
r['date_block'] = r['date_block'].timestamp()
o = {
'low': lowest_block,
'high': highest_block,
'data': tx_cache,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)

View File

@ -12,36 +12,24 @@ from cic_cache.db import dsn_from_config
from cic_cache.db.models.base import SessionBase from cic_cache.db.models.base import SessionBase
from cic_cache.runnable.daemons.query import ( from cic_cache.runnable.daemons.query import (
process_transactions_account_bloom, process_transactions_account_bloom,
process_transactions_account_data,
process_transactions_all_bloom, process_transactions_all_bloom,
process_transactions_all_data, process_transactions_all_data,
) )
import cic_cache.cli
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
migrationsdir = os.path.join(dbdir, 'migrations')
config_dir = os.path.join('/usr/local/etc/cic-cache') arg_flags = cic_cache.cli.argflag_std_read
local_arg_flags = cic_cache.cli.argflag_local_sync
argparser = argparse.ArgumentParser() argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.add_argument('-c', type=str, default=config_dir, help='config file') argparser.process_local_flags(local_arg_flags)
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be more verbose')
args = argparser.parse_args() args = argparser.parse_args()
if args.vv: # process config
logging.getLogger().setLevel(logging.DEBUG) config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
elif args.v:
logging.getLogger().setLevel(logging.INFO)
config = confini.Config(args.c, args.env_prefix)
config.process()
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config:\n{}'.format(config))
dsn = dsn_from_config(config) dsn = dsn_from_config(config)
SessionBase.connect(dsn, config.true('DATABASE_DEBUG')) SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
@ -58,6 +46,7 @@ def application(env, start_response):
process_transactions_all_data, process_transactions_all_data,
process_transactions_all_bloom, process_transactions_all_bloom,
process_transactions_account_bloom, process_transactions_account_bloom,
process_transactions_account_data,
]: ]:
r = None r = None
try: try:

View File

@ -1,3 +0,0 @@
[celery]
broker_url = redis:///
result_url = redis:///

View File

@ -1,3 +0,0 @@
[cic]
registry_address =
trust_address =

View File

@ -1,9 +0,0 @@
[database]
NAME=cic_cache
USER=postgres
PASSWORD=
HOST=localhost
PORT=5432
ENGINE=postgresql
DRIVER=psycopg2
DEBUG=0

View File

@ -1,3 +0,0 @@
[celery]
broker_url = redis://localhost:63379
result_url = redis://localhost:63379

View File

@ -1,3 +0,0 @@
[cic]
registry_address =
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

View File

@ -1,9 +0,0 @@
[database]
NAME=cic_cache
USER=grassroots
PASSWORD=
HOST=localhost
PORT=63432
ENGINE=postgresql
DRIVER=psycopg2
DEBUG=0

View File

@ -1,4 +0,0 @@
[syncer]
loop_interval = 1
offset = 0
no_history = 0

View File

@ -1,2 +0,0 @@
[bancor]
dir =

View File

@ -1,4 +0,0 @@
[cic]
registry_address =
chain_spec =
trust_address =

View File

@ -1,9 +0,0 @@
[database]
NAME=cic-cache-test
USER=postgres
PASSWORD=
HOST=localhost
PORT=5432
ENGINE=sqlite
DRIVER=pysqlite
DEBUG=1

View File

@ -1,5 +0,0 @@
[eth]
#ws_provider = ws://localhost:8546
#ttp_provider = http://localhost:8545
provider = http://localhost:8545
#chain_id =

View File

@ -1,2 +0,0 @@
[syncer]
loop_interval = 1

View File

@ -2,7 +2,7 @@ alembic==1.4.2
confini>=0.3.6rc4,<0.5.0 confini>=0.3.6rc4,<0.5.0
uwsgi==2.0.19.1 uwsgi==2.0.19.1
moolb~=0.1.1b2 moolb~=0.1.1b2
cic-eth-registry~=0.6.1a1 cic-eth-registry~=0.6.1a5
SQLAlchemy==1.3.20 SQLAlchemy==1.3.20
semver==2.13.0 semver==2.13.0
psycopg2==2.8.6 psycopg2==2.8.6

View File

@ -1,14 +1,19 @@
#!/usr/bin/python #!/usr/bin/python3
# standard imports
import os import os
import argparse import argparse
import logging import logging
import re import re
# external imports
import alembic import alembic
from alembic.config import Config as AlembicConfig from alembic.config import Config as AlembicConfig
import confini import confini
# local imports
from cic_cache.db import dsn_from_config from cic_cache.db import dsn_from_config
import cic_cache.cli
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
@ -18,30 +23,27 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
dbdir = os.path.join(rootdir, 'cic_cache', 'db') dbdir = os.path.join(rootdir, 'cic_cache', 'db')
migrationsdir = os.path.join(dbdir, 'migrations') migrationsdir = os.path.join(dbdir, 'migrations')
config_dir = os.path.join('/usr/local/etc/cic-cache') #config_dir = os.path.join('/usr/local/etc/cic-cache')
argparser = argparse.ArgumentParser() arg_flags = cic_cache.cli.argflag_std_base
argparser.add_argument('-c', type=str, default=config_dir, help='config file') local_arg_flags = cic_cache.cli.argflag_local_sync
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') argparser.process_local_flags(local_arg_flags)
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
argparser.add_argument('-f', action='store_true', help='force action') argparser.add_argument('-f', '--force', action='store_true', help='force action')
argparser.add_argument('-v', action='store_true', help='be verbose') argparser.add_argument('--migrations-dir', dest='migrations_dir', type=str, help='migrations directory')
argparser.add_argument('-vv', action='store_true', help='be more verbose')
args = argparser.parse_args() args = argparser.parse_args()
if args.vv: extra_args = {
logging.getLogger().setLevel(logging.DEBUG) 'reset': None,
elif args.v: 'force': None,
logging.getLogger().setLevel(logging.INFO) 'migrations_dir': None,
}
config = confini.Config(args.c, args.env_prefix) # process config
config.process() config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config:\n{}'.format(config))
migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE')) migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE'))
if not os.path.isdir(migrations_dir): if not os.path.isdir(migrations_dir):
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE'))) logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
migrations_dir = os.path.join(args.migrations_dir, 'default') migrations_dir = os.path.join(args.migrations_dir, 'default')

View File

@ -68,7 +68,11 @@ admin_tests = [
'local_key', 'local_key',
] ]
all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests cache_tests = [
'cache_tx_user',
]
all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
@ -77,11 +81,14 @@ argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', defa
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url') argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url')
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications') argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications')
argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications') argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification')
argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers')
argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
@ -111,6 +118,7 @@ config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL') config.censor('PASSWORD', 'SSL')
config.add(args.meta_provider, '_META_PROVIDER', True) config.add(args.meta_provider, '_META_PROVIDER', True)
config.add(args.ussd_provider, '_USSD_PROVIDER', True) config.add(args.ussd_provider, '_USSD_PROVIDER', True)
config.add(args.cache_provider, '_CACHE_PROVIDER', True)
token_symbol = args.token_symbol token_symbol = args.token_symbol
@ -349,6 +357,22 @@ class Verifier:
raise VerifierError(o_retrieved, 'metadata (person)') raise VerifierError(o_retrieved, 'metadata (person)')
def verify_cache_tx_user(self, address, balance=None):
url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address)
req = urllib.request.Request(url)
req.add_header('X_CIC_CACHE_MODE', 'all')
try:
res = urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
raise VerifierError(
'({}) {}'.format(url, e),
'cache',
)
r = json.load(res)
for tx in r['data']:
logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address))
def verify_metadata_phone(self, address, balance=None): def verify_metadata_phone(self, address, balance=None):
upper_address = strip_0x(address).upper() upper_address = strip_0x(address).upper()
f = open(os.path.join( f = open(os.path.join(
@ -395,11 +419,13 @@ class Verifier:
if m != 'CON Welcome': if m != 'CON Welcome':
raise VerifierError(response_data, 'ussd') raise VerifierError(response_data, 'ussd')
def verify_ussd_pins(self, address, balance): def verify_ussd_pins(self, address, balance):
response_data = send_ussd_request(address, self.data_dir) response_data = send_ussd_request(address, self.data_dir)
if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio': if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
raise VerifierError(response_data, 'pins') raise VerifierError(response_data, 'pins')
def verify(self, address, balance, debug_stem=None): def verify(self, address, balance, debug_stem=None):
for k in active_tests: for k in active_tests:

View File

@ -245,6 +245,8 @@ services:
DATABASE_PORT: ${DATABASE_PORT:-5432} DATABASE_PORT: ${DATABASE_PORT:-5432}
#DATABASE_PASSWORD: ${DATABASE_PASSWORD:- #DATABASE_PASSWORD: ${DATABASE_PASSWORD:-
DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: 1 DATABASE_DEBUG: 1
#PGPASSWORD: $DATABASE_PASSWORD #PGPASSWORD: $DATABASE_PASSWORD
SERVER_PORT: 8000 SERVER_PORT: 8000