diff --git a/apps/cic-cache/cic_cache/data/config/cic.ini b/apps/cic-cache/cic_cache/data/config/cic.ini
index b4275352..d2382746 100644
--- a/apps/cic-cache/cic_cache/data/config/cic.ini
+++ b/apps/cic-cache/cic_cache/data/config/cic.ini
@@ -1,4 +1,4 @@
[cic]
registry_address =
trust_address =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
+health_modules =
diff --git a/apps/cic-cache/cic_cache/runnable/daemons/query.py b/apps/cic-cache/cic_cache/runnable/daemons/query.py
index d7edf6ab..bd67d962 100644
--- a/apps/cic-cache/cic_cache/runnable/daemons/query.py
+++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py
@@ -5,7 +5,10 @@ import re
import base64
# external imports
-from hexathon import add_0x
+from hexathon import (
+ add_0x,
+ strip_0x,
+ )
# local imports
from cic_cache.cache import (
@@ -19,6 +22,7 @@ logg = logging.getLogger(__name__)
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
+re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
DEFAULT_LIMIT = 100
@@ -28,9 +32,7 @@ def process_transactions_account_bloom(session, env):
if not r:
return None
- address = r[1]
- if r[2] == None:
- address = add_0x(address)
+ address = strip_0x(r[1])
offset = 0
if r.lastindex > 2:
offset = r[4]
@@ -113,3 +115,38 @@ def process_transactions_all_data(session, env):
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_account_data(session, env):
+ r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
+ if not r:
+ return None
+ if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+ return None
+
+ logg.debug('got data request {}'.format(env))
+ address = strip_0x(r[1])
+ #if r[2] == None:
+ # address = add_0x(address)
+ offset = 0
+ if r.lastindex > 2:
+ offset = r[4]
+ limit = DEFAULT_LIMIT
+ if r.lastindex > 4:
+ limit = r[6]
+
+ c = DataCache(session)
+ (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
+
+ for r in tx_cache:
+ r['date_block'] = r['date_block'].timestamp()
+
+ o = {
+ 'low': lowest_block,
+ 'high': highest_block,
+ 'data': tx_cache,
+ }
+
+ j = json.dumps(o)
+
+ return ('application/json', j.encode('utf-8'),)
diff --git a/apps/cic-cache/cic_cache/runnable/daemons/server.py b/apps/cic-cache/cic_cache/runnable/daemons/server.py
index 58542d03..a83cbbc0 100644
--- a/apps/cic-cache/cic_cache/runnable/daemons/server.py
+++ b/apps/cic-cache/cic_cache/runnable/daemons/server.py
@@ -13,20 +13,18 @@ from cic_cache.db import dsn_from_config
from cic_cache.db.models.base import SessionBase
from cic_cache.runnable.daemons.query import (
process_transactions_account_bloom,
+ process_transactions_account_data,
process_transactions_all_bloom,
process_transactions_all_data,
)
+import cic_cache.cli
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-# process args
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_task
+arg_flags = cic_cache.cli.argflag_std_read
+local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task
argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
@@ -50,6 +48,7 @@ def application(env, start_response):
process_transactions_all_data,
process_transactions_all_bloom,
process_transactions_account_bloom,
+ process_transactions_account_data,
]:
r = None
try:
diff --git a/apps/cic-cache/config/celery.ini b/apps/cic-cache/config/celery.ini
deleted file mode 100644
index 6136b166..00000000
--- a/apps/cic-cache/config/celery.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis:///
-result_url = redis:///
diff --git a/apps/cic-cache/config/cic.ini b/apps/cic-cache/config/cic.ini
deleted file mode 100644
index 48c32fad..00000000
--- a/apps/cic-cache/config/cic.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[cic]
-registry_address =
-trust_address =
diff --git a/apps/cic-cache/config/config.ini b/apps/cic-cache/config/config.ini
new file mode 100644
index 00000000..e69de29b
diff --git a/apps/cic-cache/config/database.ini b/apps/cic-cache/config/database.ini
deleted file mode 100644
index 7c5836a2..00000000
--- a/apps/cic-cache/config/database.ini
+++ /dev/null
@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0
diff --git a/apps/cic-cache/config/docker/celery.ini b/apps/cic-cache/config/docker/celery.ini
deleted file mode 100644
index 98c5012f..00000000
--- a/apps/cic-cache/config/docker/celery.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis://localhost:63379
-result_url = redis://localhost:63379
diff --git a/apps/cic-cache/config/docker/cic.ini b/apps/cic-cache/config/docker/cic.ini
deleted file mode 100644
index af112724..00000000
--- a/apps/cic-cache/config/docker/cic.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[cic]
-registry_address =
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
diff --git a/apps/cic-cache/config/docker/database.ini b/apps/cic-cache/config/docker/database.ini
deleted file mode 100644
index db1f5086..00000000
--- a/apps/cic-cache/config/docker/database.ini
+++ /dev/null
@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=grassroots
-PASSWORD=
-HOST=localhost
-PORT=63432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0
diff --git a/apps/cic-cache/config/docker/syncer.ini b/apps/cic-cache/config/docker/syncer.ini
deleted file mode 100644
index 255dc00f..00000000
--- a/apps/cic-cache/config/docker/syncer.ini
+++ /dev/null
@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0
diff --git a/apps/cic-cache/docker/Dockerfile b/apps/cic-cache/docker/Dockerfile
index dc4bf6db..6a49edea 100644
--- a/apps/cic-cache/docker/Dockerfile
+++ b/apps/cic-cache/docker/Dockerfile
@@ -18,11 +18,6 @@ COPY . .
RUN python setup.py install
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z
(executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-cache/
-
# for db migrations
COPY ./aux/wait-for-it/wait-for-it.sh ./
COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt
index 8f47f9b5..4cabe432 100644
--- a/apps/cic-cache/requirements.txt
+++ b/apps/cic-cache/requirements.txt
@@ -2,7 +2,7 @@ alembic==1.4.2
confini>=0.3.6rc4,<0.5.0
uwsgi==2.0.19.1
moolb~=0.1.1b2
-cic-eth-registry~=0.6.1a1
+cic-eth-registry~=0.6.1a5
SQLAlchemy==1.3.20
semver==2.13.0
psycopg2==2.8.6
@@ -12,3 +12,4 @@ chainsyncer[sql]>=0.0.6a3,<0.1.0
erc20-faucet>=0.3.2a2, <0.4.0
chainlib-eth>=0.0.9a14,<0.1.0
eth-address-index>=0.2.3a4,<0.3.0
+okota>=0.2.4a6,<0.3.0
diff --git a/apps/cic-cache/scripts/migrate.py b/apps/cic-cache/scripts/migrate.py
index 458b1ff2..581fd432 100644
--- a/apps/cic-cache/scripts/migrate.py
+++ b/apps/cic-cache/scripts/migrate.py
@@ -1,14 +1,19 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+
+# standard imports
import os
import argparse
import logging
import re
+# external imports
import alembic
from alembic.config import Config as AlembicConfig
import confini
+# local imports
from cic_cache.db import dsn_from_config
+import cic_cache.cli
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
@@ -21,28 +26,25 @@ configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
#config_dir = os.path.join('/usr/local/etc/cic-cache')
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_sync
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
+argparser.add_argument('-f', '--force', action='store_true', help='force action')
+argparser.add_argument('--migrations-dir', dest='migrations_dir', type=str, help='migrations directory')
args = argparser.parse_args()
-if args.vv:
- logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
- logging.getLogger().setLevel(logging.INFO)
+extra_args = {
+ 'reset': None,
+ 'force': None,
+ 'migrations_dir': None,
+ }
-config = confini.Config(configdir, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
-migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
+migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE'))
if not os.path.isdir(migrations_dir):
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
migrations_dir = os.path.join(args.migrations_dir, 'default')
diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py
index b01a5e8a..ca4a14b7 100644
--- a/apps/data-seeding/verify.py
+++ b/apps/data-seeding/verify.py
@@ -68,7 +68,11 @@ admin_tests = [
'local_key',
]
-all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
+cache_tests = [
+ 'cache_tx_user',
+ ]
+
+all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
@@ -77,11 +81,14 @@ argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', defa
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
+argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url')
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications')
argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
+argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification')
+argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers')
argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
@@ -115,6 +122,7 @@ config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
config.add(args.meta_provider, '_META_PROVIDER', True)
config.add(args.ussd_provider, '_USSD_PROVIDER', True)
+config.add(args.cache_provider, '_CACHE_PROVIDER', True)
token_symbol = args.token_symbol
@@ -351,6 +359,24 @@ class Verifier:
raise VerifierError(o_retrieved, 'metadata (person)')
+ def verify_cache_tx_user(self, address, balance=None):
+ url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address)
+ req = urllib.request.Request(url)
+ req.add_header('X_CIC_CACHE_MODE', 'all')
+ try:
+ res = urllib.request.urlopen(req)
+ except urllib.error.HTTPError as e:
+ raise VerifierError(
+ '({}) {}'.format(url, e),
+ 'cache (tx user)',
+ )
+ r = json.load(res)
+ if len(r['data']) == 0:
+ raise VerifierError('empty tx list for address {}'.format(address), 'cache (tx user)')
+ for tx in r['data']:
+ logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address))
+
+
def verify_metadata_phone(self, address, balance=None):
upper_address = strip_0x(address).upper()
f = open(os.path.join(
@@ -397,11 +423,13 @@ class Verifier:
if m != 'CON Welcome':
raise VerifierError(response_data, 'ussd')
+
def verify_ussd_pins(self, address, balance):
response_data = send_ussd_request(address, self.data_dir)
if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
raise VerifierError(response_data, 'pins')
+
def verify(self, address, balance, debug_stem=None):
for k in active_tests:
diff --git a/docker-compose.yml b/docker-compose.yml
index 85c808d1..87b741bc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -160,14 +160,6 @@ services:
volumes:
- signer-data:/run/crypto-dev-signer
- contract-config:/tmp/cic/config/:ro
- command:
- - /bin/bash
- - -c
- - |
- set -a
- if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
- set +a
- ./start_tasker.sh --aux-all -q cic-eth -vv
cic-eth-tracker: