Revert "Add gas cache backend, test, filter"

This reverts commit fb953d0318.

.gitlab-ci.yml

@@ -10,7 +10,6 @@ include:
     #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
-  - version
   - build
   - test
   - deploy
@@ -21,39 +20,9 @@ variables:
   DOCKER_BUILDKIT: "1"
   COMPOSE_DOCKER_CLI_BUILD: "1"
   CI_DEBUG_TRACE: "true"
-  SEMVERBOT_VERSION: "0.2.0"
 
-  #before_script:
-  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-
-version:
-  #image: python:3.7-stretch
-  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
-  stage: version
-  script:
-    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
-    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
-    - eval $(ssh-agent -s)
-    - ssh-add <(echo "$SSH_PRIVATE_KEY")
-    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
-    - export TAG=$(sbot predict version -m auto)
-    - |
-      if [[ -z $TAG ]]
-      then
-        echo "tag could not be set $@"
-        exit 1
-      fi
-    - echo $TAG > version
-    - git tag -a v$TAG -m "ci tagged"
-    - git push origin v$TAG
-  artifacts:
-    paths:
-      - version
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: always
+before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
 
 # runs on protected branches and pushes to repo
 build-push:
@@ -61,17 +30,12 @@ build-push:
   tags:
     - integration
   #script:
-  #  - TAG=$CI_Cbefore_script:
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
   script:
-    - TAG=latest ./scripts/build-push.sh
-    - TAG=$(cat ./version) ./scripts/build-push.sh
+    - TAG=latest sh ./scripts/build-push.sh
   rules:
   - if: $CI_COMMIT_REF_PROTECTED == "true"
     when: always
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: always
 
 deploy-dev:
   stage: deploy

.semverbot.toml

@@ -1,16 +0,0 @@
-[git]
-
-[git.config]
-email = "semverbot@grassroots.org"
-name = "semvervot"
-
-[git.tags]
-prefix = "v"
-
-[semver]
-mode = "git-commit"
-
-[semver.detection]
-patch = ["fix", "[fix]", "patch", "[patch]"]
-minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
-major = ["BREAKING CHANGE"]

README.md

@@ -4,7 +4,6 @@
 
 This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
-
 ```
 export COMPOSE_DOCKER_CLI_BUILD=1
 export DOCKER_BUILDKIT=1

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -1,4 +1,4 @@
 [cic]
 registry_address = 
 trust_address =
-health_modules =
+health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -5,10 +5,7 @@ import re
 import base64
 
 # external imports
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
+from hexathon import add_0x
 
 # local imports
 from cic_cache.cache import (
@@ -22,7 +19,6 @@ logg = logging.getLogger(__name__)
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
 re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
 re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
-re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
 
 DEFAULT_LIMIT = 100
 
@@ -32,7 +28,9 @@ def process_transactions_account_bloom(session, env):
     if not r:
         return None
 
-    address = strip_0x(r[1])
+    address = r[1]
+    if r[2] == None:
+        address = add_0x(address)
     offset = 0
     if r.lastindex > 2:
         offset = r[4]
@@ -115,38 +113,3 @@ def process_transactions_all_data(session, env):
     j = json.dumps(o)
 
     return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_account_data(session, env):
-    r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
-    if not r:
-        return None
-    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-        return None
-
-    logg.debug('got data request {}'.format(env))
-    address = strip_0x(r[1])
-    #if r[2] == None:
-    #    address = add_0x(address)
-    offset = 0
-    if r.lastindex > 2:
-        offset = r[4]
-    limit = DEFAULT_LIMIT
-    if r.lastindex > 4:
-        limit = r[6]
-
-    c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
-
-    for r in tx_cache:
-        r['date_block'] = r['date_block'].timestamp()
-
-    o = {
-        'low': lowest_block,
-        'high': highest_block,
-        'data': tx_cache,
-    }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -13,18 +13,20 @@ from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.runnable.daemons.query import (
         process_transactions_account_bloom,
-        process_transactions_account_data,
         process_transactions_all_bloom,
         process_transactions_all_data,
         )
-import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+dbdir = os.path.join(rootdir, 'cic_cache', 'db')
+migrationsdir = os.path.join(dbdir, 'migrations')
 
-arg_flags = cic_cache.cli.argflag_std_read
-local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task 
+# process args
+arg_flags = cic_cache.cli.argflag_std_base
+local_arg_flags = cic_cache.cli.argflag_local_task
 argparser = cic_cache.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
@@ -48,7 +50,6 @@ def application(env, start_response):
             process_transactions_all_data,
             process_transactions_all_bloom,
             process_transactions_account_bloom,
-            process_transactions_account_data,
             ]:
         r = None
         try:

apps/cic-cache/config/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis:///
+result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -0,0 +1,3 @@
+[cic]
+registry_address = 
+trust_address = 

apps/cic-cache/config/database.ini

@@ -0,0 +1,9 @@
+[database]
+NAME=cic_cache
+USER=postgres
+PASSWORD=
+HOST=localhost
+PORT=5432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=0

apps/cic-cache/config/docker/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis://localhost:63379
+result_url = redis://localhost:63379

apps/cic-cache/config/docker/cic.ini

@@ -0,0 +1,3 @@
+[cic]
+registry_address =  
+trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -0,0 +1,9 @@
+[database]
+NAME=cic_cache
+USER=grassroots
+PASSWORD=
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=0

apps/cic-cache/config/docker/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+offset = 0
+no_history = 0

apps/cic-cache/docker/Dockerfile

@@ -18,6 +18,11 @@ COPY . .
 
 RUN python setup.py install
 
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+#COPY config/ /usr/local/etc/cic-cache/
+
 # for db migrations
 COPY ./aux/wait-for-it/wait-for-it.sh ./
 COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/

apps/cic-cache/requirements.txt

@@ -2,7 +2,7 @@ alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
 moolb~=0.1.1b2
-cic-eth-registry~=0.6.1a5
+cic-eth-registry~=0.6.1a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
@@ -12,4 +12,3 @@ chainsyncer[sql]>=0.0.6a3,<0.1.0
 erc20-faucet>=0.3.2a2, <0.4.0
 chainlib-eth>=0.0.9a14,<0.1.0
 eth-address-index>=0.2.3a4,<0.3.0
-okota>=0.2.4a6,<0.3.0

apps/cic-cache/scripts/migrate.py

@@ -1,19 +1,14 @@
-#!/usr/bin/python3
-
-# standard imports
+#!/usr/bin/python
 import os
 import argparse
 import logging
 import re
 
-# external imports
 import alembic
 from alembic.config import Config as AlembicConfig
 import confini
 
-# local imports
 from cic_cache.db import dsn_from_config
-import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -26,25 +21,28 @@ configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
 
 #config_dir = os.path.join('/usr/local/etc/cic-cache')
 
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_sync
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', '--force', action='store_true', help='force action')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', type=str, help='migrations directory')
+argparser.add_argument('-f', action='store_true', help='force action')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-extra_args = {
-    'reset': None,
-    'force': None,
-    'migrations_dir': None,
-        }
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
 
-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
+config = confini.Config(configdir, args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
 
-migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE'))
+migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
 if not os.path.isdir(migrations_dir):
     logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
     migrations_dir = os.path.join(args.migrations_dir, 'default')

apps/cic-meta/scripts/server/server.ts

@@ -193,7 +193,6 @@ async function processRequest(req, res) {
 						res.end();
 						return;
 					}
-					content = '';
 					break;
 
 				case 'get:automerge:client':
@@ -252,7 +251,7 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty content', mod, digest, data);
+			console.error('empty content', data);
 			res.writeHead(404, {"Content-Type": "text/plain"});
 			res.end();
 			return;

apps/data-seeding/verify.py

@@ -68,11 +68,7 @@ admin_tests = [
         'local_key',
         ]
 
-cache_tests = [
-        'cache_tx_user',
-        ]
-
-all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests
+all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
@@ -81,14 +77,11 @@ argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', defa
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
 argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
-argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url')
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications')
 argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
-argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
-argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers')
 argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
@@ -122,7 +115,6 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 config.add(args.meta_provider, '_META_PROVIDER', True)
 config.add(args.ussd_provider, '_USSD_PROVIDER', True)
-config.add(args.cache_provider, '_CACHE_PROVIDER', True)
 
 token_symbol = args.token_symbol
 
@@ -359,24 +351,6 @@ class Verifier:
             raise VerifierError(o_retrieved, 'metadata (person)')
 
 
-    def verify_cache_tx_user(self, address, balance=None):
-        url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address)
-        req = urllib.request.Request(url)
-        req.add_header('X_CIC_CACHE_MODE', 'all')
-        try:
-            res = urllib.request.urlopen(req)
-        except urllib.error.HTTPError as e:
-            raise VerifierError(
-                    '({}) {}'.format(url, e),
-                    'cache (tx user)',
-                    )
-        r = json.load(res)
-        if len(r['data']) == 0:
-            raise VerifierError('empty tx list for address {}'.format(address), 'cache (tx user)')
-        for tx in r['data']:
-            logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address))
-
-
     def verify_metadata_phone(self, address, balance=None):
         upper_address = strip_0x(address).upper()
         f = open(os.path.join(
@@ -423,13 +397,11 @@ class Verifier:
         if m != 'CON Welcome':
             raise VerifierError(response_data, 'ussd')
 
-
     def verify_ussd_pins(self, address, balance):
         response_data = send_ussd_request(address, self.data_dir)
         if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
             raise VerifierError(response_data, 'pins')
 
-
     def verify(self, address, balance, debug_stem=None):
   
         for k in active_tests:

docker-compose.yml

@@ -159,6 +159,14 @@ services:
     volumes:
       - signer-data:/run/crypto-dev-signer
       - contract-config:/tmp/cic/config/:ro
+    command:
+      - /bin/bash
+      - -c
+      - |
+        set -a
+        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
+        set +a
+        ./start_tasker.sh --aux-all -q cic-eth  -vv
 
 
   cic-eth-tracker:

scripts/build.sh

@@ -6,7 +6,4 @@ set -e
 TAG=${TAG?Variable not set} \
 docker-compose \
 -f docker-compose.yml \
-build \
---no-cache \
---parallel \
---progress plain
+build