Fix confini instantiation in ussd

Fix dispatcher memory leak

See merge request grassrootseconomics/cic-internal-integration!220

.gitignore

@@ -13,3 +13,4 @@ build/
 **/coverage
 **/.venv
 .idea
+**/.vim

apps/cic-cache/.coveragerc

@@ -2,4 +2,5 @@
 omit = 
 	.venv/*
 	scripts/*
-	cic_cache/db/postgres/*
+	cic_cache/db/migrations/*
+	cic_cache/version.py

apps/cic-cache/.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

apps/cic-cache/.gitlab-ci.yml

@@ -1,22 +1,52 @@
 .cic_cache_variables:
     variables:
         APP_NAME: cic-cache
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_cache_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-cache:
     extends:
-        - .cic_cache_changes_target
         - .py_build_merge_request
         - .cic_cache_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
+
+test-mr-cic-cache:
+    stage: test
+    extends:
+        - .cic_cache_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-cache"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
 
 build-push-cic-cache:
     extends:
         - .py_build_push
         - .cic_cache_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
         
 

apps/cic-cache/cic_cache/cache.py

@@ -15,6 +15,8 @@ from cic_cache.db.list import (
 logg = logging.getLogger()
 
 
+DEFAULT_FILTER_SIZE = 8192 * 8
+
 class Cache:
 
     def __init__(self, session):
@@ -25,7 +27,7 @@ class BloomCache(Cache):
 
     @staticmethod
     def __get_filter_size(n):
-        n = 8192 * 8
+        n = DEFAULT_FILTER_SIZE
         logg.warning('filter size hardcoded to {}'.format(n))
         return n
 

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,3 +100,4 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -4,6 +4,9 @@ import json
 import re
 import base64
 
+# external imports
+from hexathon import add_0x
+
 # local imports
 from cic_cache.cache import (
         BloomCache,
@@ -11,10 +14,11 @@ from cic_cache.cache import (
     )
 
 logg = logging.getLogger(__name__)
+#logg = logging.getLogger()
 
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
+re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
 
 DEFAULT_LIMIT = 100
 
@@ -26,13 +30,13 @@ def process_transactions_account_bloom(session, env):
 
     address = r[1]
     if r[2] == None:
-        address = '0x' + address
-    offset = DEFAULT_LIMIT
+        address = add_0x(address)
+    offset = 0
     if r.lastindex > 2:
-        offset = r[3]
-    limit = 0
-    if r.lastindex > 3:
-        limit = r[4]
+        offset = r[4]
+    limit = DEFAULT_LIMIT
+    if r.lastindex > 4:
+        limit = r[6]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)

apps/cic-cache/docker/Dockerfile

@@ -1,52 +1,38 @@
-FROM python:3.8.6-slim-buster
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
 
-WORKDIR /usr/src/cic-cache
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
+COPY . . 
 
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-# Copy shared requirements from top of mono-repo
-RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY cic-cache/requirements.txt ./
-COPY cic-cache/setup.cfg \
-	 cic-cache/setup.py \
- 	 ./
-COPY cic-cache/cic_cache/ ./cic_cache/
-COPY cic-cache/scripts/ ./scripts/
-COPY cic-cache/test_requirements.txt ./
-RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-RUN pip install $pip_extra_index_url_flag .
-RUN pip install .[server]
-
-COPY cic-cache/tests/ ./tests/
-#COPY db/ cic-cache/db
-#RUN apk add postgresql-client
+RUN python setup.py install
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-cache/config/ /usr/local/etc/cic-cache/
+COPY config/ /usr/local/etc/cic-cache/
 
 # for db migrations
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
-COPY cic-cache/docker/db.sh ./db.sh
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server
 # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []

apps/cic-cache/docker/Dockerfile_ci

@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . . 
+
+RUN python setup.py install
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-cache/
+
+# for db migrations
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
+RUN chmod 755 ./*.sh
+# Tracker
+# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
+# Server
+# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []

apps/cic-cache/requirements.txt

@@ -1,13 +1,13 @@
-cic-base==0.1.3a3+build.984b5cff
+cic-base~=0.2.0a4
 alembic==1.4.2
-confini~=0.3.6rc3
+confini>=0.3.6rc3,<0.5.0
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.6a1
+cic-eth-registry~=0.5.6a2
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a1
+chainsyncer[sql]~=0.0.3a5
+erc20-faucet~=0.2.2a2

apps/cic-cache/tests/conftest.py

@@ -5,9 +5,12 @@ import datetime
 
 # external imports
 import pytest
+import moolb
 
 # local imports
 from cic_cache import db
+from cic_cache import BloomCache
+from cic_cache.cache import DEFAULT_FILTER_SIZE
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -101,3 +104,7 @@ def tag_txs(
 
     db.tag_transaction(init_database, txs[1], 'taag', domain='test')
 
+
+@pytest.fixture(scope='session')
+def zero_filter():
+    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/test_erc20.py

@@ -10,6 +10,7 @@ from sqlalchemy import text
 from chainlib.eth.tx import Tx
 from chainlib.eth.block import Block
 from chainlib.chain import ChainSpec
+from chainlib.eth.error import RequestMismatchException
 from hexathon import (
         strip_0x,
         add_0x,
@@ -18,10 +19,21 @@ from hexathon import (
 # local imports
 from cic_cache.db import add_tag
 from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
+from cic_cache.runnable.daemons.filters.base import TagSyncFilter
 
 logg = logging.getLogger()
 
 
+def test_base_filter_str(
+        init_database,
+        ):
+        f = TagSyncFilter('foo')
+        assert 'foo' == str(f)
+        f = TagSyncFilter('foo', domain='bar')
+        assert 'bar.foo' == str(f)
+
+
+
 def test_erc20_filter(
         eth_rpc,
         foo_token,
@@ -67,3 +79,95 @@ def test_erc20_filter(
     s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
     r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
     assert r[0] == tx.hash
+
+
+def test_erc20_filter_nocontract(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    # incomplete args
+    data = 'a9059cbb'
+    data += strip_0x(list_actors['alice'])
+    data += '1000'.ljust(64, '0')
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': os.urandom(20).hex(),
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+
+@pytest.mark.parametrize(
+        'contract_method,contract_input,expected_exception',
+        [
+            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
+            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
+            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
+            ],
+        )
+def test_erc20_filter_bogus(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        contract_method,
+        contract_input,
+        expected_exception,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    # incomplete args
+    data = contract_method
+    data += contract_input 
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': foo_token,
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/test_query.py

@@ -0,0 +1,230 @@
+# standard imports
+import logging
+import json
+import base64
+import copy
+import re
+
+# external imports
+import pytest
+from hexathon import strip_0x
+
+# local imports
+from cic_cache.runnable.daemons.query import *
+
+logg = logging.getLogger()
+
+
+
+@pytest.mark.parametrize(
+        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
+        [
+            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
+            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
+            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
+            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
+            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
+            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
+            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
+            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
+            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
+            ],
+        )
+def test_query_regex(
+        list_actors,
+        query_path_prefix,
+        query_role,
+        query_address_index,
+        query_offset,
+        query_offset_index,
+        query_limit,
+        query_limit_index,
+        match_re,
+        ):
+
+        paths = []
+        path = query_path_prefix
+        query_address = None
+        if query_role != None:
+            query_address = strip_0x(list_actors[query_role])
+            paths.append(path + '0x' + query_address)
+            paths.append(path + query_address)
+        if query_offset != None:
+            if query_limit != None:
+                for i in range(len(paths)-1):
+                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
+            else:
+                for i in range(len(paths)-1):
+                    paths[i] += '/' + str(query_offset)
+
+        for i in range(len(paths)):
+            paths.append(paths[i] + '/')
+
+        for p in paths:
+            logg.debug('testing path {} against {}'.format(p, match_re))
+            m = re.match(match_re, p)
+            l = len(m.groups())
+            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
+            for i in range(l+1):
+                logg.debug('group {} {}'.format(i, m[i]))
+            if m.lastindex >= query_offset_index:
+                assert query_offset == int(m[query_offset_index + 1])
+            if m.lastindex >= query_limit_index:
+                assert query_limit == int(m[query_limit_index + 1])
+            if query_address_index != None:
+                match_address = strip_0x(m[query_address_index + 1])
+                assert query_address == match_address
+
+
+
+@pytest.mark.parametrize(
+        'role_name, query_offset, query_limit, query_match',
+        [ 
+            ('alice', None, None, [(420000, 13), (419999, 42)]),
+            ('alice', None, 1, [(420000, 13)]),
+            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
+            ('alice', 2, None, []), # 420000 == list_defaults['block']
+            ],
+        )
+def test_query_process_txs_account(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        role_name,
+        query_offset,
+        query_limit,
+        query_match,
+        ):
+
+    actor = None
+    try:
+        actor = list_actors[role_name]
+    except KeyError:
+        actor = os.urandom(20).hex()
+    path_info = '/tx/user/0x' + strip_0x(actor)
+    if query_offset != None:
+        path_info += '/' + str(query_offset)
+    if query_limit != None:
+        if query_offset == None:
+            path_info += '/0'
+        path_info += '/' + str(query_limit)
+    env = {
+            'PATH_INFO': path_info,
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_account_bloom(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
+    zero_filter_data = zero_filter.to_bytes()
+    if len(query_match) == 0:
+        assert block_filter_data == zero_filter_data
+        return
+
+    assert block_filter_data != zero_filter_data
+    block_filter = copy.copy(zero_filter)
+    block_filter.merge(block_filter_data)
+    block_filter_data = block_filter.to_bytes()
+    assert block_filter_data != zero_filter_data
+
+    for (block, tx) in query_match:
+        block = block.to_bytes(4, byteorder='big')
+        assert block_filter.check(block)
+
+
+@pytest.mark.parametrize(
+        'query_offset, query_limit, query_match',
+        [ 
+            (None, 2, [(420000, 13), (419999, 42)]),
+            (0, 1, [(420000, 13)]),
+            (1, 1, [(419999, 42)]),
+            (2, 0, []),
+            ],
+        )
+def test_query_process_txs_bloom(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        query_offset,
+        query_limit,
+        query_match,
+        ):
+
+    path_info = '/tx'
+    if query_offset != None:
+        path_info += '/' + str(query_offset)
+    if query_limit != None:
+        if query_offset == None:
+            path_info += '/0'
+        path_info += '/' + str(query_limit)
+    env = {
+            'PATH_INFO': path_info,
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_all_bloom(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
+    zero_filter_data = zero_filter.to_bytes()
+    if len(query_match) == 0:
+        assert block_filter_data == zero_filter_data
+        return
+
+    assert block_filter_data != zero_filter_data
+    block_filter = copy.copy(zero_filter)
+    block_filter.merge(block_filter_data)
+    block_filter_data = block_filter.to_bytes()
+    assert block_filter_data != zero_filter_data
+
+    for (block, tx) in query_match:
+        block = block.to_bytes(4, byteorder='big')
+        assert block_filter.check(block)
+
+
+@pytest.mark.parametrize(
+        'query_block_start, query_block_end, query_match_count',
+        [ 
+            (None, 42, 0),
+            (420000, 420001, 1),
+            (419999, 419999, 1), # matches are inclusive
+            (419999, 420000, 2),
+            (419999, 420001, 2),
+            ],
+        )
+def test_query_process_txs_data(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        query_block_start,
+        query_block_end,
+        query_match_count,
+        ):
+
+    path_info = '/txa'
+    if query_block_start != None:
+        path_info += '/' + str(query_block_start)
+    if query_block_end != None:
+        if query_block_start == None:
+            path_info += '/0'
+        path_info += '/' + str(query_block_end)
+    env = {
+            'PATH_INFO': path_info,
+            'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_all_data(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    assert len(o['data']) == query_match_count

apps/cic-eth-aux/erc20-demurrage-token/MANIFEST.in

@@ -0,0 +1 @@
+include *requirements.txt

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py

@@ -0,0 +1,53 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from erc20_demurrage_token.demurrage import DemurrageCalculator
+from chainlib.connection import RPCConnection
+from chainlib.chain import ChainSpec
+from chainlib.eth.constant import ZERO_ADDRESS
+from cic_eth_registry import CICRegistry
+
+logg = logging.getLogger(__name__)
+
+celery_app = celery.current_app
+
+
+class NoopCalculator:
+
+    def amount_since(self, amount, timestamp):
+        logg.debug('noopcalculator amount {} timestamp {}'.format(amount, timestamp))
+        return amount
+
+
+class DemurrageCalculationTask(celery.Task):
+
+    demurrage_token_calcs = {}
+
+    @classmethod
+    def register_token(cls, rpc, chain_spec, token_symbol, sender_address=ZERO_ADDRESS):
+        registry = CICRegistry(chain_spec, rpc)
+        token_address = registry.by_name(token_symbol, sender_address=sender_address)
+        try:
+            c = DemurrageCalculator.from_contract(rpc, chain_spec, token_address, sender_address=sender_address)
+            logg.info('found demurrage calculator for ERC20 {} @ {}'.format(token_symbol, token_address))
+        except:
+            logg.warning('Token {} at address {} does not appear to be a demurrage contract. Calls to balance adjust for this token will always return the same amount'.format(token_symbol, token_address))
+            c = NoopCalculator()
+
+        cls.demurrage_token_calcs[token_symbol] = c
+
+
+@celery_app.task(bind=True, base=DemurrageCalculationTask)
+def get_adjusted_balance(self, token_symbol, amount, timestamp):
+    c = self.demurrage_token_calcs[token_symbol]
+    return c.amount_since(amount, timestamp)
+
+
+def aux_setup(rpc, config, sender_address=ZERO_ADDRESS):
+    chain_spec_str = config.get('CIC_CHAIN_SPEC')
+    chain_spec = ChainSpec.from_chain_str(chain_spec_str)
+    token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
+    
+    DemurrageCalculationTask.register_token(rpc, chain_spec, token_symbol, sender_address=sender_address)

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/api.py

@@ -0,0 +1,30 @@
+# standard imports
+import logging
+
+# external imports 
+import celery
+from cic_eth.api.base import ApiBase
+
+app = celery.current_app
+
+logg = logging.getLogger(__name__)
+
+
+class Api(ApiBase):
+   
+    def get_adjusted_balance(self, token_symbol, balance, timestamp):
+        s = celery.signature(
+                'cic_eth_aux.erc20_demurrage_token.get_adjusted_balance',
+                [
+                    token_symbol,
+                    balance,
+                    timestamp,
+                    ],
+                queue=None,
+                )
+        if self.callback_param != None:
+            s.link(self.callback_success)
+            s.link.on_error(self.callback_error)
+
+        t = s.apply_async(queue=self.queue)
+        return t

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -0,0 +1,5 @@
+celery==4.4.7
+erc20-demurrage-token~=0.0.2a3
+cic-eth-registry~=0.5.6a1
+chainlib~=0.0.5a1
+cic_eth~=0.12.0a2

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -0,0 +1,30 @@
+[metadata]
+name = cic-eth-aux-erc20-demurrage-token
+version = 0.0.2a4
+description = cic-eth tasks supporting erc20 demurrage token
+author = Louis Holbrook
+author_email = dev@holbrook.no
+url = https://gitlab.com/ccicnet/erc20-demurrage-token
+keywords =
+	ethereum
+	blockchain
+	cryptocurrency
+	erc20
+classifiers =
+	Programming Language :: Python :: 3
+	Operating System :: OS Independent
+	Development Status :: 3 - Alpha
+	Environment :: No Input/Output (Daemon)
+	Intended Audience :: Developers
+	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+	Topic :: Internet
+	#Topic :: Blockchain :: EVM
+license = GPL3
+licence_files =
+	LICENSE
+
+[options]
+include_package_data = True
+python_requires = >= 3.6
+packages =
+	cic_eth_aux.erc20_demurrage_token

apps/cic-eth-aux/erc20-demurrage-token/setup.py

@@ -0,0 +1,25 @@
+from setuptools import setup
+
+requirements = []
+f = open('requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    requirements.append(l.rstrip())
+f.close()
+
+test_requirements = []
+f = open('test_requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    test_requirements.append(l.rstrip())
+f.close()
+
+
+setup(
+        install_requires=requirements,
+        tests_require=test_requirements,
+        )

apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt

@@ -0,0 +1,12 @@
+pytest==6.0.1
+pytest-celery==0.0.0a1
+pytest-mock==3.3.1
+pytest-cov==2.10.1
+eth-tester==0.5.0b3
+py-evm==0.3.0a20
+SQLAlchemy==1.3.20
+cic-eth~=0.12.0a1
+liveness~=0.0.1a7
+eth-accounts-index==0.0.12a1
+eth-contract-registry==0.5.6a1
+eth-address-index==0.1.2a1

apps/cic-eth-aux/erc20-demurrage-token/tests/conftest.py

@@ -0,0 +1,88 @@
+# external imports
+import celery
+from chainlib.eth.pytest.fixtures_chain import *
+from chainlib.eth.pytest.fixtures_ethtester import *
+from cic_eth_registry.pytest.fixtures_contracts import *
+from cic_eth_registry.pytest.fixtures_tokens import *
+from erc20_demurrage_token.unittest.base import TestTokenDeploy
+from erc20_demurrage_token.token import DemurrageToken
+from eth_token_index.index import TokenUniqueSymbolIndex
+from eth_address_declarator.declarator import AddressDeclarator
+
+# cic-eth imports
+from cic_eth.pytest.fixtures_celery import *
+from cic_eth.pytest.fixtures_token import *
+from cic_eth.pytest.fixtures_config import *
+
+
+@pytest.fixture(scope='function')
+def demurrage_token(
+        default_chain_spec,
+        eth_rpc,
+        token_registry,
+        contract_roles,
+        eth_signer,
+        ):
+    d = TestTokenDeploy(eth_rpc, token_symbol='BAR', token_name='Bar Token')
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
+    c = DemurrageToken(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    token_address =  d.deploy(eth_rpc, contract_roles['CONTRACT_DEPLOYER'], c, 'SingleNocap')
+    logg.debug('demurrage token contract "BAR" deployed to {}'.format(token_address))
+
+    return token_address
+    
+
+@pytest.fixture(scope='function')
+def demurrage_token_symbol(
+        default_chain_spec,
+        eth_rpc,
+        demurrage_token,
+        contract_roles,
+        ):
+
+    c = DemurrageToken(default_chain_spec)
+    o = c.symbol(demurrage_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = eth_rpc.do(o)
+    return c.parse_symbol(r)
+
+
+@pytest.fixture(scope='function')
+def demurrage_token_declaration(
+        foo_token_declaration,
+        ):
+    return foo_token_declaration
+
+
+@pytest.fixture(scope='function')
+def register_demurrage_token(
+        default_chain_spec,
+        token_registry,
+        eth_rpc,
+        eth_signer,
+        register_lookups,
+        contract_roles,
+        demurrage_token_declaration,
+        demurrage_token,
+        address_declarator,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
+
+    c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], demurrage_token)
+    eth_rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    nonce_oracle = RPCNonceOracle(contract_roles['TRUSTED_DECLARATOR'], eth_rpc)
+    c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['TRUSTED_DECLARATOR'], demurrage_token, demurrage_token_declaration)
+
+    eth_rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    return token_registry
+

apps/cic-eth-aux/erc20-demurrage-token/tests/test_demurrage_task.py

@@ -0,0 +1,69 @@
+# standard imports
+import logging
+import copy
+import datetime
+
+# external imports
+import celery
+
+# cic-eth imports
+from cic_eth_aux.erc20_demurrage_token import (
+        DemurrageCalculationTask,
+        aux_setup,
+        )
+from cic_eth_aux.erc20_demurrage_token.api import Api as AuxApi
+
+logg = logging.getLogger()
+
+
+def test_demurrage_calulate_task(
+        default_chain_spec,
+        eth_rpc,
+        cic_registry,
+        celery_session_worker,
+        register_demurrage_token,
+        demurrage_token_symbol,
+        contract_roles,
+        load_config,
+        ):
+ 
+    config = copy.copy(load_config)
+    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    config.add(demurrage_token_symbol, 'CIC_DEFAULT_TOKEN_SYMBOL', exists_ok=True)
+    aux_setup(eth_rpc, load_config, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    since = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
+    s = celery.signature(
+            'cic_eth_aux.erc20_demurrage_token.get_adjusted_balance',
+            [
+                demurrage_token_symbol,
+                1000,
+                since.timestamp(),
+                ],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    assert r == 980
+
+
+
+def test_demurrage_calculate_api(
+        default_chain_spec,
+        eth_rpc,
+        cic_registry,
+        celery_session_worker,
+        register_demurrage_token,
+        demurrage_token_symbol,
+        contract_roles,
+        load_config,
+        ):
+
+        api = AuxApi(str(default_chain_spec), queue=None)
+        since = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
+        t = api.get_adjusted_balance(demurrage_token_symbol, 1000, since.timestamp())
+        r = t.get_leaf()
+        assert t.successful()
+        assert r == 980
+

apps/cic-eth/.dockerignore

@@ -0,0 +1,6 @@
+.git
+.cache
+.dot
+**/doc
+**/.venv
+**/venv

apps/cic-eth/.gitlab-ci.yml

@@ -1,31 +1,52 @@
 .cic_eth_variables:
     variables:
         APP_NAME: cic-eth
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_eth_mr_changes_target:
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - $CONTEXT/$APP_NAME/**/*
-      when: always
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-eth:
     extends:
         - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-        - .py_build_target_test
+        - .py_build_target_dev
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-eth/**/*
+      when: always
 
 test-mr-cic-eth:
+    stage: test
     extends:
         - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-    stage: test
-    image: $IMAGE_TAG_BASE
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
     script:
         - cd apps/$APP_NAME/
-        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r admin_requirements.txt
+          -r services_requirements.txt
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-eth"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-eth/**/*
+      when: always
 
 build-push-cic-eth:
     extends:
         - .py_build_push
+        - .cic_eth_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-eth/**/*
+      when: always

apps/cic-eth/MANIFEST.in

@@ -1,2 +1,2 @@
-include *requirements.txt
+include *requirements.txt config/test/*
 

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry~=0.5.6a1
+cic-eth-registry~=0.5.6a2
 hexathon~=0.0.1a7
-chainqueue~=0.0.2b5
-eth-erc20==0.0.10a2
+chainqueue~=0.0.2b6
+eth-erc20~=0.0.10a3

apps/cic-eth/cic_eth/api/api_task.py

@@ -11,6 +11,7 @@ import celery
 from chainlib.chain import ChainSpec
 
 # local imports
+from cic_eth.api.base import ApiBase
 from cic_eth.enum import LockEnum
 
 app = celery.current_app
@@ -18,48 +19,8 @@ app = celery.current_app
 logg = logging.getLogger(__name__)
 
 
-class Api:
-    """Creates task chains to perform well-known CIC operations.
-
-    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
-
-    :param callback_param: Static value to pass to callback
-    :type callback_param: str
-    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
-    :type callback_task: string
-    :param queue: Name of worker queue to submit tasks to
-    :type queue: str
-    """
-    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop.noop', callback_queue=None):
-        self.chain_str = chain_str
-        self.chain_spec = ChainSpec.from_chain_str(chain_str)
-        self.callback_param = callback_param
-        self.callback_task = callback_task
-        self.queue = queue
-        logg.debug('api using queue {}'.format(self.queue))
-        self.callback_success = None
-        self.callback_error = None
-        if callback_queue == None:
-            callback_queue=self.queue
-
-        if callback_param != None:
-            self.callback_success = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        0,
-                        ],
-                    queue=callback_queue,
-                    )
-            self.callback_error = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        1,
-                        ],
-                    queue=callback_queue,
-                    )       
-
+class Api(ApiBase):
+    
 
     def default_token(self):
         s_token = celery.signature(

apps/cic-eth/cic_eth/api/base.py

@@ -0,0 +1,52 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+class ApiBase:
+    """Creates task chains to perform well-known CIC operations.
+
+    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
+
+    :param callback_param: Static value to pass to callback
+    :type callback_param: str
+    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
+    :type callback_task: string
+    :param queue: Name of worker queue to submit tasks to
+    :type queue: str
+    """
+    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop.noop', callback_queue=None):
+        self.chain_str = chain_str
+        self.chain_spec = ChainSpec.from_chain_str(chain_str)
+        self.callback_param = callback_param
+        self.callback_task = callback_task
+        self.queue = queue
+        logg.debug('api using queue {}'.format(self.queue))
+        self.callback_success = None
+        self.callback_error = None
+        if callback_queue == None:
+            callback_queue=self.queue
+
+        if callback_param != None:
+            self.callback_success = celery.signature(
+                    callback_task,
+                    [
+                        callback_param,
+                        0,
+                        ],
+                    queue=callback_queue,
+                    )
+            self.callback_error = celery.signature(
+                    callback_task,
+                    [
+                        callback_param,
+                        1,
+                        ],
+                    queue=callback_queue,
+                    )       
+
+

apps/cic-eth/cic_eth/db/models/base.py

@@ -126,3 +126,4 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,9 +14,11 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
+from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
+from cic_eth_registry.error import UnknownContractError
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
@@ -114,9 +116,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
-                        token = ERC20Token(chain_spec, rpc, tx['to'])
-                        token_symbol = token.symbol
-                        token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
@@ -126,12 +125,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
                             'destination_token': tx['to'],
-                            'source_token_symbol': token_symbol,
-                            'destination_token_symbol': token_symbol,
-                            'source_token_decimals': token_decimals,
-                            'destination_token_decimals': token_decimals,
-                            'source_token_chain': chain_str,
-                            'destination_token_chain': chain_str,
                             'nonce': tx['nonce'],
                                 }
                         if times['queue'] != None:
@@ -146,8 +139,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
-@celery_app.task()
-def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
+@celery_app.task(bind=True, base=BaseTask)
+def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=True, verify_contracts=True):
     """Merges transaction data from multiple sources and sorts them in chronological order.
 
     :param tx_batches: Transaction data inputs
@@ -196,6 +189,32 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
     if newest_first:
         ks.reverse()
     for k in ks:
-        txs.append(txs_by_block[k])
+        tx = txs_by_block[k]
+        if verify_contracts:
+            try:
+                tx = verify_and_expand(tx, chain_spec, sender_address=BaseTask.call_address)
+            except UnknownContractError:
+                logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
+                continue
+        txs.append(tx)
 
     return txs
+
+
+def verify_and_expand(tx, chain_spec, sender_address=ZERO_ADDRESS):
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+
+    if tx.get('source_token_symbol') == None and tx['source_token'] != ZERO_ADDRESS:
+        r = registry.by_address(tx['source_token'], sender_address=sender_address)
+        token = ERC20Token(chain_spec, rpc, tx['source_token'])
+        tx['source_token_symbol'] = token.symbol
+        tx['source_token_decimals'] = token.decimals
+
+    if tx.get('destination_token_symbol') == None and tx['destination_token'] != ZERO_ADDRESS:
+        r = registry.by_address(tx['destination_token'], sender_address=sender_address)
+        token = ERC20Token(chain_spec, rpc, tx['destination_token'])
+        tx['destination_token_symbol'] = token.symbol
+        tx['destination_token_decimals'] = token.decimals
+
+    return tx

apps/cic-eth/cic_eth/pytest/fixtures_celery.py

@@ -37,7 +37,7 @@ def celery_includes():
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
-        'tests.mock.filter',
+        'cic_eth.pytest.mock.filter',
     ]
 
 

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -13,7 +13,9 @@ logg = logging.getLogger(__name__)
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.path.join(root_dir, 'config/test')
+    config_dir = os.environ.get('CONFINI_DIR')
+    if config_dir == None:
+        config_dir = os.path.join(root_dir, 'config/test')
     conf = confini.Config(config_dir, 'CICTEST')
     conf.process()
     logg.debug('config {}'.format(conf))

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -27,7 +27,7 @@ def database_engine(
         SessionBase.poolable = False
     dsn = dsn_from_config(load_config)
     #SessionBase.connect(dsn, True)
-    SessionBase.connect(dsn, debug=load_config.get('DATABASE_DEBUG') != None)
+    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
     return dsn
 
 

apps/cic-eth/cic_eth/pytest/fixtures_token.py

@@ -0,0 +1,19 @@
+# external imports
+import pytest
+from eth_erc20 import ERC20
+
+# TODO: missing dep fixture includes
+
+
+@pytest.fixture(scope='function')
+def foo_token_symbol(
+    default_chain_spec,
+    foo_token,
+    eth_rpc,
+    contract_roles,
+    ):
+    
+    c = ERC20(default_chain_spec)
+    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = eth_rpc.do(o)
+    return c.parse_symbol(r)

apps/cic-eth/cic_eth/queue/query.py

@@ -100,6 +100,7 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     q_outer = q_outer.join(Lock, isouter=True)
     q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
 
+
     if not is_alive(status):
         SessionBase.release_session(session)
         raise ValueError('not a valid non-final tx value: {}'.format(status))

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -90,6 +90,7 @@ class DispatchSyncer:
 
     def __init__(self, chain_spec):
         self.chain_spec = chain_spec
+        self.session = None
 
 
     def chain(self):
@@ -100,16 +101,18 @@ class DispatchSyncer:
         c = len(txs.keys())
         logg.debug('processing {} txs {}'.format(c, list(txs.keys())))
         chain_str = str(self.chain_spec)
-        session = SessionBase.create_session()
+        self.session = SessionBase.create_session()
         for k in txs.keys():
             tx_raw = txs[k]
             tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
             tx = unpack(tx_raw_bytes, self.chain_spec)
             
             try:
-                set_reserved(self.chain_spec, tx['hash'], session=session)
+                set_reserved(self.chain_spec, tx['hash'], session=self.session)
+                self.session.commit()
             except NotLocalTxError as e:
                 logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash']))
+                self.session.rollback()
                 continue
 
             s_check = celery.signature(
@@ -132,16 +135,25 @@ class DispatchSyncer:
             s_check.link(s_send)
             t = s_check.apply_async()
             logg.info('processed {}'.format(k))
+        self.session.close()
+        self.session = None
 
 
-    def loop(self, w3, interval):
+    def loop(self, interval):
         while run:
             txs = {}
             typ = StatusBits.QUEUED
             utxs = get_upcoming_tx(self.chain_spec, typ)
             for k in utxs.keys():
                 txs[k] = utxs[k]
-            self.process(w3, txs)
+            try:
+                conn = RPCConnection.connect(self.chain_spec, 'default')
+                self.process(conn, txs)
+            except ConnectionError as e:
+                if self.session != None:
+                    self.session.close()
+                    self.session = None
+                logg.error('connection to node failed: {}'.format(e))
 
             if len(utxs) > 0:
                 time.sleep(self.yield_delay)
@@ -151,8 +163,7 @@ class DispatchSyncer:
 
 def main(): 
     syncer = DispatchSyncer(chain_spec)
-    conn = RPCConnection.connect(chain_spec, 'default')
-    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    syncer.loop(float(config.get('DISPATCHER_LOOP_INTERVAL')))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -12,20 +12,24 @@ from hexathon import (
 # local imports
 from .base import SyncFilter
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 
 
 class RegistrationFilter(SyncFilter):
 
-    def __init__(self, chain_spec, queue):
+    def __init__(self, chain_spec, contract_address, queue=None):
         self.chain_spec = chain_spec
         self.queue = queue
+        self.contract_address = contract_address
 
 
     def filter(self, conn, block, tx, db_session=None): 
-        registered_address = None
+        if self.contract_address != tx.inputs[0]:
+            logg.debug('not an account registry tx; {} != {}'.format(self.contract_address, tx.inputs[0]))
+            return None
+
         for l in tx.logs:
             event_topic_hex = l['topics'][0]
             if event_topic_hex == account_registry_add_log_hash:

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -7,6 +7,8 @@ import tempfile
 import re
 import urllib
 import websocket
+import stat
+import importlib
 
 # external imports
 import celery
@@ -68,6 +70,8 @@ from cic_eth.task import BaseTask
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
 config_dir = os.path.join('/usr/local/etc/cic-eth')
 
 argparser = argparse.ArgumentParser()
@@ -79,6 +83,8 @@ argparser.add_argument('--default-token-symbol', dest='default_token_symbol', ty
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
+argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -109,6 +115,8 @@ if len(health_modules) != 0:
     health_modules = health_modules.split(',')
 logg.debug('health mods {}'.format(health_modules))
 
+
+
 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
@@ -167,6 +175,84 @@ Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 #    raise RuntimeError()
 liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker')
 
+rpc = RPCConnection.connect(chain_spec, 'default')
+try:
+    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+except UnknownContractError as e:
+    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+    sys.exit(1)
+logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
+
+trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
+if trusted_addresses_src == None:
+    logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
+    sys.exit(1)
+trusted_addresses = trusted_addresses_src.split(',')
+for address in trusted_addresses:
+    logg.info('using trusted address {}'.format(address))
+
+connect_declarator(rpc, chain_spec, trusted_addresses)
+connect_token_registry(rpc, chain_spec)
+
+# detect aux 
+# TODO: move to separate file
+#aux_dir = os.path.join(script_dir, '..', '..', 'aux')
+aux = []
+if args.aux_all:
+    if len(args.aux) > 0:
+        logg.warning('--aux-all is set so --aux will have no effect')
+    for p in sys.path:
+        logg.debug('checking for aux modules in {}'.format(p))
+        aux_dir = os.path.join(p, 'cic_eth_aux')
+        try:
+            d = os.listdir(aux_dir)
+        except FileNotFoundError:
+            logg.debug('no aux module found in {}'.format(aux_dir))
+            continue
+        for v in d:
+            if v[:1] == '.':
+                logg.debug('dotfile, skip {}'.format(v))
+                continue
+            aux_mod_path = os.path.join(aux_dir, v)
+            st = os.stat(aux_mod_path)
+            if not stat.S_ISDIR(st.st_mode):
+                logg.debug('not a dir, skip {}'.format(v))
+                continue
+            aux_mod_file = os.path.join(aux_dir, v,'__init__.py')
+            try:
+                st = os.stat(aux_mod_file)
+            except FileNotFoundError:
+                logg.debug('__init__.py not found, skip {}'.format(v))
+                continue 
+            aux.append(v)
+            logg.debug('found module {} in {}'.format(v, aux_dir))
+
+elif len(args.aux) > 0:
+    for p in sys.path:
+        v_found = None
+        for v in args.aux:
+            aux_dir = os.path.join(p, 'cic_eth_aux')
+            aux_mod_file = os.path.join(aux_dir, v, '__init__.py')
+            try:
+                st = os.stat(aux_mod_file)
+                v_found = v
+            except FileNotFoundError:
+                logg.debug('cannot find explicity requested aux module {} in path {}'.format(v, aux_dir))
+                continue
+        if v_found == None:
+            logg.critical('excplicity requested aux module {} not found in any path'.format(v))
+            sys.exit(1)
+
+        logg.info('aux module {} found in path {}'.format(v, aux_dir))
+        aux.append(v)
+
+for v in aux:
+    mname = 'cic_eth_aux.' + v
+    mod = importlib.import_module(mname)
+    mod.aux_setup(rpc, config)
+    logg.info('loaded aux module {}'.format(mname))
+
+
 def main():
     argv = ['worker']
     if args.vv:
@@ -189,24 +275,6 @@ def main():
 
     rpc = RPCConnection.connect(chain_spec, 'default')
 
-    try:
-        registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-    except UnknownContractError as e:
-        logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-        sys.exit(1)
-    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
-
-    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-    if trusted_addresses_src == None:
-        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-        sys.exit(1)
-    trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
-        logg.info('using trusted address {}'.format(address))
-
-    connect_declarator(rpc, chain_spec, trusted_addresses)
-    connect_token_registry(rpc, chain_spec)
-
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
     default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -78,6 +78,14 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
+rpc = RPCConnection.connect(chain_spec, 'default')
+registry = None
+try:
+    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+except UnknownContractError as e:
+    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+    sys.exit(1)
+logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 
 def main():
@@ -85,7 +93,6 @@ def main():
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # Connect to blockchain with chainlib
-    rpc = RPCConnection.connect(chain_spec, 'default')
 
     o = block_latest()
     r = rpc.do(o)
@@ -151,7 +158,8 @@ def main():
 
     tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
 
-    registration_filter = RegistrationFilter(chain_spec, config.get('_CELERY_QUEUE'))
+    account_registry_address = registry.by_name('AccountRegistry')
+    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE'))
 
     gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
 

apps/cic-eth/cic_eth/task.py

@@ -1,6 +1,5 @@
 # import
 import time
-import requests
 import logging
 import uuid
 
@@ -76,7 +75,7 @@ class CriticalSQLAlchemyTask(CriticalTask):
 
 class CriticalWeb3Task(CriticalTask):
     autoretry_for = (
-        requests.exceptions.ConnectionError,
+        ConnectionError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
@@ -86,7 +85,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
     autoretry_for = (
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
-        requests.exceptions.ConnectionError,
+        ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
@@ -102,7 +101,7 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
 
 class CriticalWeb3AndSignerTask(CriticalTask):
     autoretry_for = (
-        requests.exceptions.ConnectionError,
+        ConnectionError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         12,
-        0,
-        'alpha.1',
+        1,
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/test/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=
+DEBUG=0

apps/cic-eth/docker/Dockerfile

@@ -1,74 +1,71 @@
-FROM python:3.8.6-slim-buster as compile
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-WORKDIR /usr/src/cic-eth
-
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-#RUN python -m venv venv && . venv/bin/activate
-
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN /usr/local/bin/python -m pip install --upgrade pip
-RUN pip install semver
-
-COPY cic-eth/ .
-RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    	--extra-index-url $EXTRA_INDEX_URL .
-
-# --- TEST IMAGE ---
-FROM python:3.8.6-slim-buster as test 
-
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-WORKDIR /usr/src/cic-eth
-
-RUN /usr/local/bin/python -m pip install --upgrade pip
-
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-# TODO we could use venv inside container to isolate the system and app deps further
-# COPY --from=compile /usr/src/cic-eth/ .
-# RUN . venv/bin/activate
+# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+# do load these all into setup.py later
+# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+#COPY cic-eth/requirements.txt .
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-
-COPY cic-eth/test_requirements.txt .
-RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    	--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
-
-COPY cic-eth . 
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#    pip install --index-url https://pypi.org/simple \
+#    --force-reinstall \
+#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+#    -r requirements.txt 
+COPY *requirements.txt . 
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt \
+    -r services_requirements.txt \
+    -r admin_requirements.txt
+    
+COPY . .
+RUN python setup.py install
 
 ENV PYTHONPATH .
 
-ENTRYPOINT ["pytest"]
-
-# --- RUNTIME ---
-FROM python:3.8.6-slim-buster as runtime 
-
-RUN apt-get update && \
-	apt install -y gnupg libpq-dev procps
-
-WORKDIR /usr/src/cic-eth
-
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-
-COPY cic-eth/docker/* ./ 
+COPY docker/entrypoints/* ./ 
 RUN chmod 755 *.sh
 
-COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-eth/config/ /usr/local/etc/cic-eth/
-COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+COPY config/ /usr/local/etc/cic-eth/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-COPY util/liveness/health.sh /usr/local/bin/health.sh
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
+ENTRYPOINT []
+
+## ------------------ PRODUCTION CONTAINER ----------------------
+#FROM python:3.8.6-slim-buster as prod 
+#
+#RUN apt-get update && \
+#	apt install -y gnupg libpq-dev procps
+#
+#WORKDIR /root
+#
+#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+#                          /usr/local/lib/python3.8/site-packages/
+#
+#COPY docker/entrypoints/* ./ 
+#RUN chmod 755 *.sh
+#
+## # ini files in config directory defines the configurable parameters for the application
+## # they can all be overridden by environment variables
+## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+#COPY config/ /usr/local/etc/cic-eth/
+#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+#COPY scripts/ scripts/ 
+#
+## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+##COPY util/liveness/health.sh /usr/local/bin/health.sh
+#
+#ENTRYPOINT []
 

apps/cic-eth/docker/Dockerfile_ci

@@ -0,0 +1,69 @@
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+WORKDIR /usr/src/cic-eth
+
+# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+# do load these all into setup.py later
+# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+#COPY cic-eth/requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#    pip install --index-url https://pypi.org/simple \
+#    --force-reinstall \
+#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+#    -r requirements.txt 
+COPY *requirements.txt . 
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt \
+    -r services_requirements.txt \
+    -r admin_requirements.txt
+    
+COPY . .
+RUN python setup.py install
+
+COPY docker/entrypoints/* ./ 
+RUN chmod 755 *.sh
+
+# # ini files in config directory defines the configurable parameters for the application
+# # they can all be overridden by environment variables
+# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-eth/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
+ENTRYPOINT []
+
+# ------------------ PRODUCTION CONTAINER ----------------------
+#FROM python:3.8.6-slim-buster as prod 
+#
+#RUN apt-get update && \
+#	apt install -y gnupg libpq-dev procps
+#
+#WORKDIR /root
+#
+#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+#                          /usr/local/lib/python3.8/site-packages/
+#
+#COPY docker/entrypoints/* ./ 
+#RUN chmod 755 *.sh
+#
+## # ini files in config directory defines the configurable parameters for the application
+## # they can all be overridden by environment variables
+## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+#COPY config/ /usr/local/etc/cic-eth/
+#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+#COPY scripts/ scripts/ 
+#
+## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+##COPY util/liveness/health.sh /usr/local/bin/health.sh
+#
+#ENTRYPOINT []
+#

apps/cic-eth/docker/entrypoints/start_tasker.sh

@@ -8,7 +8,7 @@ set -e
 echo "!!! starting signer"
 python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
 
-echo "!!! starting tracker"
+echo "!!! starting taskerd"
 /usr/local/bin/cic-eth-taskerd $@
 
 # thanks! https://docs.docker.com/config/containers/multi-service_container/

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib~=0.0.5a1
+chainlib~=0.0.5a2
 semver==2.13.0

apps/cic-eth/services_requirements.txt

@@ -1,7 +1,15 @@
-chainsyncer[sql]~=0.0.3a3
-chainqueue~=0.0.2b5
+chainsyncer[sql]~=0.0.3a5
+chainqueue~=0.0.2b7
 alembic==1.4.2
-confini~=0.3.6rc4
+confini>=0.3.6rc4,<0.5.0
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1
+liveness~=0.0.1a7
+eth-address-index~=0.1.2a2
+eth-accounts-index~=0.0.12a2
+cic-eth-registry~=0.5.6a2
+erc20-faucet~=0.2.2a2
+erc20-transfer-authorization~=0.3.2a2
+sarafu-faucet~=0.0.4a3
+moolb~=0.1.1b2

apps/cic-eth/setup.cfg

@@ -39,6 +39,9 @@ packages =
 	cic_eth.callbacks
 	cic_eth.sync
 	cic_eth.check
+	# should be concealed behind extras "test" if possible (but its not unfortunately)
+	cic_eth.pytest
+	cic_eth.pytest.mock
 scripts =
 	./scripts/migrate.py
 

apps/cic-eth/setup.py

@@ -48,4 +48,4 @@ setup(
         'admin_api': admin_requirements,
         'services': services_requirements,
         }
-        )
+    )

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.0.10a2
+eth-erc20~=0.0.10a3

apps/cic-eth/tests/conftest.py

@@ -5,6 +5,7 @@ import logging
 import uuid
 
 # external imports
+import pytest
 from eth_erc20 import ERC20
 import redis
 
@@ -22,6 +23,7 @@ from cic_eth.pytest.fixtures_celery import *
 from cic_eth.pytest.fixtures_database import *
 from cic_eth.pytest.fixtures_role import *
 from cic_eth.pytest.fixtures_contract import *
+from cic_eth.pytest.fixtures_token import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *
@@ -37,20 +39,6 @@ def api(
     return Api(chain_str, queue=None, callback_param='foo')
 
 
-@pytest.fixture(scope='function')
-def foo_token_symbol(
-    default_chain_spec,
-    foo_token,
-    eth_rpc,
-    contract_roles,
-    ):
-    
-    c = ERC20(default_chain_spec)
-    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)
-
-
 @pytest.fixture(scope='function')
 def default_token(
         foo_token,

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -18,6 +18,7 @@ def test_filter_bogus(
         cic_registry,
         contract_roles,
         register_lookups,
+        account_registry,
         ):
 
     fltrs = [
@@ -26,7 +27,7 @@ def test_filter_bogus(
         TxFilter(default_chain_spec, None),
         CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
         StragglerFilter(default_chain_spec, None),
-        RegistrationFilter(default_chain_spec, queue=None),
+        RegistrationFilter(default_chain_spec, account_registry, queue=None),
         ]
       
     for fltr in fltrs:

apps/cic-eth/tests/filters/test_register_filter.py

@@ -1,3 +1,7 @@
+# standard imports
+import logging
+import os
+
 # external imports
 from eth_accounts_index.registry import AccountRegistry
 from chainlib.connection import RPCConnection
@@ -14,12 +18,17 @@ from chainlib.eth.block import (
         Block,
         )
 from erc20_faucet import Faucet
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
 from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
 
+logg = logging.getLogger()
+
 
 def test_register_filter(
         default_chain_spec,
@@ -60,7 +69,11 @@ def test_register_filter(
     tx = Tx(tx_src, block=block, rcpt=rcpt)
     tx.apply_receipt(rcpt)
 
-    fltr = RegistrationFilter(default_chain_spec, queue=None)
+    fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    assert t == None
+
+    fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
 
     t.get_leaf()

apps/cic-eth/tests/task/api/test_list.py

@@ -15,7 +15,7 @@ from cic_eth.db.models.nonce import (
         )
 
 # test imports
-from tests.mock.filter import (
+from cic_eth.pytest.mock.filter import (
         block_filter,
         tx_filter,
         )
@@ -110,7 +110,7 @@ def test_list_tx(
     logg.debug('r {}'.format(r))
 
     # test the api
-    t = api.list(agent_roles['ALICE'], external_task='tests.mock.filter.filter')
+    t = api.list(agent_roles['ALICE'], external_task='cic_eth.pytest.mock.filter.filter')
     r = t.get_leaf()
     assert t.successful()
 

apps/cic-eth/tests/task/test_task_address.py

@@ -1,4 +1,5 @@
 # external imports
+import celery
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
@@ -20,6 +21,7 @@ def test_translate(
         cic_registry,
         init_celery_tasks,
         register_lookups,
+        celery_session_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
@@ -46,6 +48,20 @@ def test_translate(
         'recipient': agent_roles['BOB'],
         'recipient_label': None,
             }
-    tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
-    assert tx['sender_label'] == 'alice'
-    assert tx['recipient_label'] == 'bob'
+    
+    #tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
+    s = celery.signature(
+            'cic_eth.ext.address.translate_tx_addresses',
+            [
+                tx,
+                [contract_roles['CONTRACT_DEPLOYER']],
+                default_chain_spec.asdict(),
+                ],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    assert r['sender_label'] == 'alice'
+    assert r['recipient_label'] == 'bob'

apps/cic-eth/tests/task/test_task_list.py

@@ -0,0 +1,92 @@
+# external imports
+import celery
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.gas import (
+        RPCGasOracle,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from eth_erc20 import ERC20
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainqueue.db.models.tx import TxCache
+from chainqueue.db.models.otx import Otx
+
+
+def test_ext_tx_collate(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        custodial_roles,
+        agent_roles,
+        foo_token,
+        bar_token,
+        register_tokens,
+        cic_registry,
+        register_lookups,
+        init_celery_tasks,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], eth_rpc)
+    gas_oracle = RPCGasOracle(eth_rpc)
+
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    transfer_value_foo = 1000
+    transfer_value_bar = 1024
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], transfer_value_foo, tx_format=TxFormat.RLP_SIGNED)
+    tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+
+    otx = Otx(
+        tx['nonce'],
+        tx_hash_hex,
+        tx_signed_raw_hex,
+        )
+    init_database.add(otx)
+    init_database.commit()
+
+    txc = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        tx['to'],
+        foo_token,
+        bar_token,
+        transfer_value_foo,
+        transfer_value_bar,
+        666,
+        13,
+        session=init_database,
+            )
+    init_database.add(txc)
+    init_database.commit()
+    
+    s = celery.signature(
+            'cic_eth.ext.tx.tx_collate',
+            [
+                {tx_hash_hex: tx_signed_raw_hex},
+                default_chain_spec.asdict(),
+                0,
+                100,
+                ],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    assert len(r) == 1
+
+    tx = r[0]
+    assert tx['source_token_symbol'] == 'FOO'
+    assert tx['source_token_decimals'] == 6
+    assert tx['destination_token_symbol'] == 'BAR'
+    assert tx['destination_token_decimals'] == 9

apps/cic-eth/tests/unit/db/test_nonce_db.py

@@ -1,6 +1,7 @@
-# third-party imports
+# extended imports
 import pytest
 import uuid
+import unittest
 
 # local imports
 from cic_eth.db.models.nonce import (
@@ -55,7 +56,7 @@ def test_nonce_reserve(
     o = q.first()
     assert o.nonce == 43
 
-    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu))
+    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu), session=init_database)
     init_database.commit()
     assert nonce == (str(uu), 42)
 

apps/cic-eth/tools_requirements.txt

@@ -1,7 +1,7 @@
-crypto-dev-signer~=0.4.14b6
+crypto-dev-signer~=0.4.14b7
 chainqueue~=0.0.2b5
-confini~=0.3.6rc4
-cic-eth-registry~=0.5.6a1
+confini>=0.3.6rc4,<0.5.0
+cic-eth-registry~=0.5.6a2
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1

apps/cic-meta/.dockerignore

@@ -0,0 +1,5 @@
+.git
+.cache
+.dot
+**/doc
+**/node_modules

apps/cic-meta/.gitlab-ci.yml

@@ -2,43 +2,42 @@
 .cic_meta_variables:
     variables:
         APP_NAME: cic-meta
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
-.cic_meta_changes_target:
+build-mr-cic-meta:
+    extends:
+        - .py_build_merge_request
+        - .cic_meta_variables
     rules:
-        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-#        - changes:
-#            - $CONTEXT/$APP_NAME/*
-        - when: always
-
-cic-meta-build-mr:
-  stage: build
-  extends:
-    - .cic_meta_variables
-    - .cic_meta_changes_target
-  script:
-    - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-meta/**/*
+      when: always
 
 test-mr-cic-meta:
     extends:
         - .cic_meta_variables
-        - .cic_meta_changes_target
     stage: test
-    image: $IMAGE_TAG 
+    image: $MR_IMAGE_TAG 
     script:
-        - cd /tmp/src/cic-meta
+        - cd /root
         - npm install --dev 
         - npm run test
         - npm run test:coverage
-    needs: ["cic-meta-build-mr"]
+    needs: ["build-mr-cic-meta"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-meta/**/*
+      when: always
 
 build-push-cic-meta:
     extends:
         - .py_build_push
         - .cic_meta_variables
-        
-
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-meta/**/*
+      when: always

apps/cic-meta/docker/Dockerfile

@@ -1,31 +1,33 @@
-FROM node:15.3.0-alpine3.10
+# syntax = docker/dockerfile:1.2
+#FROM node:15.3.0-alpine3.10
+FROM node:lts-alpine3.14
 
-WORKDIR /tmp/src/cic-meta
+WORKDIR /root
 
 RUN apk add --no-cache postgresql bash
 
-# required to build the cic-client-meta module
-COPY cic-meta/src/ src/
-COPY cic-meta/scripts/ scripts/
-
 # copy the dependencies
-COPY cic-meta/package.json .
-COPY cic-meta/tsconfig.json . 
-COPY cic-meta/webpack.config.js .
-     
-RUN npm install
+COPY package.json package-lock.json .
+RUN --mount=type=cache,mode=0755,target=/root/.npm \
+    npm set cache /root/.npm && \
+    npm ci
 
-COPY cic-meta/tests/ tests/
-COPY cic-meta/tests/*.asc /root/pgp/
+COPY webpack.config.js .
+COPY tsconfig.json . 
+## required to build the cic-client-meta module
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
+COPY tests/*.asc /root/pgp/
 
-# copy runtime configs
-COPY cic-meta/.config/ /usr/local/etc/cic-meta/
-
-# db migrations
-COPY cic-meta/docker/db.sh ./db.sh
+## copy runtime configs
+COPY .config/ /usr/local/etc/cic-meta/
+#
+## db migrations
+COPY docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
-
+#
 RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY cic-meta/docker/start_server.sh ./start_server.sh
+COPY docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+#FROM node:15.3.0-alpine3.10
+FROM node:lts-alpine3.14
+
+WORKDIR /root
+
+RUN apk add --no-cache postgresql bash
+
+# copy the dependencies
+COPY package.json package-lock.json .
+RUN npm set cache /root/.npm && \
+    npm ci
+
+COPY webpack.config.js .
+COPY tsconfig.json . 
+## required to build the cic-client-meta module
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
+COPY tests/*.asc /root/pgp/
+
+## copy runtime configs
+COPY .config/ /usr/local/etc/cic-meta/
+#
+## db migrations
+COPY docker/db.sh ./db.sh
+RUN chmod 755 ./db.sh
+#
+RUN alias tsc=node_modules/typescript/bin/tsc 
+COPY docker/start_server.sh ./start_server.sh
+RUN chmod 755 ./start_server.sh
+ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-notify/.gitlab-ci.yml

@@ -1,22 +1,52 @@
 .cic_notify_variables:
     variables:
         APP_NAME: cic-notify
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_notify_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-notify:
     extends:
-        - .cic_notify_changes_target
         - .py_build_merge_request
         - .cic_notify_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-notify/**/*
+      when: always
+
+test-mr-cic-notify:
+    stage: test
+    extends:
+        - .cic_notify_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-notify"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
 
 build-push-cic-notify:
     extends:
         - .py_build_push
         - .cic_notify_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-notify/**/*
+      when: always
         
 

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.7')
+version = (0, 4, 0, 'alpha.8')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/docker/Dockerfile

@@ -1,38 +1,30 @@
-FROM python:3.8.6-slim-buster
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps
+#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
 
-WORKDIR /usr/src/cic-notify
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
 
-COPY cic-notify/setup.cfg \
-	 cic-notify/setup.py \
-     ./
+COPY . . 
 
-COPY cic-notify/cic_notify/ ./cic_notify/
+RUN python setup.py install
 
-COPY cic-notify/requirements.txt \
-	 cic-notify/test_requirements.txt \
-     ./
-
-COPY cic-notify/scripts/ scripts/
+# TODO please review..can this go into requirements?
 RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
 
-COPY cic-notify/tests/ tests/
-COPY cic-notify/docker/db.sh \
-     cic-notify/docker/start_tasker.sh \
-	 /root/
-
-#RUN apk add postgresql-client
-#RUN apk add bash
+COPY docker/*.sh .
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-notify/.config/ /usr/local/etc/cic-notify/
-COPY cic-notify/cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+COPY .config/ /usr/local/etc/cic-notify/
+COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
 
-WORKDIR /root
+ENTRYPOINT []

apps/cic-notify/docker/Dockerfile_ci

@@ -0,0 +1,29 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
+
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . . 
+
+RUN python setup.py install
+
+# TODO please review..can this go into requirements?
+RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
+
+COPY docker/*.sh .
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY .config/ /usr/local/etc/cic-notify/
+COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+
+ENTRYPOINT []

apps/cic-notify/requirements.txt

@@ -1 +1 @@
-cic_base[full_graph]==0.1.3a3+build.984b5cff
+cic_base[full_graph]~=0.2.0a3

apps/cic-ussd/.config/app.ini

@@ -1,7 +1,7 @@
 [app]
 ALLOWED_IP=0.0.0.0/0
 LOCALE_FALLBACK=en
-LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
+LOCALE_PATH=var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#,*483*061#,*384*96#
@@ -11,13 +11,13 @@ SUPPORT_PHONE_NUMBER=0757628885
 REGION=KE
 
 [ussd]
-MENU_FILE=/usr/src/data/ussd_menu.json
+MENU_FILE=data/ussd_menu.json
 user = 
 pass = 
 
 [statemachine]
-STATES=/usr/src/cic-ussd/states/
-TRANSITIONS=/usr/src/cic-ussd/transitions/
+STATES=states/
+TRANSITIONS=transitions/
 
 [client]
 host =

apps/cic-ussd/.config/pgp.ini

@@ -1,5 +1,5 @@
 [pgp]
-export_dir = /usr/src/pgp/keys/
+export_dir = pgp/keys/
 keys_path = /usr/src/secrets/
 private_keys = privatekeys_meta.asc
 passphrase =

apps/cic-ussd/.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

apps/cic-ussd/.gitlab-ci.yml

@@ -1,22 +1,52 @@
 .cic_ussd_variables:
     variables:
         APP_NAME: cic-ussd
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_ussd_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-ussd:
     extends:
-        - .cic_ussd_changes_target
         - .py_build_merge_request
         - .cic_ussd_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-ussd/**/*
+      when: always
+
+test-mr-cic-ussd:
+    stage: test
+    extends:
+        - .cic_ussd_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-ussd"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
 
 build-push-cic-ussd:
     extends:
         - .py_build_push
         - .cic_ussd_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-ussd/**/*
+      when: always
 
 

apps/cic-ussd/cic_ussd/db/models/base.py

@@ -127,3 +127,4 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -28,7 +28,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config = Config(args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -45,7 +45,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config = Config(args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/docker/Dockerfile

@@ -1,14 +1,5 @@
-# FROM python:3.8.5-alpine
-FROM python:3.8.6-slim-buster
-
-# set working directory
-WORKDIR /usr/src
-
-# add args for installing from self-hosted packages
-ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
-
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
 # create secrets directory
 RUN mkdir -vp pgp/keys
@@ -17,39 +8,25 @@ RUN mkdir -vp pgp/keys
 RUN mkdir -vp cic-ussd
 RUN mkdir -vp data
 
-COPY cic-ussd/setup.cfg \
-     cic-ussd/setup.py \
-     cic-ussd/
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
 
-COPY cic-ussd/requirements.txt \
-     cic-ussd/test_requirements.txt \
-     cic-ussd/
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
 
-# install requirements
-RUN cd cic-ussd && \
-    pip install -r requirements.txt $pip_extra_index_url_flag
+COPY . .
+RUN python setup.py install
 
-# copy all necessary files
-COPY cic-ussd/cic_ussd/ cic-ussd/cic_ussd/
-COPY cic-ussd/cic_ussd/db/ussd_menu.json data/
-COPY cic-ussd/scripts/ cic-ussd/scripts/
-COPY cic-ussd/states/ cic-ussd/states/
-COPY cic-ussd/transitions/ cic-ussd/transitions/
-COPY cic-ussd/var/ cic-ussd/var/
-
-COPY cic-ussd/docker/db.sh \
-     cic-ussd/docker/start_cic_user_tasker.sh \
-     cic-ussd/docker/start_cic_user_ussd_server.sh\
-     cic-ussd/docker/start_cic_user_server.sh\
-     /root/
+COPY cic_ussd/db/ussd_menu.json data/
 
+COPY docker/*.sh .
 RUN chmod +x /root/*.sh
-RUN cd cic-ussd && \
-    pip install $pip_extra_index_url_flag .
-
 
 # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY cic-ussd/.config/ /usr/local/etc/cic-ussd/
-COPY cic-ussd/cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+COPY .config/ /usr/local/etc/cic-ussd/
+COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
 
-WORKDIR /root
+ENTRYPOINT []

apps/cic-ussd/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+
+# create secrets directory
+RUN mkdir -vp pgp/keys
+
+# create application directory
+RUN mkdir -vp cic-ussd
+RUN mkdir -vp data
+
+COPY requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . .
+RUN python setup.py install
+
+COPY cic_ussd/db/ussd_menu.json data/
+
+COPY docker/*.sh .
+RUN chmod +x /root/*.sh
+
+# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
+COPY .config/ /usr/local/etc/cic-ussd/
+COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+
+ENTRYPOINT []

apps/cic-ussd/docker/start_cic_user_server.sh

@@ -4,4 +4,4 @@
 
 user_server_port=${SERVER_PORT:-9500}
 
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_cic_user_ussd_server.sh

@@ -4,4 +4,4 @@
 
 user_ussd_server_port=${SERVER_PORT:-9000}
 
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]==0.1.3a3+build.984b5cff
+cic_base[full_graph]==0.2.0a3
 cic-eth~=0.12.0a1
-cic-notify~=0.4.0a7
+cic-notify~=0.4.0a8
 cic-types~=0.1.0a11

apps/cic-ussd/scripts/migrate.py

@@ -30,7 +30,7 @@ arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
 args = arg_parser.parse_args()
 
 
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config = Config(args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug(f'config:\n{config}')

apps/contract-migration/.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

apps/contract-migration/.gitlab-ci.yml

@@ -1,20 +1,25 @@
 .contract_migration_variables:
     variables:
         APP_NAME: contract-migration
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
-
-.contract_migration_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-contract-migration:
     extends:
-        - .contract_migration_changes_target
         - .py_build_merge_request
         - .contract_migration_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/contract-migration/**/*
+      when: always
 
 build-push-contract-migration:
     extends:
         - .py_build_push
         - .contract_migration_variables
+    rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/contract-migration/**/*
+      when: always

apps/contract-migration/docker/Dockerfile

@@ -25,7 +25,7 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-COPY contract-migration/requirements.txt . 
+COPY requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@@ -38,5 +38,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
 
-COPY contract-migration/ .
+COPY . .
 RUN chmod +x *.sh

apps/contract-migration/docker/Dockerfile_ci

@@ -3,12 +3,11 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55
 
 WORKDIR /root
 
-# solc install which we needed for bancor. Leaving as an artfact of HOW to do it. 
-#RUN touch /etc/apt/sources.list.d/ethereum.list
-#RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
-#RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
-#RUN cat /etc/apt/sources.list.d/ethereum.list
-#RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
+RUN touch /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
+RUN cat /etc/apt/sources.list.d/ethereum.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
 
 #RUN apt-get install solc
 
@@ -26,7 +25,7 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-COPY contract-migration/requirements.txt . 
+COPY requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@@ -38,5 +37,5 @@ RUN pip install --index-url https://pypi.org/simple \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
 
-COPY contract-migration/ .
+COPY . .
 RUN chmod +x *.sh

apps/contract-migration/requirements.txt

@@ -1,5 +1,5 @@
 cic_base[full]==0.1.3a4+build.ce68c833
 sarafu-faucet~=0.0.4a1
-cic-eth==0.11.1a3+build.a3c49324
+cic-eth[tools]==0.12.0a2
 eth-erc20~=0.0.10a3
 erc20-demurrage-token==0.0.2a3

apps/contract-migration/reset.sh

@@ -3,14 +3,9 @@
 set -a
 
 default_token=giftable_erc20_token
-CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
 TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL}
-TOKEN_NAME=${TOKEN_NAME:-$TOKEN_SYMBOL}
+TOKEN_NAME=${TOKEN_NAME}
 TOKEN_TYPE=${TOKEN_TYPE:-$default_token}
-if [ $TOKEN_TYPE == 'default' ]; then
-	>&2 echo resolving "default" token to $default_token
-	TOKEN_TYPE=$default_token
-fi
 cat <<EOF
 external token settings:
 token_type: $TOKEN_TYPE
@@ -23,7 +18,6 @@ token_supply_limit: $TOKEN_SUPPLY_LIMIT
 EOF
 
 CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
-TOKEN_SYMBOL=${TOKEN_SYMBOL:-GFT}
 DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
@@ -75,17 +69,34 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 	fi
 
-	if [ $TOKEN_TYPE == $default_token ]; then
+	if [ "$TOKEN_TYPE" == "$default_token" ]; then
+		if [ -z "$TOKEN_SYMBOL" ]; then
+			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
+			TOKEN_SYMBOL="GFT"
+			TOKEN_NAME="Giftable Token"
+		elif [ -z "$TOKEN_NAME" ]; then
+			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
+			TOKEN_NAME=$TOKEN_SYMBOL
+		fi
 		>&2 echo deploying default token $TOKEN_TYPE
-		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name $TOKEN_NAME --symbol $TOKEN_SYMBOL --decimals 6 -vv`
-	elif [ $TOKEN_TYPE == 'erc20_demurrage_token' ]; then
+		echo giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
+	elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
+		if [ -z "$TOKEN_SYMBOL" ]; then
+			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
+			TOKEN_SYMBOL="SARAFU"
+			TOKEN_NAME="Sarafu Token"
+		elif [ -z "$TOKEN_NAME" ]; then
+			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
+			TOKEN_NAME=$TOKEN_SYMBOL
+		fi
 		>&2 echo deploying token $TOKEN_TYPE
 		if [ -z $TOKEN_SINK_ADDRESS ]; then
 			if [ ! -z $TOKEN_REDISTRIBUTION_PERIOD ]; then
 				>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
 			fi
 		fi
-		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww`
+		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww`
 	else
 		>&2 echo unknown token type $TOKEN_TYPE
 		exit 1
@@ -137,6 +148,7 @@ else
 fi
 
 mkdir -p $CIC_DATA_DIR
+>&2 echo using data dir $CIC_DATA_DIR for environment variable dump
 
 # this is consumed in downstream services to set environment variables
 cat << EOF > $CIC_DATA_DIR/.env

apps/contract-migration/seed_cic_eth.sh

@@ -48,18 +48,18 @@ EOF
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
-DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
 
 
 >&2 echo "create account for sarafu gifter"
-DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER
 
 >&2 echo "create account for approval escrow owner"
-DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER 
 
@@ -69,7 +69,7 @@ cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRA
 #cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER
 
 >&2 echo "create account for accounts index writer"
-DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"

apps/data-seeding/.dockerignore

@@ -0,0 +1,8 @@
+.git
+.cache
+.dot
+**/doc
+**/node_modules
+**/venv
+**/.venv
+

apps/data-seeding/.gitlab-ci.yml

@@ -1,21 +1,26 @@
 .data_seeding_variables:
     variables:
         APP_NAME: data-seeding
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
-
-.data_seeding_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-data-seeding:
     extends:
-        - .data_seeding_changes_target
         - .py_build_merge_request
         - .data_seeding_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/data-seeding/**/*
+      when: always
 
 build-push-data-seeding:
     extends:
         - .py_build_push
         - .data_seeding_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/data-seeding/**/*
+      when: always
 

apps/data-seeding/README.md

@@ -167,7 +167,7 @@ If you have previously run the `cic_ussd` import incompletely, it could be a goo
 
 Then, in sequence, run in first terminal:
 
-`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
+`python cic_ussd/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
 
 In second terminal:
 

apps/data-seeding/cic_eth/import_balance.py

@@ -11,7 +11,6 @@ import csv
 import json
 
 # external imports
-import eth_abi
 import confini
 from hexathon import (
         strip_0x,
@@ -29,7 +28,10 @@ from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
 from chainlib.jsonrpc import JSONRPCRequest
-from chainlib.eth.error import EthException
+from chainlib.eth.error import (
+        EthException,
+        RequestMismatchException,
+        )
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
@@ -37,6 +39,9 @@ from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
 from cic_base.eth.syncer import chain_interface
+from eth_accounts_index import AccountsIndex
+from eth_contract_registry import Registry
+from eth_token_index import TokenUniqueSymbolIndex
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -131,58 +136,52 @@ class Handler:
             logg.debug('no payload, skipping {}'.format(tx))
             return
 
-        if tx.payload[:8] == self.account_index_add_signature:
-            recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
-            #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
-            user_file = 'new/{}/{}/{}.json'.format(
-                    recipient[2:4].upper(),
-                    recipient[4:6].upper(),
-                    recipient[2:].upper(),
-                    )
-            filepath = os.path.join(self.user_dir, user_file)
-            o = None
-            try:
-                f = open(filepath, 'r')
-                o = json.load(f)
-                f.close()
-            except FileNotFoundError:
-                logg.error('no import record of address {}'.format(recipient))
-                return
-            u = Person.deserialize(o)
-            original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
-            try:
-                balance = self.balances[original_address]
-            except KeyError as e:
-                logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
-                return
-
-            # TODO: store token object in handler ,get decimals from there
-            multiplier = 10**6
-            balance_full = balance * multiplier
-            logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
-
-            (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
-            logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
-            r = conn.do(o)
-
-            tx_path = os.path.join(
-                    user_dir,
-                    'txs',
-                    strip_0x(tx_hash_hex),
-                    )
-            f = open(tx_path, 'w')
-            f.write(strip_0x(o['params'][0]))
+        recipient = None
+        try:
+            r = AccountsIndex.parse_add_request(tx.payload)
+        except RequestMismatchException:
+            return
+        recipient = r[0]
+        
+        user_file = 'new/{}/{}/{}.json'.format(
+                recipient[2:4].upper(),
+                recipient[4:6].upper(),
+                recipient[2:].upper(),
+                )
+        filepath = os.path.join(self.user_dir, user_file)
+        o = None
+        try:
+            f = open(filepath, 'r')
+            o = json.load(f)
             f.close()
-#        except TypeError as e:
-#            logg.warning('typerror {}'.format(e))
-#            pass
-#        except IndexError as e:
-#            logg.warning('indexerror {}'.format(e))
-#            pass
-#        except EthException as e:
-#            logg.error('send error {}'.format(e).ljust(200))
-        #except KeyError as e:
-        #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
+        except FileNotFoundError:
+            logg.error('no import record of address {}'.format(recipient))
+            return
+        u = Person.deserialize(o)
+        original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
+        try:
+            balance = self.balances[original_address]
+        except KeyError as e:
+            logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
+            return
+
+        # TODO: store token object in handler ,get decimals from there
+        multiplier = 10**6
+        balance_full = balance * multiplier
+        logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
+
+        (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
+        logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
+        r = conn.do(o)
+
+        tx_path = os.path.join(
+                user_dir,
+                'txs',
+                strip_0x(tx_hash_hex),
+                )
+        f = open(tx_path, 'w')
+        f.write(strip_0x(o['params'][0]))
+        f.close()
 
 
 def progress_callback(block_number, tx_index):
@@ -198,49 +197,26 @@ def main():
     nonce_oracle = RPCNonceOracle(signer_address, conn)
 
     # Get Token registry address
-    txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
-    tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
-
-    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
-    data = add_0x(registry_addressof_method)
-    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
-    txf.set_code(tx, data)
-   
-    j = JSONRPCRequest()
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
+    registry = Registry(chain_spec)
+    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
     r = conn.do(o)
-    token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    token_index_address = registry.parse_address_of(r)
+    token_index_address = to_checksum_address(token_index_address)
     logg.info('found token index address {}'.format(token_index_address))
-
-
+    
     # Get Sarafu token address
-    tx = txf.template(signer_address, token_index_address)
-    data = add_0x(registry_addressof_method)
-    h = hashlib.new('sha256')
-    h.update(token_symbol.encode('utf-8'))
-    z = h.digest()
-    data += eth_abi.encode_single('bytes32', z).hex()
-    txf.set_code(tx, data)
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
+    token_index = TokenUniqueSymbolIndex(chain_spec)
+    o = token_index.address_of(token_index_address, token_symbol)
     r = conn.do(o)
+    token_address = token_index.parse_address_of(r)
     try:
-        sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+        token_address = to_checksum_address(token_address)
     except ValueError as e:
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
-
-    if sarafu_token_address == ZERO_ADDRESS:
-        raise KeyError('token address for symbol {} is zero'.format(token_symbol))
-
-    logg.info('found token address {}'.format(sarafu_token_address))
+    logg.info('found token address {}'.format(token_address))
+    
+    sys.exit(0)
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -248,22 +224,6 @@ def main():
         o = block_latest()
         r = conn.do(o)
         block_offset = int(strip_0x(r), 16) + 1
-#
-#    addresses = {}
-#    f = open('{}/addresses.csv'.format(user_dir, 'r'))
-#    while True:
-#        l = f.readline()
-#        if l == None:
-#            break
-#        r = l.split(',')
-#        try:
-#            k = r[0]
-#            v = r[1].rstrip()
-#            addresses[k] = v
-#            sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
-#        except IndexError as e:
-#            break
-#    f.close()
 
     # TODO get decimals from token
     balances = {}