Reinstate skipped admin noncritical tests

Apply config changes to contract migration

See merge request grassrootseconomics/cic-internal-integration!229

.gitignore

@@ -13,3 +13,4 @@ build/
 **/coverage
 **/.venv
 .idea
+**/.vim

apps/cic-cache/.coveragerc

@@ -2,4 +2,5 @@
 omit = 
 	.venv/*
 	scripts/*
-	cic_cache/db/postgres/*
+	cic_cache/db/migrations/*
+	cic_cache/version.py

apps/cic-cache/.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

apps/cic-cache/.gitlab-ci.yml

@@ -1,22 +1,52 @@
 .cic_cache_variables:
     variables:
         APP_NAME: cic-cache
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_cache_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-cache:
     extends:
-        - .cic_cache_changes_target
         - .py_build_merge_request
         - .cic_cache_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
+
+test-mr-cic-cache:
+    stage: test
+    extends:
+        - .cic_cache_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-cache"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
 
 build-push-cic-cache:
     extends:
         - .py_build_push
         - .cic_cache_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
         
 

apps/cic-cache/cic_cache/cache.py

@@ -15,6 +15,8 @@ from cic_cache.db.list import (
 logg = logging.getLogger()
 
 
+DEFAULT_FILTER_SIZE = 8192 * 8
+
 class Cache:
 
     def __init__(self, session):
@@ -25,7 +27,7 @@ class BloomCache(Cache):
 
     @staticmethod
     def __get_filter_size(n):
-        n = 8192 * 8
+        n = DEFAULT_FILTER_SIZE
         logg.warning('filter size hardcoded to {}'.format(n))
         return n
 

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,3 +100,4 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -4,6 +4,9 @@ import json
 import re
 import base64
 
+# external imports
+from hexathon import add_0x
+
 # local imports
 from cic_cache.cache import (
         BloomCache,
@@ -11,10 +14,11 @@ from cic_cache.cache import (
     )
 
 logg = logging.getLogger(__name__)
+#logg = logging.getLogger()
 
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
+re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
 
 DEFAULT_LIMIT = 100
 
@@ -26,13 +30,13 @@ def process_transactions_account_bloom(session, env):
 
     address = r[1]
     if r[2] == None:
-        address = '0x' + address
-    offset = DEFAULT_LIMIT
+        address = add_0x(address)
+    offset = 0
     if r.lastindex > 2:
-        offset = r[3]
-    limit = 0
-    if r.lastindex > 3:
-        limit = r[4]
+        offset = r[4]
+    limit = DEFAULT_LIMIT
+    if r.lastindex > 4:
+        limit = r[6]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)

apps/cic-cache/docker/Dockerfile

@@ -1,52 +1,38 @@
-FROM python:3.8.6-slim-buster
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
 
-WORKDIR /usr/src/cic-cache
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
+COPY . . 
 
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-# Copy shared requirements from top of mono-repo
-RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY cic-cache/requirements.txt ./
-COPY cic-cache/setup.cfg \
-	 cic-cache/setup.py \
- 	 ./
-COPY cic-cache/cic_cache/ ./cic_cache/
-COPY cic-cache/scripts/ ./scripts/
-COPY cic-cache/test_requirements.txt ./
-RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-RUN pip install $pip_extra_index_url_flag .
-RUN pip install .[server]
-
-COPY cic-cache/tests/ ./tests/
-#COPY db/ cic-cache/db
-#RUN apk add postgresql-client
+RUN python setup.py install
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-cache/config/ /usr/local/etc/cic-cache/
+COPY config/ /usr/local/etc/cic-cache/
 
 # for db migrations
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
-COPY cic-cache/docker/db.sh ./db.sh
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server
 # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []

apps/cic-cache/docker/Dockerfile_ci

@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . . 
+
+RUN python setup.py install
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-cache/
+
+# for db migrations
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
+RUN chmod 755 ./*.sh
+# Tracker
+# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
+# Server
+# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []

apps/cic-cache/requirements.txt

@@ -1,13 +1,13 @@
-cic-base==0.1.3a3+build.984b5cff
+cic-base~=0.2.0a4
 alembic==1.4.2
-confini~=0.3.6rc3
+confini>=0.3.6rc3,<0.5.0
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.6a1
+cic-eth-registry~=0.5.6a2
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a1
+chainsyncer[sql]~=0.0.3a5
+erc20-faucet~=0.2.2a2

apps/cic-cache/tests/conftest.py

@@ -5,9 +5,12 @@ import datetime
 
 # external imports
 import pytest
+import moolb
 
 # local imports
 from cic_cache import db
+from cic_cache import BloomCache
+from cic_cache.cache import DEFAULT_FILTER_SIZE
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -101,3 +104,7 @@ def tag_txs(
 
     db.tag_transaction(init_database, txs[1], 'taag', domain='test')
 
+
+@pytest.fixture(scope='session')
+def zero_filter():
+    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/test_erc20.py

@@ -10,6 +10,7 @@ from sqlalchemy import text
 from chainlib.eth.tx import Tx
 from chainlib.eth.block import Block
 from chainlib.chain import ChainSpec
+from chainlib.eth.error import RequestMismatchException
 from hexathon import (
         strip_0x,
         add_0x,
@@ -18,10 +19,21 @@ from hexathon import (
 # local imports
 from cic_cache.db import add_tag
 from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
+from cic_cache.runnable.daemons.filters.base import TagSyncFilter
 
 logg = logging.getLogger()
 
 
+def test_base_filter_str(
+        init_database,
+        ):
+        f = TagSyncFilter('foo')
+        assert 'foo' == str(f)
+        f = TagSyncFilter('foo', domain='bar')
+        assert 'bar.foo' == str(f)
+
+
+
 def test_erc20_filter(
         eth_rpc,
         foo_token,
@@ -67,3 +79,95 @@ def test_erc20_filter(
     s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
     r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
     assert r[0] == tx.hash
+
+
+def test_erc20_filter_nocontract(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    # incomplete args
+    data = 'a9059cbb'
+    data += strip_0x(list_actors['alice'])
+    data += '1000'.ljust(64, '0')
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': os.urandom(20).hex(),
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+
+@pytest.mark.parametrize(
+        'contract_method,contract_input,expected_exception',
+        [
+            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
+            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
+            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
+            ],
+        )
+def test_erc20_filter_bogus(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        contract_method,
+        contract_input,
+        expected_exception,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    # incomplete args
+    data = contract_method
+    data += contract_input 
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': foo_token,
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/test_query.py

@@ -0,0 +1,230 @@
+# standard imports
+import logging
+import json
+import base64
+import copy
+import re
+
+# external imports
+import pytest
+from hexathon import strip_0x
+
+# local imports
+from cic_cache.runnable.daemons.query import *
+
+logg = logging.getLogger()
+
+
+
+@pytest.mark.parametrize(
+        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
+        [
+            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
+            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
+            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
+            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
+            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
+            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
+            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
+            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
+            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
+            ],
+        )
+def test_query_regex(
+        list_actors,
+        query_path_prefix,
+        query_role,
+        query_address_index,
+        query_offset,
+        query_offset_index,
+        query_limit,
+        query_limit_index,
+        match_re,
+        ):
+
+        paths = []
+        path = query_path_prefix
+        query_address = None
+        if query_role != None:
+            query_address = strip_0x(list_actors[query_role])
+            paths.append(path + '0x' + query_address)
+            paths.append(path + query_address)
+        if query_offset != None:
+            if query_limit != None:
+                for i in range(len(paths)-1):
+                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
+            else:
+                for i in range(len(paths)-1):
+                    paths[i] += '/' + str(query_offset)
+
+        for i in range(len(paths)):
+            paths.append(paths[i] + '/')
+
+        for p in paths:
+            logg.debug('testing path {} against {}'.format(p, match_re))
+            m = re.match(match_re, p)
+            l = len(m.groups())
+            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
+            for i in range(l+1):
+                logg.debug('group {} {}'.format(i, m[i]))
+            if m.lastindex >= query_offset_index:
+                assert query_offset == int(m[query_offset_index + 1])
+            if m.lastindex >= query_limit_index:
+                assert query_limit == int(m[query_limit_index + 1])
+            if query_address_index != None:
+                match_address = strip_0x(m[query_address_index + 1])
+                assert query_address == match_address
+
+
+
+@pytest.mark.parametrize(
+        'role_name, query_offset, query_limit, query_match',
+        [ 
+            ('alice', None, None, [(420000, 13), (419999, 42)]),
+            ('alice', None, 1, [(420000, 13)]),
+            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
+            ('alice', 2, None, []), # 420000 == list_defaults['block']
+            ],
+        )
+def test_query_process_txs_account(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        role_name,
+        query_offset,
+        query_limit,
+        query_match,
+        ):
+
+    actor = None
+    try:
+        actor = list_actors[role_name]
+    except KeyError:
+        actor = os.urandom(20).hex()
+    path_info = '/tx/user/0x' + strip_0x(actor)
+    if query_offset != None:
+        path_info += '/' + str(query_offset)
+    if query_limit != None:
+        if query_offset == None:
+            path_info += '/0'
+        path_info += '/' + str(query_limit)
+    env = {
+            'PATH_INFO': path_info,
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_account_bloom(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
+    zero_filter_data = zero_filter.to_bytes()
+    if len(query_match) == 0:
+        assert block_filter_data == zero_filter_data
+        return
+
+    assert block_filter_data != zero_filter_data
+    block_filter = copy.copy(zero_filter)
+    block_filter.merge(block_filter_data)
+    block_filter_data = block_filter.to_bytes()
+    assert block_filter_data != zero_filter_data
+
+    for (block, tx) in query_match:
+        block = block.to_bytes(4, byteorder='big')
+        assert block_filter.check(block)
+
+
+@pytest.mark.parametrize(
+        'query_offset, query_limit, query_match',
+        [ 
+            (None, 2, [(420000, 13), (419999, 42)]),
+            (0, 1, [(420000, 13)]),
+            (1, 1, [(419999, 42)]),
+            (2, 0, []),
+            ],
+        )
+def test_query_process_txs_bloom(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        query_offset,
+        query_limit,
+        query_match,
+        ):
+
+    path_info = '/tx'
+    if query_offset != None:
+        path_info += '/' + str(query_offset)
+    if query_limit != None:
+        if query_offset == None:
+            path_info += '/0'
+        path_info += '/' + str(query_limit)
+    env = {
+            'PATH_INFO': path_info,
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_all_bloom(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
+    zero_filter_data = zero_filter.to_bytes()
+    if len(query_match) == 0:
+        assert block_filter_data == zero_filter_data
+        return
+
+    assert block_filter_data != zero_filter_data
+    block_filter = copy.copy(zero_filter)
+    block_filter.merge(block_filter_data)
+    block_filter_data = block_filter.to_bytes()
+    assert block_filter_data != zero_filter_data
+
+    for (block, tx) in query_match:
+        block = block.to_bytes(4, byteorder='big')
+        assert block_filter.check(block)
+
+
+@pytest.mark.parametrize(
+        'query_block_start, query_block_end, query_match_count',
+        [ 
+            (None, 42, 0),
+            (420000, 420001, 1),
+            (419999, 419999, 1), # matches are inclusive
+            (419999, 420000, 2),
+            (419999, 420001, 2),
+            ],
+        )
+def test_query_process_txs_data(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        query_block_start,
+        query_block_end,
+        query_match_count,
+        ):
+
+    path_info = '/txa'
+    if query_block_start != None:
+        path_info += '/' + str(query_block_start)
+    if query_block_end != None:
+        if query_block_start == None:
+            path_info += '/0'
+        path_info += '/' + str(query_block_end)
+    env = {
+            'PATH_INFO': path_info,
+            'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_all_data(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    assert len(o['data']) == query_match_count

apps/cic-eth/.dockerignore

@@ -0,0 +1,6 @@
+.git
+.cache
+.dot
+**/doc
+**/.venv
+**/venv

apps/cic-eth/.gitlab-ci.yml

@@ -1,31 +1,52 @@
 .cic_eth_variables:
     variables:
         APP_NAME: cic-eth
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_eth_mr_changes_target:
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - $CONTEXT/$APP_NAME/**/*
-      when: always
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-eth:
     extends:
         - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-        - .py_build_target_test
+        - .py_build_target_dev
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-eth/**/*
+      when: always
 
 test-mr-cic-eth:
+    stage: test
     extends:
         - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-    stage: test
-    image: $IMAGE_TAG_BASE
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
     script:
         - cd apps/$APP_NAME/
-        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r admin_requirements.txt
+          -r services_requirements.txt
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-eth"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-eth/**/*
+      when: always
 
 build-push-cic-eth:
     extends:
         - .py_build_push
+        - .cic_eth_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-eth/**/*
+      when: always

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry~=0.5.6a1
+cic-eth-registry>=0.5.6a2,<0.6.0
 hexathon~=0.0.1a7
-chainqueue~=0.0.2b5
-eth-erc20==0.0.10a2
+chainqueue>=0.0.3a1,<0.1.0
+eth-erc20>=0.0.10a3,<0.1.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -6,6 +6,11 @@ import logging
 import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
+from hexathon import (
+        add_0x,
+        strip_0x,
+        uniform as hex_uniform,
+        )
 
 # local imports
 from cic_eth.db.enum import LockEnum
@@ -19,6 +24,12 @@ from cic_eth.error import LockedError
 celery_app = celery.current_app
 logg = logging.getLogger()
 
+
+def normalize_address(a):
+    if a == None:
+        return None
+    return add_0x(hex_uniform(strip_0x(a)))
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
@@ -32,6 +43,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
+    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -53,6 +65,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
+    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -72,6 +85,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -89,6 +103,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -106,6 +121,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -123,6 +139,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -131,6 +148,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
+    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))

apps/cic-eth/cic_eth/admin/nonce.py

@@ -14,7 +14,11 @@ from chainqueue.sql.query import get_tx
 from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        uniform as hex_uniform,
+        )
 from potaahto.symbols import snake_and_camel
 
 # local imports
@@ -69,15 +73,17 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
+    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
     q = session.query(Otx)
     q = q.join(TxCache)
-    q = q.filter(TxCache.sender==address)
+    q = q.filter(TxCache.sender==query_address)
     q = q.filter(Otx.nonce>=nonce+delta)
     q = q.order_by(Otx.nonce.asc())
     otxs = q.all()
 
     tx_hashes = []
     txs = []
+    gas_total = 0
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
@@ -89,8 +95,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         tx_new['gas_price'] += 1
         tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
+        gas_total += tx_new['gas_price'] * tx_new['gas']
 
         logg.debug('tx_new {}'.format(tx_new))
+        logg.debug('gas running total {}'.format(gas_total))
 
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
@@ -122,8 +130,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     s = create_check_gas_task(
          txs, 
          chain_spec,
-         tx_new['from'],
-         gas=tx_new['gas'],
+         #tx_new['from'],
+         address,
+         #gas=tx_new['gas'],
+         gas=gas_total,
          tx_hashes_hex=tx_hashes, 
          queue=queue,
         )
@@ -132,7 +142,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_send',
         [
             chain_spec.asdict(),
-            tx_new['from'],
+            address,
+            #tx_new['from'],
             ],
         queue=queue,
         )
@@ -140,7 +151,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_queue',
         [
             chain_spec.asdict(),
-            tx_new['from'],
+            address,
+            #tx_new['from'],
             ],
         queue=queue,
         )

apps/cic-eth/cic_eth/api/admin.py

@@ -21,6 +21,7 @@ from chainlib.hash import keccak256_hex_to_hex
 from hexathon import (
         strip_0x,
         add_0x,
+        uniform as hex_uniform,
         )
 from chainlib.eth.gas import balance
 from chainqueue.db.enum import (
@@ -307,6 +308,8 @@ class AdminApi:
         :param address: Ethereum address to return transactions for
         :type address: str, 0x-hex
         """
+
+        address = add_0x(hex_uniform(strip_0x(address)))
         last_nonce = -1
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',

apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py

@@ -8,7 +8,8 @@ Create Date: 2021-04-02 18:30:55.398388
 from alembic import op
 import sqlalchemy as sa
 
-from chainqueue.db.migrations.sqlalchemy import (
+#from chainqueue.db.migrations.sqlalchemy import (
+from chainqueue.db.migrations.default.export import (
         chainqueue_upgrade,
         chainqueue_downgrade,
         )

apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py

@@ -8,7 +8,8 @@ Create Date: 2021-04-02 18:36:44.459603
 from alembic import op
 import sqlalchemy as sa
 
-from chainsyncer.db.migrations.sqlalchemy import (
+#from chainsyncer.db.migrations.sqlalchemy import (
+from chainsyncer.db.migrations.default.export import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-eth/cic_eth/db/models/base.py

@@ -126,3 +126,4 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-eth/cic_eth/eth/account.py

@@ -23,7 +23,7 @@ from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
 from eth_accounts_index import AccountsIndex 
 from sarafu_faucet import MinterFaucet
-from chainqueue.db.models.tx import TxCache
+from chainqueue.sql.tx import cache_tx_dict
 
 # local import
 from cic_eth_registry import CICRegistry
@@ -300,20 +300,17 @@ def cache_gift_data(
 
     session = self.create_session()
 
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        ZERO_ADDRESS,
-        ZERO_ADDRESS,
-        0,
-        0,
-        session=session,
-            )
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS,
+            'from_value': 0,
+            'to_value': 0,
+            }
 
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -342,18 +339,15 @@ def cache_account_data(
     tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        ZERO_ADDRESS,
-        ZERO_ADDRESS,
-        0,
-        0,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS,
+            'from_value': 0,
+            'to_value': 0,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)

apps/cic-eth/cic_eth/eth/bancor.py.bak

@@ -1,385 +0,0 @@
-# standard imports
-import os
-import logging
-
-# third-party imports
-import celery
-import web3
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db import SessionBase
-from cic_eth.db.models.convert import TxConvertTransfer
-from cic_eth.db.models.otx import Otx
-from cic_eth.db.models.tx import TxCache
-from cic_eth.eth.task import sign_and_register_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.eth.token import TokenTxFactory
-from cic_eth.eth.factory import TxFactory
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.rpc import RpcClient
-
-celery_app = celery.current_app 
-#logg = celery_app.log.get_default_logger()
-logg = logging.getLogger()
-
-contract_function_signatures = {
-        'convert': 'f3898a97',
-        'convert2': '569706eb',
-        }
-
-
-class BancorTxFactory(TxFactory):
-
-    """Factory for creating Bancor network transactions.
-    """
-    def convert(
-            self,
-            source_token_address,
-            destination_token_address,
-            reserve_address,
-            source_amount,
-            minimum_return,
-            chain_spec,
-            fee_beneficiary='0x0000000000000000000000000000000000000000',
-            fee_ppm=0,
-            ):
-        """Create a BancorNetwork "convert" transaction.
-
-        :param source_token_address: ERC20 contract address for token to convert from
-        :type source_token_address: str, 0x-hex
-        :param destination_token_address: ERC20 contract address for token to convert to
-        :type destination_token_address: str, 0x-hex
-        :param reserve_address: ERC20 contract address of Common reserve token
-        :type reserve_address: str, 0x-hex
-        :param source_amount: Amount of source tokens to convert
-        :type source_amount: int
-        :param minimum_return: Minimum amount of destination tokens to accept as result for conversion
-        :type source_amount: int
-        :return: Unsigned "convert" transaction in standard Ethereum format
-        :rtype: dict
-        """
-        network_contract = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
-        network_gas = network_contract.gas('convert')
-        tx_convert_buildable = network_contract.contract.functions.convert2(
-            [
-                source_token_address,
-                source_token_address,
-                reserve_address,
-                destination_token_address,
-                destination_token_address,
-            ],
-            source_amount,
-            minimum_return,
-            fee_beneficiary,
-            fee_ppm,
-            )
-        tx_convert = tx_convert_buildable.buildTransaction({
-                'from': self.address,
-                'gas': network_gas,
-                'gasPrice': self.gas_price,
-                'chainId': chain_spec.chain_id(),
-                'nonce': self.next_nonce(),
-                })
-        return tx_convert
-
-
-def unpack_convert(data):
-    f = data[2:10]
-    if f != contract_function_signatures['convert2']:
-        raise ValueError('Invalid convert data ({})'.format(f))
-
-    d = data[10:]
-    path = d[384:]
-    source = path[64-40:64]
-    destination = path[-40:]
-
-    amount = int(d[64:128], 16)
-    min_return = int(d[128:192], 16)
-    fee_recipient = d[192:256]
-    fee = int(d[256:320], 16)
-    return {
-        'amount': amount,
-        'min_return': min_return,
-        'source_token': web3.Web3.toChecksumAddress('0x' + source),
-        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
-        'fee_recipient': fee_recipient,
-        'fee': fee,
-        }
-
-
-
-# Kept for historical reference, it unpacks a convert call without fee parameters
-#def _unpack_convert_mint(data):
-#    f = data[2:10]
-#    if f != contract_function_signatures['convert2']:
-#        raise ValueError('Invalid convert data ({})'.format(f))
-#
-#    d = data[10:]
-#    path = d[256:]
-#    source = path[64-40:64]
-#    destination = path[-40:]
-#
-#    amount = int(d[64:128], 16)
-#    min_return = int(d[128:192], 16)
-#    return {
-#        'amount': amount,
-#        'min_return': min_return,
-#        'source_token': web3.Web3.toChecksumAddress('0x' + source),
-#        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
-#        }
-
-
-@celery_app.task(bind=True)
-def convert_with_default_reserve(self, tokens, from_address, source_amount, minimum_return, to_address, chain_str):
-    """Performs a conversion between two liquid tokens using Bancor network.
-
-    :param tokens: Token pair, source and destination respectively
-    :type tokens: list of str, 0x-hex
-    :param from_address: Ethereum address of sender
-    :type from_address: str, 0x-hex
-    :param source_amount: Amount of source tokens to convert
-    :type source_amount: int
-    :param minimum_return: Minimum about of destination tokens to receive
-    :type minimum_return: int
-    """
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    queue = self.request.delivery_info['routing_key']
-
-    c = RpcClient(chain_spec, holder_address=from_address)
-
-    cr = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
-    source_token = CICRegistry.get_address(chain_spec, tokens[0]['address'])
-    reserve_address = CICRegistry.get_contract(chain_spec, 'BNTToken', 'ERC20').address()
-
-    tx_factory = TokenTxFactory(from_address, c)
-   
-    tx_approve_zero = tx_factory.approve(source_token.address(), cr.address(), 0, chain_spec)
-    (tx_approve_zero_hash_hex, tx_approve_zero_signed_hex) = sign_and_register_tx(tx_approve_zero, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
-
-    tx_approve = tx_factory.approve(source_token.address(), cr.address(), source_amount, chain_spec)
-    (tx_approve_hash_hex, tx_approve_signed_hex) = sign_and_register_tx(tx_approve, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
-
-    tx_factory = BancorTxFactory(from_address, c)
-    tx_convert = tx_factory.convert(
-            tokens[0]['address'],
-            tokens[1]['address'],
-            reserve_address,
-            source_amount,
-            minimum_return,
-            chain_spec,
-            )
-    (tx_convert_hash_hex, tx_convert_signed_hex) = sign_and_register_tx(tx_convert, chain_str, queue, 'cic_eth.eth.bancor.otx_cache_convert')
-
-    # TODO: consider moving save recipient to async task / chain it before the tx send
-    if to_address != None:
-        save_convert_recipient(tx_convert_hash_hex, to_address, chain_str)
-
-    s = create_check_gas_and_send_task(
-            [tx_approve_zero_signed_hex, tx_approve_signed_hex, tx_convert_signed_hex],
-            chain_str,
-            from_address,
-            tx_approve_zero['gasPrice'] * tx_approve_zero['gas'],
-            tx_hashes_hex=[tx_approve_hash_hex],
-            queue=queue,
-            )
-    s.apply_async()
-    return tx_convert_hash_hex
-
-
-#@celery_app.task()
-#def process_approval(tx_hash_hex):
-#    t = session.query(TxConvertTransfer).query(TxConvertTransfer.approve_tx_hash==tx_hash_hex).first()
-#    c = session.query(Otx).query(Otx.tx_hash==t.convert_tx_hash)
-#    gas_limit = 8000000
-#    gas_price = GasOracle.gas_price()
-#
-#    # TODO: use celery group instead
-#    s_queue = celery.signature(
-#            'cic_eth.queue.tx.create',
-#            [
-#                nonce,
-#                c['address'], # TODO: check that this is in fact sender address
-#                c['tx_hash'],
-#                c['signed_tx'],
-#                ]
-#            )
-#    s_queue.apply_async()
-#
-#    s_check_gas = celery.signature(
-#            'cic_eth.eth.gas.check_gas',
-#            [
-#                c['address'],
-#                [c['signed_tx']],
-#                gas_limit * gas_price,
-#                ]
-#            )
-#    s_send = celery.signature(
-#            'cic_eth.eth.tx.send',
-#            [],
-#            )
-#
-#    s_set_sent = celery.signature(
-#            'cic_eth.queue.state.set_sent',
-#            [False],
-#            )
-#    s_send.link(s_set_sent)
-#    s_check_gas.link(s_send)
-#    s_check_gas.apply_async()
-#    return tx_hash_hex
-
-
-
-@celery_app.task()
-def save_convert_recipient(convert_hash, recipient_address, chain_str):
-    """Registers the recipient target for a convert-and-transfer operation.
-
-    :param convert_hash: Transaction hash of convert operation
-    :type convert_hash: str, 0x-hex
-    :param recipient_address: Address of consequtive transfer recipient
-    :type recipient_address: str, 0x-hex
-    """
-    session = SessionBase.create_session()
-    t = TxConvertTransfer(convert_hash, recipient_address, chain_str)
-    session.add(t)
-    session.commit()
-    session.close()
-
-
-@celery_app.task()
-def save_convert_transfer(convert_hash, transfer_hash):
-    """Registers that the transfer part of a convert-and-transfer operation has been executed.
-
-    :param convert_hash: Transaction hash of convert operation
-    :type convert_hash: str, 0x-hex
-    :param convert_hash: Transaction hash of transfer operation
-    :type convert_hash: str, 0x-hex
-    :returns: transfer_hash,
-    :rtype: list, single str, 0x-hex
-    """
-    session = SessionBase.create_session()
-    t = TxConvertTransfer.get(convert_hash)
-    t.transfer(transfer_hash)
-    session.add(t)
-    session.commit()
-    session.close()
-    return [transfer_hash]
-
-
-# TODO: seems unused, consider removing
-@celery_app.task()
-def resolve_converters_by_tokens(tokens, chain_str):
-    """Return converters for a list of tokens.
-
-    :param tokens: Token addresses to look up
-    :type tokens: list of str, 0x-hex
-    :return: Addresses of matching converters
-    :rtype: list of str, 0x-hex
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    for t in tokens:
-        c = CICRegistry.get_contract(chain_spec, 'ConverterRegistry')
-        fn = c.function('getConvertersByAnchors')
-        try:
-            converters = fn([t['address']]).call()
-        except Exception as e:
-            raise e
-        t['converters'] = converters
-
-    return tokens
-
-
-@celery_app.task(bind=True)
-def transfer_converted(self, tokens, holder_address, receiver_address, value, tx_convert_hash_hex, chain_str):
-    """Execute the ERC20 transfer of a convert-and-transfer operation.
-
-    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
-
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param holder_address: Token receiver address
-    :type holder_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
-    :return: Transaction hash
-    :rtype: str, 0x-hex
-    """
-    # we only allow one token, one transfer
-    if len(tokens) != 1:
-        raise TokenCountError
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    queue = self.request.delivery_info['routing_key']
-
-    c = RpcClient(chain_spec, holder_address=holder_address)
-
-    # get transaction parameters
-    gas_price = c.gas_price()
-    tx_factory = TokenTxFactory(holder_address, c)
-
-    token_address = tokens[0]['address']
-    tx_transfer = tx_factory.transfer(
-        token_address,
-        receiver_address,
-        value,
-        chain_spec,
-            )
-    (tx_transfer_hash_hex, tx_transfer_signed_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.token.otx_cache_transfer')
-
-    # send transaction
-    logg.info('transfer converted token {} from {} to {} value {} {}'.format(token_address, holder_address, receiver_address, value, tx_transfer_signed_hex))
-    s = create_check_gas_and_send_task(
-            [tx_transfer_signed_hex],
-            chain_str,
-            holder_address,
-            tx_transfer['gasPrice'] * tx_transfer['gas'],
-            None,
-            queue,
-            )
-    s_save = celery.signature(
-            'cic_eth.eth.bancor.save_convert_transfer',
-            [
-                tx_convert_hash_hex,
-                tx_transfer_hash_hex,
-                ],
-            queue=queue,
-            )
-    s_save.link(s)
-    s_save.apply_async()
-    return tx_transfer_hash_hex
-
-
-@celery_app.task()
-def otx_cache_convert(
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        chain_str,
-        ):
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = unpack_convert(tx['data'])
-    logg.debug('tx data {}'.format(tx_data))
-
-    session = TxCache.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['from'],
-        tx_data['source_token'],
-        tx_data['destination_token'],
-        tx_data['amount'],
-        tx_data['amount'],
-            )
-    session.add(tx_cache)
-    session.commit()
-    session.close()
-    return tx_hash_hex
-

apps/cic-eth/cic_eth/eth/erc20.py

@@ -13,9 +13,9 @@ from chainlib.eth.tx import (
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
-from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
+from chainqueue.sql.tx import cache_tx_dict
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -375,19 +375,16 @@ def cache_transfer_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': recipient_address,
+            'source_token': tx['to'],
+            'destination_token': tx['to'],
+            'from_value': token_value,
+            'to_value': token_value,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -417,19 +414,16 @@ def cache_transfer_from_data(
     token_value = tx_data[2]
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': recipient_address,
+            'source_token': tx['to'],
+            'destination_token': tx['to'],
+            'from_value': token_value,
+            'to_value': token_value,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -458,19 +452,16 @@ def cache_approve_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': recipient_address,
+            'source_token': tx['to'],
+            'destination_token': tx['to'],
+            'from_value': token_value,
+            'to_value': token_value,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 

apps/cic-eth/cic_eth/eth/gas.py

@@ -9,6 +9,7 @@ from chainlib.chain import ChainSpec
 from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
+from chainqueue.sql.tx import cache_tx_dict
 from chainlib.eth.gas import (
         balance,
         price,
@@ -133,20 +134,17 @@ def cache_gas_data(
 
     session = SessionBase.create_session()
 
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        ZERO_ADDRESS,
-        ZERO_ADDRESS,
-        tx['value'],
-        tx['value'],
-        session=session,
-            )
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS,
+            'from_value': tx['value'],
+            'to_value': tx['value'],
+            }
 
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -18,7 +18,6 @@ from hexathon import (
         strip_0x,
         )
 from chainqueue.db.models.tx import Otx
-from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
 from potaahto.symbols import snake_and_camel

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,9 +14,11 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
+from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
+from cic_eth_registry.error import UnknownContractError
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
@@ -114,9 +116,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
-                        token = ERC20Token(chain_spec, rpc, tx['to'])
-                        token_symbol = token.symbol
-                        token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
@@ -126,12 +125,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
                             'destination_token': tx['to'],
-                            'source_token_symbol': token_symbol,
-                            'destination_token_symbol': token_symbol,
-                            'source_token_decimals': token_decimals,
-                            'destination_token_decimals': token_decimals,
-                            'source_token_chain': chain_str,
-                            'destination_token_chain': chain_str,
                             'nonce': tx['nonce'],
                                 }
                         if times['queue'] != None:
@@ -146,8 +139,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
-@celery_app.task()
-def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
+@celery_app.task(bind=True, base=BaseTask)
+def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=True, verify_contracts=True):
     """Merges transaction data from multiple sources and sorts them in chronological order.
 
     :param tx_batches: Transaction data inputs
@@ -196,6 +189,32 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
     if newest_first:
         ks.reverse()
     for k in ks:
-        txs.append(txs_by_block[k])
+        tx = txs_by_block[k]
+        if verify_contracts:
+            try:
+                tx = verify_and_expand(tx, chain_spec, sender_address=BaseTask.call_address)
+            except UnknownContractError:
+                logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
+                continue
+        txs.append(tx)
 
     return txs
+
+
+def verify_and_expand(tx, chain_spec, sender_address=ZERO_ADDRESS):
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+
+    if tx.get('source_token_symbol') == None and tx['source_token'] != ZERO_ADDRESS:
+        r = registry.by_address(tx['source_token'], sender_address=sender_address)
+        token = ERC20Token(chain_spec, rpc, tx['source_token'])
+        tx['source_token_symbol'] = token.symbol
+        tx['source_token_decimals'] = token.decimals
+
+    if tx.get('destination_token_symbol') == None and tx['destination_token'] != ZERO_ADDRESS:
+        r = registry.by_address(tx['destination_token'], sender_address=sender_address)
+        token = ERC20Token(chain_spec, rpc, tx['destination_token'])
+        tx['destination_token_symbol'] = token.symbol
+        tx['destination_token_decimals'] = token.decimals
+
+    return tx

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -27,7 +27,7 @@ def database_engine(
         SessionBase.poolable = False
     dsn = dsn_from_config(load_config)
     #SessionBase.connect(dsn, True)
-    SessionBase.connect(dsn, debug=load_config.get('DATABASE_DEBUG') != None)
+    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
     return dsn
 
 

apps/cic-eth/cic_eth/queue/query.py

@@ -100,6 +100,7 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     q_outer = q_outer.join(Lock, isouter=True)
     q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
 
+
     if not is_alive(status):
         SessionBase.release_session(session)
         raise ValueError('not a valid non-final tx value: {}'.format(status))

apps/cic-eth/cic_eth/runnable/create.py

@@ -80,7 +80,12 @@ def main():
     t = api.create_account(register=register)
 
     ps.get_message()
-    o = ps.get_message(timeout=args.timeout)
+    try:
+        o = ps.get_message(timeout=args.timeout)
+    except TimeoutError as e:
+        sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e))
+        sys.exit(1) 
+    ps.unsubscribe()
     m = json.loads(o['data'])
     print(m['result'])
 

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -90,6 +90,7 @@ class DispatchSyncer:
 
     def __init__(self, chain_spec):
         self.chain_spec = chain_spec
+        self.session = None
 
 
     def chain(self):
@@ -100,16 +101,18 @@ class DispatchSyncer:
         c = len(txs.keys())
         logg.debug('processing {} txs {}'.format(c, list(txs.keys())))
         chain_str = str(self.chain_spec)
-        session = SessionBase.create_session()
+        self.session = SessionBase.create_session()
         for k in txs.keys():
             tx_raw = txs[k]
             tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
             tx = unpack(tx_raw_bytes, self.chain_spec)
             
             try:
-                set_reserved(self.chain_spec, tx['hash'], session=session)
+                set_reserved(self.chain_spec, tx['hash'], session=self.session)
+                self.session.commit()
             except NotLocalTxError as e:
                 logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash']))
+                self.session.rollback()
                 continue
 
             s_check = celery.signature(
@@ -132,16 +135,25 @@ class DispatchSyncer:
             s_check.link(s_send)
             t = s_check.apply_async()
             logg.info('processed {}'.format(k))
+        self.session.close()
+        self.session = None
 
 
-    def loop(self, w3, interval):
+    def loop(self, interval):
         while run:
             txs = {}
             typ = StatusBits.QUEUED
             utxs = get_upcoming_tx(self.chain_spec, typ)
             for k in utxs.keys():
                 txs[k] = utxs[k]
-            self.process(w3, txs)
+            try:
+                conn = RPCConnection.connect(self.chain_spec, 'default')
+                self.process(conn, txs)
+            except ConnectionError as e:
+                if self.session != None:
+                    self.session.close()
+                    self.session = None
+                logg.error('connection to node failed: {}'.format(e))
 
             if len(utxs) > 0:
                 time.sleep(self.yield_delay)
@@ -151,8 +163,7 @@ class DispatchSyncer:
 
 def main(): 
     syncer = DispatchSyncer(chain_spec)
-    conn = RPCConnection.connect(chain_spec, 'default')
-    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    syncer.loop(float(config.get('DISPATCHER_LOOP_INTERVAL')))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -11,6 +11,7 @@ from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
+from chainlib.eth.address import to_checksum_address
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -47,12 +48,13 @@ class GasFilter(SyncFilter):
 
             SessionBase.release_session(session)
 
+            address = to_checksum_address(r[0])
             logg.info('resuming gas-in-waiting txs for {}'.format(r[0]))
             if len(txs) > 0:
                 s = create_check_gas_task(
                         list(txs.values()),
                         self.chain_spec,
-                        r[0],
+                        address,
                         0,
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -12,20 +12,24 @@ from hexathon import (
 # local imports
 from .base import SyncFilter
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 
 
 class RegistrationFilter(SyncFilter):
 
-    def __init__(self, chain_spec, queue):
+    def __init__(self, chain_spec, contract_address, queue=None):
         self.chain_spec = chain_spec
         self.queue = queue
+        self.contract_address = contract_address
 
 
     def filter(self, conn, block, tx, db_session=None): 
-        registered_address = None
+        if self.contract_address != tx.inputs[0]:
+            logg.debug('not an account registry tx; {} != {}'.format(self.contract_address, tx.inputs[0]))
+            return None
+
         for l in tx.logs:
             event_topic_hex = l['topics'][0]
             if event_topic_hex == account_registry_add_log_hash:

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -78,6 +78,14 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
+rpc = RPCConnection.connect(chain_spec, 'default')
+registry = None
+try:
+    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+except UnknownContractError as e:
+    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+    sys.exit(1)
+logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 
 def main():
@@ -85,7 +93,6 @@ def main():
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # Connect to blockchain with chainlib
-    rpc = RPCConnection.connect(chain_spec, 'default')
 
     o = block_latest()
     r = rpc.do(o)
@@ -151,7 +158,8 @@ def main():
 
     tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
 
-    registration_filter = RegistrationFilter(chain_spec, config.get('_CELERY_QUEUE'))
+    account_registry_address = registry.by_name('AccountRegistry')
+    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE'))
 
     gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
 

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         12,
-        0,
-        'alpha.2',
+        2,
+        'alpha.3',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/test/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=
+DEBUG=0

apps/cic-eth/docker/Dockerfile

@@ -1,81 +1,71 @@
-FROM python:3.8.6-slim-buster as compile
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-WORKDIR /usr/src
-
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-#RUN python -m venv venv && . venv/bin/activate
-
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN /usr/local/bin/python -m pip install --upgrade pip
-RUN pip install semver
-
-COPY cic-eth-aux/  ./cic-eth-aux/
-WORKDIR /usr/src/cic-eth-aux/erc20-demurrage-token
-RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    	--extra-index-url $EXTRA_INDEX_URL .
-
-WORKDIR /usr/src/cic-eth
-
-COPY cic-eth/ .
-RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    	--extra-index-url $EXTRA_INDEX_URL .[services]
-
-# --- TEST IMAGE ---
-FROM python:3.8.6-slim-buster as test 
-
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-WORKDIR /usr/src/cic-eth
-
-RUN /usr/local/bin/python -m pip install --upgrade pip
-
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-# TODO we could use venv inside container to isolate the system and app deps further
-# COPY --from=compile /usr/src/cic-eth/ .
-# RUN . venv/bin/activate
+# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+# do load these all into setup.py later
+# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+#COPY cic-eth/requirements.txt .
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-
-COPY cic-eth/test_requirements.txt .
-RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    	--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
-
-COPY cic-eth . 
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#    pip install --index-url https://pypi.org/simple \
+#    --force-reinstall \
+#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+#    -r requirements.txt 
+COPY *requirements.txt ./
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt \
+    -r services_requirements.txt \
+    -r admin_requirements.txt
+    
+COPY . .
+RUN python setup.py install
 
 ENV PYTHONPATH .
 
-ENTRYPOINT ["pytest"]
-
-# --- RUNTIME ---
-FROM python:3.8.6-slim-buster as runtime 
-
-RUN apt-get update && \
-	apt install -y gnupg libpq-dev procps
-
-WORKDIR /usr/src/cic-eth
-
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-
-COPY cic-eth/docker/* ./ 
+COPY docker/entrypoints/* ./ 
 RUN chmod 755 *.sh
 
-COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-eth/config/ /usr/local/etc/cic-eth/
-COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+COPY config/ /usr/local/etc/cic-eth/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-COPY util/liveness/health.sh /usr/local/bin/health.sh
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
+ENTRYPOINT []
+
+## ------------------ PRODUCTION CONTAINER ----------------------
+#FROM python:3.8.6-slim-buster as prod 
+#
+#RUN apt-get update && \
+#	apt install -y gnupg libpq-dev procps
+#
+#WORKDIR /root
+#
+#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+#                          /usr/local/lib/python3.8/site-packages/
+#
+#COPY docker/entrypoints/* ./ 
+#RUN chmod 755 *.sh
+#
+## # ini files in config directory defines the configurable parameters for the application
+## # they can all be overridden by environment variables
+## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+#COPY config/ /usr/local/etc/cic-eth/
+#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+#COPY scripts/ scripts/ 
+#
+## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+##COPY util/liveness/health.sh /usr/local/bin/health.sh
+#
+#ENTRYPOINT []
 

apps/cic-eth/docker/Dockerfile_ci

@@ -0,0 +1,69 @@
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+WORKDIR /usr/src/cic-eth
+
+# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+# do load these all into setup.py later
+# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+#COPY cic-eth/requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#    pip install --index-url https://pypi.org/simple \
+#    --force-reinstall \
+#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+#    -r requirements.txt 
+COPY *requirements.txt . 
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt \
+    -r services_requirements.txt \
+    -r admin_requirements.txt
+    
+COPY . .
+RUN python setup.py install
+
+COPY docker/entrypoints/* ./ 
+RUN chmod 755 *.sh
+
+# # ini files in config directory defines the configurable parameters for the application
+# # they can all be overridden by environment variables
+# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-eth/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
+ENTRYPOINT []
+
+# ------------------ PRODUCTION CONTAINER ----------------------
+#FROM python:3.8.6-slim-buster as prod 
+#
+#RUN apt-get update && \
+#	apt install -y gnupg libpq-dev procps
+#
+#WORKDIR /root
+#
+#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+#                          /usr/local/lib/python3.8/site-packages/
+#
+#COPY docker/entrypoints/* ./ 
+#RUN chmod 755 *.sh
+#
+## # ini files in config directory defines the configurable parameters for the application
+## # they can all be overridden by environment variables
+## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+#COPY config/ /usr/local/etc/cic-eth/
+#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+#COPY scripts/ scripts/ 
+#
+## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+##COPY util/liveness/health.sh /usr/local/bin/health.sh
+#
+#ENTRYPOINT []
+#

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib~=0.0.5a1
+chainlib-eth>=0.0.6a1,<0.1.0
 semver==2.13.0

apps/cic-eth/services_requirements.txt

@@ -1,14 +1,15 @@
-chainsyncer[sql]~=0.0.3a3
-chainqueue~=0.0.2b5
+chainqueue>=0.0.3a2,<0.1.0
+chainsyncer[sql]>=0.0.5a1,<0.1.0
 alembic==1.4.2
-confini~=0.3.6rc4
+confini>=0.3.6rc4,<0.5.0
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index~=0.1.2a1
-eth-accounts-index~=0.0.12a1
-cic-eth-registry~=0.5.6a1
-erc20-faucet~=0.2.2a1
-sarafu-faucet~=0.0.4a1
+eth-address-index>=0.1.3a1,<0.2.0
+eth-accounts-index>=0.0.13a1,<0.1.0
+cic-eth-registry>=0.5.7a1,<0.6.0
+erc20-faucet>=0.2.3a1,<0.3.0
+erc20-transfer-authorization>=0.3.3a1,<0.4.0
+sarafu-faucet>=0.0.4a5,<0.1.0
 moolb~=0.1.1b2

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.0.10a2
+eth-erc20~=0.0.11a1

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -18,6 +18,7 @@ def test_filter_bogus(
         cic_registry,
         contract_roles,
         register_lookups,
+        account_registry,
         ):
 
     fltrs = [
@@ -26,7 +27,7 @@ def test_filter_bogus(
         TxFilter(default_chain_spec, None),
         CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
         StragglerFilter(default_chain_spec, None),
-        RegistrationFilter(default_chain_spec, queue=None),
+        RegistrationFilter(default_chain_spec, account_registry, queue=None),
         ]
       
     for fltr in fltrs:

apps/cic-eth/tests/filters/test_register_filter.py

@@ -1,3 +1,7 @@
+# standard imports
+import logging
+import os
+
 # external imports
 from eth_accounts_index.registry import AccountRegistry
 from chainlib.connection import RPCConnection
@@ -14,12 +18,17 @@ from chainlib.eth.block import (
         Block,
         )
 from erc20_faucet import Faucet
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
 from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
 
+logg = logging.getLogger()
+
 
 def test_register_filter(
         default_chain_spec,
@@ -60,7 +69,11 @@ def test_register_filter(
     tx = Tx(tx_src, block=block, rcpt=rcpt)
     tx.apply_receipt(rcpt)
 
-    fltr = RegistrationFilter(default_chain_spec, queue=None)
+    fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    assert t == None
+
+    fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
 
     t.get_leaf()

apps/cic-eth/tests/task/api/test_admin.py

@@ -290,6 +290,7 @@ def test_fix_nonce(
     txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
     ks = txs.keys()
     assert len(ks) == 2
+
     for k in ks:
         hsh = add_0x(k)
         otx = Otx.load(hsh, session=init_database)

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -184,7 +184,7 @@ def test_admin_api_account(
 
 
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=True, include_recipient=True)
     assert len(r) == 5
 
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])

apps/cic-eth/tests/task/test_task_address.py

@@ -1,4 +1,5 @@
 # external imports
+import celery
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
@@ -20,6 +21,7 @@ def test_translate(
         cic_registry,
         init_celery_tasks,
         register_lookups,
+        celery_session_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
@@ -46,6 +48,20 @@ def test_translate(
         'recipient': agent_roles['BOB'],
         'recipient_label': None,
             }
-    tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
-    assert tx['sender_label'] == 'alice'
-    assert tx['recipient_label'] == 'bob'
+    
+    #tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
+    s = celery.signature(
+            'cic_eth.ext.address.translate_tx_addresses',
+            [
+                tx,
+                [contract_roles['CONTRACT_DEPLOYER']],
+                default_chain_spec.asdict(),
+                ],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    assert r['sender_label'] == 'alice'
+    assert r['recipient_label'] == 'bob'

apps/cic-eth/tests/task/test_task_list.py

@@ -0,0 +1,92 @@
+# external imports
+import celery
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.gas import (
+        RPCGasOracle,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from eth_erc20 import ERC20
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainqueue.db.models.tx import TxCache
+from chainqueue.db.models.otx import Otx
+
+
+def test_ext_tx_collate(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        custodial_roles,
+        agent_roles,
+        foo_token,
+        bar_token,
+        register_tokens,
+        cic_registry,
+        register_lookups,
+        init_celery_tasks,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], eth_rpc)
+    gas_oracle = RPCGasOracle(eth_rpc)
+
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    transfer_value_foo = 1000
+    transfer_value_bar = 1024
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], transfer_value_foo, tx_format=TxFormat.RLP_SIGNED)
+    tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+
+    otx = Otx(
+        tx['nonce'],
+        tx_hash_hex,
+        tx_signed_raw_hex,
+        )
+    init_database.add(otx)
+    init_database.commit()
+
+    txc = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        tx['to'],
+        foo_token,
+        bar_token,
+        transfer_value_foo,
+        transfer_value_bar,
+        666,
+        13,
+        session=init_database,
+            )
+    init_database.add(txc)
+    init_database.commit()
+    
+    s = celery.signature(
+            'cic_eth.ext.tx.tx_collate',
+            [
+                {tx_hash_hex: tx_signed_raw_hex},
+                default_chain_spec.asdict(),
+                0,
+                100,
+                ],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    assert len(r) == 1
+
+    tx = r[0]
+    assert tx['source_token_symbol'] == 'FOO'
+    assert tx['source_token_decimals'] == 6
+    assert tx['destination_token_symbol'] == 'BAR'
+    assert tx['destination_token_decimals'] == 9

apps/cic-eth/tests/unit/db/test_nonce_db.py

@@ -1,6 +1,7 @@
-# third-party imports
+# extended imports
 import pytest
 import uuid
+import unittest
 
 # local imports
 from cic_eth.db.models.nonce import (
@@ -55,7 +56,7 @@ def test_nonce_reserve(
     o = q.first()
     assert o.nonce == 43
 
-    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu))
+    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu), session=init_database)
     init_database.commit()
     assert nonce == (str(uu), 42)
 

apps/cic-eth/tests/unit/queue/test_query.py

@@ -9,6 +9,11 @@ from chainlib.eth.gas import (
         Gas,
         )
 from chainlib.chain import ChainSpec
+from hexathon import (
+        add_0x,
+        strip_0x,
+        uniform as hex_uniform,
+        )
 
 # local imports
 from cic_eth.db.enum import LockEnum
@@ -34,7 +39,10 @@ def test_upcoming_with_lock(
     gas_oracle = RPCGasOracle(eth_rpc)
     c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
 
-    (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6))
+    alice_normal = add_0x(hex_uniform(strip_0x(agent_roles['ALICE'])))
+    bob_normal = add_0x(hex_uniform(strip_0x(agent_roles['BOB'])))
+
+    (tx_hash_hex, tx_rpc) = c.create(alice_normal, bob_normal, 100 * (10 ** 6))
     tx_signed_raw_hex = tx_rpc['params'][0]
 
     register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
@@ -43,12 +51,12 @@ def test_upcoming_with_lock(
     txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
     assert len(txs.keys()) == 1
 
-    Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE'])
+    Lock.set(str(default_chain_spec), LockEnum.SEND, address=alice_normal)
 
-    txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
+    txs = get_upcoming_tx(default_chain_spec, status=StatusEnum.PENDING)
     assert len(txs.keys()) == 0
 
-    (tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6))
+    (tx_hash_hex, tx_rpc) = c.create(bob_normal, alice_normal, 100 * (10 ** 6))
     tx_signed_raw_hex = tx_rpc['params'][0]
 
     register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)

apps/cic-eth/tools_requirements.txt

@@ -1,7 +1,7 @@
-crypto-dev-signer~=0.4.14b6
-chainqueue~=0.0.2b5
-confini~=0.3.6rc4
-cic-eth-registry~=0.5.6a1
+crypto-dev-signer>=0.4.14b7,<=0.4.14
+chainqueue~=0.0.2b6
+confini>=0.3.6rc4,<0.5.0
+cic-eth-registry>=0.5.7a1,<0.6.0
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1

apps/cic-meta/.dockerignore

@@ -0,0 +1,5 @@
+.git
+.cache
+.dot
+**/doc
+**/node_modules

apps/cic-meta/.gitlab-ci.yml

@@ -2,43 +2,42 @@
 .cic_meta_variables:
     variables:
         APP_NAME: cic-meta
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
-.cic_meta_changes_target:
+build-mr-cic-meta:
+    extends:
+        - .py_build_merge_request
+        - .cic_meta_variables
     rules:
-        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-#        - changes:
-#            - $CONTEXT/$APP_NAME/*
-        - when: always
-
-cic-meta-build-mr:
-  stage: build
-  extends:
-    - .cic_meta_variables
-    - .cic_meta_changes_target
-  script:
-    - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-meta/**/*
+      when: always
 
 test-mr-cic-meta:
     extends:
         - .cic_meta_variables
-        - .cic_meta_changes_target
     stage: test
-    image: $IMAGE_TAG 
+    image: $MR_IMAGE_TAG 
     script:
-        - cd /tmp/src/cic-meta
+        - cd /root
         - npm install --dev 
         - npm run test
         - npm run test:coverage
-    needs: ["cic-meta-build-mr"]
+    needs: ["build-mr-cic-meta"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-meta/**/*
+      when: always
 
 build-push-cic-meta:
     extends:
         - .py_build_push
         - .cic_meta_variables
-        
-
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-meta/**/*
+      when: always

apps/cic-meta/docker/Dockerfile

@@ -1,31 +1,33 @@
-FROM node:15.3.0-alpine3.10
+# syntax = docker/dockerfile:1.2
+#FROM node:15.3.0-alpine3.10
+FROM node:lts-alpine3.14
 
-WORKDIR /tmp/src/cic-meta
+WORKDIR /root
 
 RUN apk add --no-cache postgresql bash
 
-# required to build the cic-client-meta module
-COPY cic-meta/src/ src/
-COPY cic-meta/scripts/ scripts/
-
 # copy the dependencies
-COPY cic-meta/package.json .
-COPY cic-meta/tsconfig.json . 
-COPY cic-meta/webpack.config.js .
-     
-RUN npm install
+COPY package.json package-lock.json .
+RUN --mount=type=cache,mode=0755,target=/root/.npm \
+    npm set cache /root/.npm && \
+    npm ci
 
-COPY cic-meta/tests/ tests/
-COPY cic-meta/tests/*.asc /root/pgp/
+COPY webpack.config.js .
+COPY tsconfig.json . 
+## required to build the cic-client-meta module
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
+COPY tests/*.asc /root/pgp/
 
-# copy runtime configs
-COPY cic-meta/.config/ /usr/local/etc/cic-meta/
-
-# db migrations
-COPY cic-meta/docker/db.sh ./db.sh
+## copy runtime configs
+COPY .config/ /usr/local/etc/cic-meta/
+#
+## db migrations
+COPY docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
-
+#
 RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY cic-meta/docker/start_server.sh ./start_server.sh
+COPY docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+#FROM node:15.3.0-alpine3.10
+FROM node:lts-alpine3.14
+
+WORKDIR /root
+
+RUN apk add --no-cache postgresql bash
+
+# copy the dependencies
+COPY package.json package-lock.json .
+RUN npm set cache /root/.npm && \
+    npm ci
+
+COPY webpack.config.js .
+COPY tsconfig.json . 
+## required to build the cic-client-meta module
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
+COPY tests/*.asc /root/pgp/
+
+## copy runtime configs
+COPY .config/ /usr/local/etc/cic-meta/
+#
+## db migrations
+COPY docker/db.sh ./db.sh
+RUN chmod 755 ./db.sh
+#
+RUN alias tsc=node_modules/typescript/bin/tsc 
+COPY docker/start_server.sh ./start_server.sh
+RUN chmod 755 ./start_server.sh
+ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-notify/.gitlab-ci.yml

@@ -1,22 +1,52 @@
 .cic_notify_variables:
     variables:
         APP_NAME: cic-notify
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_notify_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-notify:
     extends:
-        - .cic_notify_changes_target
         - .py_build_merge_request
         - .cic_notify_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-notify/**/*
+      when: always
+
+test-mr-cic-notify:
+    stage: test
+    extends:
+        - .cic_notify_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-notify"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
 
 build-push-cic-notify:
     extends:
         - .py_build_push
         - .cic_notify_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-notify/**/*
+      when: always
         
 

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.7')
+version = (0, 4, 0, 'alpha.10')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/docker/Dockerfile

@@ -1,38 +1,30 @@
-FROM python:3.8.6-slim-buster
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps
+#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
 
-WORKDIR /usr/src/cic-notify
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
 
-COPY cic-notify/setup.cfg \
-	 cic-notify/setup.py \
-     ./
+COPY . . 
 
-COPY cic-notify/cic_notify/ ./cic_notify/
+RUN python setup.py install
 
-COPY cic-notify/requirements.txt \
-	 cic-notify/test_requirements.txt \
-     ./
-
-COPY cic-notify/scripts/ scripts/
+# TODO please review..can this go into requirements?
 RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
 
-COPY cic-notify/tests/ tests/
-COPY cic-notify/docker/db.sh \
-     cic-notify/docker/start_tasker.sh \
-	 /root/
-
-#RUN apk add postgresql-client
-#RUN apk add bash
+COPY docker/*.sh .
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-notify/.config/ /usr/local/etc/cic-notify/
-COPY cic-notify/cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+COPY .config/ /usr/local/etc/cic-notify/
+COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
 
-WORKDIR /root
+ENTRYPOINT []

apps/cic-notify/docker/Dockerfile_ci

@@ -0,0 +1,29 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
+
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . . 
+
+RUN python setup.py install
+
+# TODO please review..can this go into requirements?
+RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
+
+COPY docker/*.sh .
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY .config/ /usr/local/etc/cic-notify/
+COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+
+ENTRYPOINT []

apps/cic-notify/requirements.txt

@@ -1 +1,7 @@
-cic_base[full_graph]==0.1.3a3+build.984b5cff
+confini~=0.4.1a1
+africastalking==1.2.3
+SQLAlchemy==1.3.20
+alembic==1.4.2
+psycopg2==2.8.6
+celery==4.4.7
+redis==3.5.3

apps/cic-notify/test_requirements.txt

@@ -2,3 +2,4 @@ pytest~=6.0.1
 pytest-celery~=0.0.0a1
 pytest-mock~=3.3.1
 pysqlite3~=0.4.3
+pytest-cov==2.10.1

apps/cic-ussd/.config/app.ini

@@ -1,7 +1,7 @@
 [app]
 ALLOWED_IP=0.0.0.0/0
 LOCALE_FALLBACK=en
-LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
+LOCALE_PATH=var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#,*483*061#,*384*96#
@@ -11,13 +11,13 @@ SUPPORT_PHONE_NUMBER=0757628885
 REGION=KE
 
 [ussd]
-MENU_FILE=/usr/src/data/ussd_menu.json
+MENU_FILE=data/ussd_menu.json
 user = 
 pass = 
 
 [statemachine]
-STATES=/usr/src/cic-ussd/states/
-TRANSITIONS=/usr/src/cic-ussd/transitions/
+STATES=states/
+TRANSITIONS=transitions/
 
 [client]
 host =

apps/cic-ussd/.config/pgp.ini

@@ -1,5 +1,5 @@
 [pgp]
-export_dir = /usr/src/pgp/keys/
+export_dir = pgp/keys/
 keys_path = /usr/src/secrets/
 private_keys = privatekeys_meta.asc
 passphrase =

apps/cic-ussd/.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

apps/cic-ussd/.gitlab-ci.yml

@@ -1,22 +1,52 @@
 .cic_ussd_variables:
     variables:
         APP_NAME: cic-ussd
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_ussd_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-cic-ussd:
     extends:
-        - .cic_ussd_changes_target
         - .py_build_merge_request
         - .cic_ussd_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-ussd/**/*
+      when: always
+
+test-mr-cic-ussd:
+    stage: test
+    extends:
+        - .cic_ussd_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-ussd"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
 
 build-push-cic-ussd:
     extends:
         - .py_build_push
         - .cic_ussd_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-ussd/**/*
+      when: always
 
 

apps/cic-ussd/cic_ussd/db/models/base.py

@@ -127,3 +127,4 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -28,7 +28,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config = Config(args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -45,7 +45,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config = Config(args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/docker/Dockerfile

@@ -1,14 +1,5 @@
-# FROM python:3.8.5-alpine
-FROM python:3.8.6-slim-buster
-
-# set working directory
-WORKDIR /usr/src
-
-# add args for installing from self-hosted packages
-ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
-
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
 # create secrets directory
 RUN mkdir -vp pgp/keys
@@ -17,39 +8,25 @@ RUN mkdir -vp pgp/keys
 RUN mkdir -vp cic-ussd
 RUN mkdir -vp data
 
-COPY cic-ussd/setup.cfg \
-     cic-ussd/setup.py \
-     cic-ussd/
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
 
-COPY cic-ussd/requirements.txt \
-     cic-ussd/test_requirements.txt \
-     cic-ussd/
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
 
-# install requirements
-RUN cd cic-ussd && \
-    pip install -r requirements.txt $pip_extra_index_url_flag
+COPY . .
+RUN python setup.py install
 
-# copy all necessary files
-COPY cic-ussd/cic_ussd/ cic-ussd/cic_ussd/
-COPY cic-ussd/cic_ussd/db/ussd_menu.json data/
-COPY cic-ussd/scripts/ cic-ussd/scripts/
-COPY cic-ussd/states/ cic-ussd/states/
-COPY cic-ussd/transitions/ cic-ussd/transitions/
-COPY cic-ussd/var/ cic-ussd/var/
-
-COPY cic-ussd/docker/db.sh \
-     cic-ussd/docker/start_cic_user_tasker.sh \
-     cic-ussd/docker/start_cic_user_ussd_server.sh\
-     cic-ussd/docker/start_cic_user_server.sh\
-     /root/
+COPY cic_ussd/db/ussd_menu.json data/
 
+COPY docker/*.sh .
 RUN chmod +x /root/*.sh
-RUN cd cic-ussd && \
-    pip install $pip_extra_index_url_flag .
-
 
 # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY cic-ussd/.config/ /usr/local/etc/cic-ussd/
-COPY cic-ussd/cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+COPY .config/ /usr/local/etc/cic-ussd/
+COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
 
-WORKDIR /root
+ENTRYPOINT []

apps/cic-ussd/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+
+# create secrets directory
+RUN mkdir -vp pgp/keys
+
+# create application directory
+RUN mkdir -vp cic-ussd
+RUN mkdir -vp data
+
+COPY requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . .
+RUN python setup.py install
+
+COPY cic_ussd/db/ussd_menu.json data/
+
+COPY docker/*.sh .
+RUN chmod +x /root/*.sh
+
+# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
+COPY .config/ /usr/local/etc/cic-ussd/
+COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+
+ENTRYPOINT []

apps/cic-ussd/docker/start_cic_user_server.sh

@@ -4,4 +4,4 @@
 
 user_server_port=${SERVER_PORT:-9500}
 
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_cic_user_ussd_server.sh

@@ -4,4 +4,4 @@
 
 user_ussd_server_port=${SERVER_PORT:-9000}
 
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

apps/cic-ussd/requirements.txt

@@ -1,4 +1,17 @@
-cic_base[full_graph]==0.1.3a3+build.984b5cff
-cic-eth~=0.12.0a1
-cic-notify~=0.4.0a7
-cic-types~=0.1.0a11
+cic-eth~=0.12.2a3
+cic-notify~=0.4.0a10
+cic-types~=0.1.0a14
+confini~=0.4.1a1
+semver==2.13.0
+alembic==1.4.2
+SQLAlchemy==1.3.20
+psycopg2==2.8.6
+tinydb==4.2.0
+phonenumbers==8.12.12
+redis==3.5.3
+celery==4.4.7
+python-i18n==0.3.9
+pyxdg==0.27
+bcrypt==3.2.0
+uWSGI==2.0.19.1
+transitions==0.8.4

apps/cic-ussd/scripts/migrate.py

@@ -30,7 +30,7 @@ arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
 args = arg_parser.parse_args()
 
 
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config = Config(args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug(f'config:\n{config}')

apps/contract-migration/.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

apps/contract-migration/.gitlab-ci.yml

@@ -1,20 +1,25 @@
 .contract_migration_variables:
     variables:
         APP_NAME: contract-migration
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
-
-.contract_migration_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-contract-migration:
     extends:
-        - .contract_migration_changes_target
         - .py_build_merge_request
         - .contract_migration_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/contract-migration/**/*
+      when: always
 
 build-push-contract-migration:
     extends:
         - .py_build_push
         - .contract_migration_variables
+    rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/contract-migration/**/*
+      when: always

apps/contract-migration/config_template/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir = /usr/local/share/cic/bancor

apps/contract-migration/config_template/chain.ini

@@ -0,0 +1,2 @@
+[chain]
+spec = evm:ethereum:1

apps/contract-migration/config_template/redis.ini

@@ -0,0 +1,4 @@
+[redis]
+host = localhost
+port = 63379
+db = 0

apps/contract-migration/config_template/rpc.ini

@@ -0,0 +1,5 @@
+[rpc]
+http_provider = http://localhost:8545
+http_authentication = 
+http_username =
+http_password =

apps/contract-migration/config_template/token.ini

@@ -0,0 +1,8 @@
+[token]
+name =
+symbol =
+decimals =
+demurrage_level =
+redistribution_period =
+sink_address =
+supply_limit = 0

apps/contract-migration/config_template/wallet.ini

@@ -0,0 +1,3 @@
+[wallet]
+key_file = 
+passphrase = 

apps/contract-migration/docker/Dockerfile

@@ -14,18 +14,19 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C20
 RUN mkdir -vp /usr/local/etc/cic
 
 ENV CONFINI_DIR /usr/local/etc/cic/
-RUN mkdir -vp $CONFINI_DIR
+#RUN mkdir -vp $CONFINI_DIR
 
-ARG cic_config_commit=0abe0867f18077907c7023bf0ef5e466a3984dd8
-ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
-RUN echo Install confini schema files && \
-	git clone --depth 1 $cic_config_url cic-config && \
-	cd cic-config && \
-	git fetch --depth 1 origin $cic_config_commit && \
-	git checkout $cic_config_commit && \	
-	cp -v *.ini $CONFINI_DIR
+#ARG cic_config_commit=24287fb253196820f23ff8a7177b122f2cd99a11
+#ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
+#RUN echo Install confini schema files && \
+#	git clone --depth 1 $cic_config_url cic-config && \
+#	cd cic-config && \
+#	git fetch --depth 1 origin $cic_config_commit && \
+#	git checkout $cic_config_commit && \	
+#	cp -v *.ini $CONFINI_DIR
 
-COPY contract-migration/requirements.txt . 
+COPY config_template/ /usr/local/etc/cic/
+COPY requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@@ -38,5 +39,5 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
 
-COPY contract-migration/ .
+COPY . .
 RUN chmod +x *.sh

apps/contract-migration/docker/Dockerfile_ci

@@ -3,12 +3,11 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55
 
 WORKDIR /root
 
-# solc install which we needed for bancor. Leaving as an artfact of HOW to do it. 
-#RUN touch /etc/apt/sources.list.d/ethereum.list
-#RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
-#RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
-#RUN cat /etc/apt/sources.list.d/ethereum.list
-#RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
+RUN touch /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
+RUN cat /etc/apt/sources.list.d/ethereum.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
 
 #RUN apt-get install solc
 
@@ -26,7 +25,7 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-COPY contract-migration/requirements.txt . 
+COPY requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
@@ -38,5 +37,5 @@ RUN pip install --index-url https://pypi.org/simple \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
 
-COPY contract-migration/ .
+COPY . .
 RUN chmod +x *.sh

apps/contract-migration/requirements.txt

@@ -1,5 +1,10 @@
-cic_base[full]==0.1.3a4+build.ce68c833
-sarafu-faucet~=0.0.4a1
-cic-eth[tools]==0.12.0a2
-eth-erc20~=0.0.10a3
-erc20-demurrage-token==0.0.2a3
+cic-eth[tools]==0.12.2a3
+eth-erc20>=0.0.11a1,<0.1.0
+erc20-demurrage-token>=0.0.2a5,<0.1.0
+eth-address-index>=0.1.3a1,<0.2.0
+eth-accounts-index>=0.0.13a1,<0.1.0
+cic-eth-registry>=0.5.7a1,<=0.6.0
+erc20-faucet>=0.2.3a1,<0.3.0
+erc20-transfer-authorization>=0.3.3a1,<0.4.0
+sarafu-faucet>=0.0.4a4,<0.1.0
+chainlib-eth>=0.0.6a1,<0.1.0

apps/contract-migration/reset.sh

@@ -3,14 +3,9 @@
 set -a
 
 default_token=giftable_erc20_token
-CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
 TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL}
-TOKEN_NAME=${TOKEN_NAME:-$TOKEN_SYMBOL}
+TOKEN_NAME=${TOKEN_NAME}
 TOKEN_TYPE=${TOKEN_TYPE:-$default_token}
-if [ $TOKEN_TYPE == 'default' ]; then
-	>&2 echo resolving "default" token to $default_token
-	TOKEN_TYPE=$default_token
-fi
 cat <<EOF
 external token settings:
 token_type: $TOKEN_TYPE
@@ -23,7 +18,6 @@ token_supply_limit: $TOKEN_SUPPLY_LIMIT
 EOF
 
 CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
-TOKEN_SYMBOL=${TOKEN_SYMBOL:-GFT}
 DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
@@ -75,17 +69,34 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 	fi
 
-	if [ $TOKEN_TYPE == $default_token ]; then
+	if [ "$TOKEN_TYPE" == "$default_token" ]; then
+		if [ -z "$TOKEN_SYMBOL" ]; then
+			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
+			TOKEN_SYMBOL="GFT"
+			TOKEN_NAME="Giftable Token"
+		elif [ -z "$TOKEN_NAME" ]; then
+			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
+			TOKEN_NAME=$TOKEN_SYMBOL
+		fi
 		>&2 echo deploying default token $TOKEN_TYPE
-		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name $TOKEN_NAME --symbol $TOKEN_SYMBOL --decimals 6 -vv`
-	elif [ $TOKEN_TYPE == 'erc20_demurrage_token' ]; then
+		echo giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
+	elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
+		if [ -z "$TOKEN_SYMBOL" ]; then
+			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
+			TOKEN_SYMBOL="SARAFU"
+			TOKEN_NAME="Sarafu Token"
+		elif [ -z "$TOKEN_NAME" ]; then
+			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
+			TOKEN_NAME=$TOKEN_SYMBOL
+		fi
 		>&2 echo deploying token $TOKEN_TYPE
 		if [ -z $TOKEN_SINK_ADDRESS ]; then
 			if [ ! -z $TOKEN_REDISTRIBUTION_PERIOD ]; then
 				>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
 			fi
 		fi
-		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww`
+		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww`
 	else
 		>&2 echo unknown token type $TOKEN_TYPE
 		exit 1
@@ -137,6 +148,7 @@ else
 fi
 
 mkdir -p $CIC_DATA_DIR
+>&2 echo using data dir $CIC_DATA_DIR for environment variable dump
 
 # this is consumed in downstream services to set environment variables
 cat << EOF > $CIC_DATA_DIR/.env

apps/contract-migration/seed_cic_eth.sh

@@ -15,10 +15,6 @@ CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic}
 ETH_PASSPHRASE=''
 CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
 TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
-if [[ $CIC_DEFAULT_TOKEN_SYMBOL != 'GFT' && $CIC_DEFAULT_TOKEN_SYMBOL != 'SRF' ]]; then
-	>&2 echo CIC_DEFAULT_TOKEN_SYMBOL must be one of [GFT,SRF], but was $CIC_DEFAULT_TOKEN_SYMBOL
-	exit 1
-fi
 
 # Debug flag
 DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
@@ -48,18 +44,18 @@ EOF
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
-DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
 
 
 >&2 echo "create account for sarafu gifter"
-DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER
 
 >&2 echo "create account for approval escrow owner"
-DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER 
 
@@ -69,21 +65,23 @@ cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRA
 #cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER
 
 >&2 echo "create account for accounts index writer"
-DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"
 eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 
 # Transfer gas to custodial gas provider adddress
+_CONFINI_DIR=$CONFINI_DIR
+unset CONFINI_DIR
 >&2 echo gift gas to gas gifter
->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
 
 >&2 echo gift gas to sarafu token owner
->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
 
 >&2 echo gift gas to account index owner
->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
 
 
 # Send token to token creator
@@ -104,6 +102,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 #echo -n 0 > $init_level_file
 
+CONFINI_DIR=$_CONFINI_DIR
 # Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
 cic-eth-ctl -i :: unlock INIT
 cic-eth-ctl -i :: unlock SEND

apps/data-seeding/.dockerignore

@@ -0,0 +1,8 @@
+.git
+.cache
+.dot
+**/doc
+**/node_modules
+**/venv
+**/.venv
+

apps/data-seeding/.gitlab-ci.yml

@@ -1,21 +1,26 @@
 .data_seeding_variables:
     variables:
         APP_NAME: data-seeding
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile_ci
-
-.data_seeding_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
 
 build-mr-data-seeding:
     extends:
-        - .data_seeding_changes_target
         - .py_build_merge_request
         - .data_seeding_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/data-seeding/**/*
+      when: always
 
 build-push-data-seeding:
     extends:
         - .py_build_push
         - .data_seeding_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/data-seeding/**/*
+      when: always
 

apps/data-seeding/README.md

@@ -167,7 +167,7 @@ If you have previously run the `cic_ussd` import incompletely, it could be a goo
 
 Then, in sequence, run in first terminal:
 
-`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
+`python cic_ussd/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
 
 In second terminal:
 

apps/data-seeding/cic_eth/import_balance.py

@@ -11,7 +11,6 @@ import csv
 import json
 
 # external imports
-import eth_abi
 import confini
 from hexathon import (
         strip_0x,
@@ -29,7 +28,10 @@ from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
 from chainlib.jsonrpc import JSONRPCRequest
-from chainlib.eth.error import EthException
+from chainlib.eth.error import (
+        EthException,
+        RequestMismatchException,
+        )
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
@@ -37,6 +39,9 @@ from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
 from cic_base.eth.syncer import chain_interface
+from eth_accounts_index import AccountsIndex
+from eth_contract_registry import Registry
+from eth_token_index import TokenUniqueSymbolIndex
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -131,58 +136,52 @@ class Handler:
             logg.debug('no payload, skipping {}'.format(tx))
             return
 
-        if tx.payload[:8] == self.account_index_add_signature:
-            recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
-            #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
-            user_file = 'new/{}/{}/{}.json'.format(
-                    recipient[2:4].upper(),
-                    recipient[4:6].upper(),
-                    recipient[2:].upper(),
-                    )
-            filepath = os.path.join(self.user_dir, user_file)
-            o = None
-            try:
-                f = open(filepath, 'r')
-                o = json.load(f)
-                f.close()
-            except FileNotFoundError:
-                logg.error('no import record of address {}'.format(recipient))
-                return
-            u = Person.deserialize(o)
-            original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
-            try:
-                balance = self.balances[original_address]
-            except KeyError as e:
-                logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
-                return
-
-            # TODO: store token object in handler ,get decimals from there
-            multiplier = 10**6
-            balance_full = balance * multiplier
-            logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
-
-            (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
-            logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
-            r = conn.do(o)
-
-            tx_path = os.path.join(
-                    user_dir,
-                    'txs',
-                    strip_0x(tx_hash_hex),
-                    )
-            f = open(tx_path, 'w')
-            f.write(strip_0x(o['params'][0]))
+        recipient = None
+        try:
+            r = AccountsIndex.parse_add_request(tx.payload)
+        except RequestMismatchException:
+            return
+        recipient = r[0]
+        
+        user_file = 'new/{}/{}/{}.json'.format(
+                recipient[2:4].upper(),
+                recipient[4:6].upper(),
+                recipient[2:].upper(),
+                )
+        filepath = os.path.join(self.user_dir, user_file)
+        o = None
+        try:
+            f = open(filepath, 'r')
+            o = json.load(f)
             f.close()
-#        except TypeError as e:
-#            logg.warning('typerror {}'.format(e))
-#            pass
-#        except IndexError as e:
-#            logg.warning('indexerror {}'.format(e))
-#            pass
-#        except EthException as e:
-#            logg.error('send error {}'.format(e).ljust(200))
-        #except KeyError as e:
-        #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
+        except FileNotFoundError:
+            logg.error('no import record of address {}'.format(recipient))
+            return
+        u = Person.deserialize(o)
+        original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
+        try:
+            balance = self.balances[original_address]
+        except KeyError as e:
+            logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
+            return
+
+        # TODO: store token object in handler ,get decimals from there
+        multiplier = 10**6
+        balance_full = balance * multiplier
+        logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
+
+        (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
+        logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
+        r = conn.do(o)
+
+        tx_path = os.path.join(
+                user_dir,
+                'txs',
+                strip_0x(tx_hash_hex),
+                )
+        f = open(tx_path, 'w')
+        f.write(strip_0x(o['params'][0]))
+        f.close()
 
 
 def progress_callback(block_number, tx_index):
@@ -198,49 +197,26 @@ def main():
     nonce_oracle = RPCNonceOracle(signer_address, conn)
 
     # Get Token registry address
-    txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
-    tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
-
-    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
-    data = add_0x(registry_addressof_method)
-    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
-    txf.set_code(tx, data)
-   
-    j = JSONRPCRequest()
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
+    registry = Registry(chain_spec)
+    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
     r = conn.do(o)
-    token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    token_index_address = registry.parse_address_of(r)
+    token_index_address = to_checksum_address(token_index_address)
     logg.info('found token index address {}'.format(token_index_address))
-
-
+    
     # Get Sarafu token address
-    tx = txf.template(signer_address, token_index_address)
-    data = add_0x(registry_addressof_method)
-    h = hashlib.new('sha256')
-    h.update(token_symbol.encode('utf-8'))
-    z = h.digest()
-    data += eth_abi.encode_single('bytes32', z).hex()
-    txf.set_code(tx, data)
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
+    token_index = TokenUniqueSymbolIndex(chain_spec)
+    o = token_index.address_of(token_index_address, token_symbol)
     r = conn.do(o)
+    token_address = token_index.parse_address_of(r)
     try:
-        sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+        token_address = to_checksum_address(token_address)
     except ValueError as e:
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
-
-    if sarafu_token_address == ZERO_ADDRESS:
-        raise KeyError('token address for symbol {} is zero'.format(token_symbol))
-
-    logg.info('found token address {}'.format(sarafu_token_address))
+    logg.info('found token address {}'.format(token_address))
+    
+    sys.exit(0)
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -248,22 +224,6 @@ def main():
         o = block_latest()
         r = conn.do(o)
         block_offset = int(strip_0x(r), 16) + 1
-#
-#    addresses = {}
-#    f = open('{}/addresses.csv'.format(user_dir, 'r'))
-#    while True:
-#        l = f.readline()
-#        if l == None:
-#            break
-#        r = l.split(',')
-#        try:
-#            k = r[0]
-#            v = r[1].rstrip()
-#            addresses[k] = v
-#            sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
-#        except IndexError as e:
-#            break
-#    f.close()
 
     # TODO get decimals from token
     balances = {}

apps/data-seeding/cic_meta/import_meta.js

@@ -11,7 +11,6 @@ const config = new crdt.Config('./config');
 config.process();
 console.log(config);
 
-
 function sendit(uid, envelope) {
 	const d = envelope.toJSON();