Bump aux version

.gitignore

@@ -13,4 +13,3 @@ build/
 **/coverage
 **/.venv
 .idea
-**/.vim

.gitlab-ci.yml

@@ -1,44 +1,14 @@
-#include:
+include:
-#  - local: 'ci_templates/.cic-template.yml'
+  - local: 'ci_templates/.cic-template.yml'
-#  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  - local: 'apps/contract-migration/.gitlab-ci.yml'
-#  - local: 'apps/cic-eth/.gitlab-ci.yml'
+  - local: 'apps/cic-eth/.gitlab-ci.yml'
-#  - local: 'apps/cic-ussd/.gitlab-ci.yml'
+  - local: 'apps/cic-ussd/.gitlab-ci.yml'
-#  - local: 'apps/cic-notify/.gitlab-ci.yml'
+  - local: 'apps/cic-notify/.gitlab-ci.yml'
-#  - local: 'apps/cic-meta/.gitlab-ci.yml'
+  - local: 'apps/cic-meta/.gitlab-ci.yml'
-#  - local: 'apps/cic-cache/.gitlab-ci.yml'
+  - local: 'apps/cic-cache/.gitlab-ci.yml'
-#  - local: 'apps/data-seeding/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'
-
-image: tiangolo/docker-with-compose
-
-before_script:
-  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
 
 stages:
   - build
-  - deploy
+  - test
-
+  - release
-variables:
-  DOCKER_BUILDKIT: 1
-  COMPOSE_DOCKER_CLI_BUILD: 1 
-
-  #services:
-  #  - docker:stable-dind
-
-build-prod:
-  tags:
-    - integration
-  stage: build
-  script:
-    -  docker-compose build
-  only:
-    - bvander/integration-tests-on-docker
-
-deploy-prod:
-  tags:
-    - integration
-  stage: deploy
-  script:
-    - docker stack deploy -c docker-compose.yml --with-registry-auth my-stack
-  only:
-    - bvander/integration-tests-on-docker
-

apps/cic-cache/.coveragerc

@@ -2,5 +2,4 @@
 omit = 
 	.venv/*
 	scripts/*
-	cic_cache/db/migrations/*
+	cic_cache/db/postgres/*
-	cic_cache/version.py

apps/cic-cache/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-cache/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_cache_variables:
     variables:
         APP_NAME: cic-cache
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_cache_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-cache:
     extends:
+        - .cic_cache_changes_target
         - .py_build_merge_request
         - .cic_cache_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-
-test-mr-cic-cache:
-    stage: test
-    extends:
-        - .cic_cache_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-cache"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
 
 build-push-cic-cache:
     extends:
         - .py_build_push
         - .cic_cache_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
         
 

apps/cic-cache/cic_cache/cache.py

@@ -15,8 +15,6 @@ from cic_cache.db.list import (
 logg = logging.getLogger()
 
 
-DEFAULT_FILTER_SIZE = 8192 * 8
-
 class Cache:
 
     def __init__(self, session):
@@ -27,7 +25,7 @@ class BloomCache(Cache):
 
     @staticmethod
     def __get_filter_size(n):
-        n = DEFAULT_FILTER_SIZE
+        n = 8192 * 8
         logg.warning('filter size hardcoded to {}'.format(n))
         return n
 

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,4 +100,3 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -4,9 +4,6 @@ import json
 import re
 import base64
 
-# external imports
-from hexathon import add_0x
-
 # local imports
 from cic_cache.cache import (
         BloomCache,
@@ -14,11 +11,10 @@ from cic_cache.cache import (
     )
 
 logg = logging.getLogger(__name__)
-#logg = logging.getLogger()
 
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
+re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
 
 DEFAULT_LIMIT = 100
 
@@ -30,13 +26,13 @@ def process_transactions_account_bloom(session, env):
 
     address = r[1]
     if r[2] == None:
-        address = add_0x(address)
+        address = '0x' + address
-    offset = 0
+    offset = DEFAULT_LIMIT
     if r.lastindex > 2:
-        offset = r[4]
+        offset = r[3]
-    limit = DEFAULT_LIMIT
+    limit = 0
-    if r.lastindex > 4:
+    if r.lastindex > 3:
-        limit = r[6]
+        limit = r[4]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)

apps/cic-cache/docker/Dockerfile

@@ -1,38 +1,52 @@
-# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-COPY requirements.txt .
+#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+WORKDIR /usr/src/cic-cache
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
 
-COPY . . 
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+ARG root_requirement_file='requirements.txt'
 
-RUN python setup.py install
+#RUN apk update && \
+#	apk add gcc musl-dev gnupg libpq
+#RUN apk add postgresql-dev
+#RUN apk add linux-headers
+#RUN apk add libffi-dev
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+# Copy shared requirements from top of mono-repo
+RUN echo "copying root req file ${root_requirement_file}"
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY cic-cache/requirements.txt ./
+COPY cic-cache/setup.cfg \
+	 cic-cache/setup.py \
+ 	 ./
+COPY cic-cache/cic_cache/ ./cic_cache/
+COPY cic-cache/scripts/ ./scripts/
+COPY cic-cache/test_requirements.txt ./
+RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+RUN pip install $pip_extra_index_url_flag .
+RUN pip install .[server]
+
+COPY cic-cache/tests/ ./tests/
+#COPY db/ cic-cache/db
+#RUN apk add postgresql-client
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
+COPY cic-cache/config/ /usr/local/etc/cic-cache/
 
 # for db migrations
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
+COPY cic-cache/docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server
 # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/Dockerfile_ci

@@ -1,37 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/requirements.txt

@@ -1,13 +1,13 @@
-cic-base~=0.2.0a4
+cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
-confini>=0.3.6rc3,<0.5.0
+confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.6a2
+cic-eth-registry~=0.5.6a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a5
+chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a2
+erc20-faucet~=0.2.2a1

apps/cic-cache/tests/conftest.py

@@ -5,12 +5,9 @@ import datetime
 
 # external imports
 import pytest
-import moolb
 
 # local imports
 from cic_cache import db
-from cic_cache import BloomCache
-from cic_cache.cache import DEFAULT_FILTER_SIZE
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -104,7 +101,3 @@ def tag_txs(
 
     db.tag_transaction(init_database, txs[1], 'taag', domain='test')
 
-
-@pytest.fixture(scope='session')
-def zero_filter():
-    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/test_erc20.py

@@ -10,7 +10,6 @@ from sqlalchemy import text
 from chainlib.eth.tx import Tx
 from chainlib.eth.block import Block
 from chainlib.chain import ChainSpec
-from chainlib.eth.error import RequestMismatchException
 from hexathon import (
         strip_0x,
         add_0x,
@@ -19,21 +18,10 @@ from hexathon import (
 # local imports
 from cic_cache.db import add_tag
 from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
-from cic_cache.runnable.daemons.filters.base import TagSyncFilter
 
 logg = logging.getLogger()
 
 
-def test_base_filter_str(
-        init_database,
-        ):
-        f = TagSyncFilter('foo')
-        assert 'foo' == str(f)
-        f = TagSyncFilter('foo', domain='bar')
-        assert 'bar.foo' == str(f)
-
-
-
 def test_erc20_filter(
         eth_rpc,
         foo_token,
@@ -79,95 +67,3 @@ def test_erc20_filter(
     s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
     r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
     assert r[0] == tx.hash
-
-
-def test_erc20_filter_nocontract(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': os.urandom(20).hex(),
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-
-@pytest.mark.parametrize(
-        'contract_method,contract_input,expected_exception',
-        [
-            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
-            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
-            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
-            ],
-        )
-def test_erc20_filter_bogus(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        contract_method,
-        contract_input,
-        expected_exception,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = contract_method
-    data += contract_input 
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/test_query.py

@@ -1,230 +0,0 @@
-# standard imports
-import logging
-import json
-import base64
-import copy
-import re
-
-# external imports
-import pytest
-from hexathon import strip_0x
-
-# local imports
-from cic_cache.runnable.daemons.query import *
-
-logg = logging.getLogger()
-
-
-
-@pytest.mark.parametrize(
-        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
-        [
-            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
-            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
-            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
-            ],
-        )
-def test_query_regex(
-        list_actors,
-        query_path_prefix,
-        query_role,
-        query_address_index,
-        query_offset,
-        query_offset_index,
-        query_limit,
-        query_limit_index,
-        match_re,
-        ):
-
-        paths = []
-        path = query_path_prefix
-        query_address = None
-        if query_role != None:
-            query_address = strip_0x(list_actors[query_role])
-            paths.append(path + '0x' + query_address)
-            paths.append(path + query_address)
-        if query_offset != None:
-            if query_limit != None:
-                for i in range(len(paths)-1):
-                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
-            else:
-                for i in range(len(paths)-1):
-                    paths[i] += '/' + str(query_offset)
-
-        for i in range(len(paths)):
-            paths.append(paths[i] + '/')
-
-        for p in paths:
-            logg.debug('testing path {} against {}'.format(p, match_re))
-            m = re.match(match_re, p)
-            l = len(m.groups())
-            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
-            for i in range(l+1):
-                logg.debug('group {} {}'.format(i, m[i]))
-            if m.lastindex >= query_offset_index:
-                assert query_offset == int(m[query_offset_index + 1])
-            if m.lastindex >= query_limit_index:
-                assert query_limit == int(m[query_limit_index + 1])
-            if query_address_index != None:
-                match_address = strip_0x(m[query_address_index + 1])
-                assert query_address == match_address
-
-
-
-@pytest.mark.parametrize(
-        'role_name, query_offset, query_limit, query_match',
-        [ 
-            ('alice', None, None, [(420000, 13), (419999, 42)]),
-            ('alice', None, 1, [(420000, 13)]),
-            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
-            ('alice', 2, None, []), # 420000 == list_defaults['block']
-            ],
-        )
-def test_query_process_txs_account(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        role_name,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    actor = None
-    try:
-        actor = list_actors[role_name]
-    except KeyError:
-        actor = os.urandom(20).hex()
-    path_info = '/tx/user/0x' + strip_0x(actor)
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_account_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_offset, query_limit, query_match',
-        [ 
-            (None, 2, [(420000, 13), (419999, 42)]),
-            (0, 1, [(420000, 13)]),
-            (1, 1, [(419999, 42)]),
-            (2, 0, []),
-            ],
-        )
-def test_query_process_txs_bloom(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    path_info = '/tx'
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_block_start, query_block_end, query_match_count',
-        [ 
-            (None, 42, 0),
-            (420000, 420001, 1),
-            (419999, 419999, 1), # matches are inclusive
-            (419999, 420000, 2),
-            (419999, 420001, 2),
-            ],
-        )
-def test_query_process_txs_data(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_block_start,
-        query_block_end,
-        query_match_count,
-        ):
-
-    path_info = '/txa'
-    if query_block_start != None:
-        path_info += '/' + str(query_block_start)
-    if query_block_end != None:
-        if query_block_start == None:
-            path_info += '/0'
-        path_info += '/' + str(query_block_end)
-    env = {
-            'PATH_INFO': path_info,
-            'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_data(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    assert len(o['data']) == query_match_count

apps/cic-eth/.dockerignore

@@ -1,6 +0,0 @@
-.git
-.cache
-.dot
-**/doc
-**/.venv
-**/venv

apps/cic-eth/.gitlab-ci.yml

@@ -1,52 +1,31 @@
 .cic_eth_variables:
     variables:
         APP_NAME: cic-eth
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_eth_mr_changes_target:
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - $CONTEXT/$APP_NAME/**/*
+      when: always
 
 build-mr-cic-eth:
     extends:
         - .cic_eth_variables
-        - .py_build_target_dev
+        - .cic_eth_mr_changes_target
-    rules:
+        - .py_build_target_test
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 test-mr-cic-eth:
-    stage: test
     extends:
         - .cic_eth_variables
-    cache:
+        - .cic_eth_mr_changes_target
-        key:
+    stage: test
-            files:
+    image: $IMAGE_TAG_BASE
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
     script:
         - cd apps/$APP_NAME/
-        - >
+        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r admin_requirements.txt
-          -r services_requirements.txt
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-eth"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 build-push-cic-eth:
     extends:
         - .py_build_push
-        - .cic_eth_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-eth/**/*
-      when: always

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.5.6a2,<0.6.0
+cic-eth-registry~=0.5.6a1
 hexathon~=0.0.1a7
-chainqueue>=0.0.3a1,<0.1.0
+chainqueue~=0.0.2b5
-eth-erc20>=0.0.10a3,<0.1.0
+eth-erc20==0.0.10a2

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -6,11 +6,6 @@ import logging
 import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
-from hexathon import (
-        add_0x,
-        strip_0x,
-        uniform as hex_uniform,
-        )
 
 # local imports
 from cic_eth.db.enum import LockEnum
@@ -24,12 +19,6 @@ from cic_eth.error import LockedError
 celery_app = celery.current_app
 logg = logging.getLogger()
 
-
-def normalize_address(a):
-    if a == None:
-        return None
-    return add_0x(hex_uniform(strip_0x(a)))
-
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
@@ -43,7 +32,6 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -65,7 +53,6 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -85,7 +72,6 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -103,7 +89,6 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -121,7 +106,6 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -139,7 +123,6 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -148,7 +131,6 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))

apps/cic-eth/cic_eth/admin/nonce.py

@@ -14,11 +14,7 @@ from chainqueue.sql.query import get_tx
 from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
-from hexathon import (
+from hexathon import strip_0x
-        strip_0x,
-        add_0x,
-        uniform as hex_uniform,
-        )
 from potaahto.symbols import snake_and_camel
 
 # local imports
@@ -73,17 +69,15 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
     q = session.query(Otx)
     q = q.join(TxCache)
-    q = q.filter(TxCache.sender==query_address)
+    q = q.filter(TxCache.sender==address)
     q = q.filter(Otx.nonce>=nonce+delta)
     q = q.order_by(Otx.nonce.asc())
     otxs = q.all()
 
     tx_hashes = []
     txs = []
-    gas_total = 0
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
@@ -95,10 +89,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         tx_new['gas_price'] += 1
         tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
-        gas_total += tx_new['gas_price'] * tx_new['gas']
 
         logg.debug('tx_new {}'.format(tx_new))
-        logg.debug('gas running total {}'.format(gas_total))
 
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
@@ -130,10 +122,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     s = create_check_gas_task(
          txs, 
          chain_spec,
-         #tx_new['from'],
+         tx_new['from'],
-         address,
+         gas=tx_new['gas'],
-         #gas=tx_new['gas'],
-         gas=gas_total,
          tx_hashes_hex=tx_hashes, 
          queue=queue,
         )
@@ -142,8 +132,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_send',
         [
             chain_spec.asdict(),
-            address,
+            tx_new['from'],
-            #tx_new['from'],
             ],
         queue=queue,
         )
@@ -151,8 +140,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_queue',
         [
             chain_spec.asdict(),
-            address,
+            tx_new['from'],
-            #tx_new['from'],
             ],
         queue=queue,
         )

apps/cic-eth/cic_eth/api/admin.py

@@ -21,7 +21,6 @@ from chainlib.hash import keccak256_hex_to_hex
 from hexathon import (
         strip_0x,
         add_0x,
-        uniform as hex_uniform,
         )
 from chainlib.eth.gas import balance
 from chainqueue.db.enum import (
@@ -308,8 +307,6 @@ class AdminApi:
         :param address: Ethereum address to return transactions for
         :type address: str, 0x-hex
         """
-
-        address = add_0x(hex_uniform(strip_0x(address)))
         last_nonce = -1
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',

apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py

@@ -8,8 +8,7 @@ Create Date: 2021-04-02 18:30:55.398388
 from alembic import op
 import sqlalchemy as sa
 
-#from chainqueue.db.migrations.sqlalchemy import (
+from chainqueue.db.migrations.sqlalchemy import (
-from chainqueue.db.migrations.default.export import (
         chainqueue_upgrade,
         chainqueue_downgrade,
         )

apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py

@@ -8,8 +8,7 @@ Create Date: 2021-04-02 18:36:44.459603
 from alembic import op
 import sqlalchemy as sa
 
-#from chainsyncer.db.migrations.sqlalchemy import (
+from chainsyncer.db.migrations.sqlalchemy import (
-from chainsyncer.db.migrations.default.export import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-eth/cic_eth/db/models/base.py

@@ -126,4 +126,3 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-eth/cic_eth/eth/account.py

@@ -23,7 +23,7 @@ from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
 from eth_accounts_index import AccountsIndex 
 from sarafu_faucet import MinterFaucet
-from chainqueue.sql.tx import cache_tx_dict
+from chainqueue.db.models.tx import TxCache
 
 # local import
 from cic_eth_registry import CICRegistry
@@ -300,17 +300,20 @@ def cache_gift_data(
 
     session = self.create_session()
 
-    tx_dict = {
+    tx_cache = TxCache(
-            'hash': tx_hash_hex,
+        tx_hash_hex,
-            'from': tx['from'],
+        tx['from'],
-            'to': tx['to'],
+        tx['to'],
-            'source_token': ZERO_ADDRESS,
+        ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
+        ZERO_ADDRESS,
-            'from_value': 0,
+        0,
-            'to_value': 0,
+        0,
-            }
+        session=session,
+            )
 
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -339,15 +342,18 @@ def cache_account_data(
     tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
-    tx_dict = {
+    tx_cache = TxCache(
-            'hash': tx_hash_hex,
+        tx_hash_hex,
-            'from': tx['from'],
+        tx['from'],
-            'to': tx['to'],
+        tx['to'],
-            'source_token': ZERO_ADDRESS,
+        ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
+        ZERO_ADDRESS,
-            'from_value': 0,
+        0,
-            'to_value': 0,
+        0,
-            }
+        session=session,
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)

apps/cic-eth/cic_eth/eth/bancor.py.bak

@@ -0,0 +1,385 @@
+# standard imports
+import os
+import logging
+
+# third-party imports
+import celery
+import web3
+from cic_registry import CICRegistry
+from cic_registry.chain import ChainSpec
+
+# local imports
+from cic_eth.db import SessionBase
+from cic_eth.db.models.convert import TxConvertTransfer
+from cic_eth.db.models.otx import Otx
+from cic_eth.db.models.tx import TxCache
+from cic_eth.eth.task import sign_and_register_tx
+from cic_eth.eth.task import create_check_gas_and_send_task
+from cic_eth.eth.token import TokenTxFactory
+from cic_eth.eth.factory import TxFactory
+from cic_eth.eth.util import unpack_signed_raw_tx
+from cic_eth.eth.rpc import RpcClient
+
+celery_app = celery.current_app 
+#logg = celery_app.log.get_default_logger()
+logg = logging.getLogger()
+
+contract_function_signatures = {
+        'convert': 'f3898a97',
+        'convert2': '569706eb',
+        }
+
+
+class BancorTxFactory(TxFactory):
+
+    """Factory for creating Bancor network transactions.
+    """
+    def convert(
+            self,
+            source_token_address,
+            destination_token_address,
+            reserve_address,
+            source_amount,
+            minimum_return,
+            chain_spec,
+            fee_beneficiary='0x0000000000000000000000000000000000000000',
+            fee_ppm=0,
+            ):
+        """Create a BancorNetwork "convert" transaction.
+
+        :param source_token_address: ERC20 contract address for token to convert from
+        :type source_token_address: str, 0x-hex
+        :param destination_token_address: ERC20 contract address for token to convert to
+        :type destination_token_address: str, 0x-hex
+        :param reserve_address: ERC20 contract address of Common reserve token
+        :type reserve_address: str, 0x-hex
+        :param source_amount: Amount of source tokens to convert
+        :type source_amount: int
+        :param minimum_return: Minimum amount of destination tokens to accept as result for conversion
+        :type source_amount: int
+        :return: Unsigned "convert" transaction in standard Ethereum format
+        :rtype: dict
+        """
+        network_contract = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
+        network_gas = network_contract.gas('convert')
+        tx_convert_buildable = network_contract.contract.functions.convert2(
+            [
+                source_token_address,
+                source_token_address,
+                reserve_address,
+                destination_token_address,
+                destination_token_address,
+            ],
+            source_amount,
+            minimum_return,
+            fee_beneficiary,
+            fee_ppm,
+            )
+        tx_convert = tx_convert_buildable.buildTransaction({
+                'from': self.address,
+                'gas': network_gas,
+                'gasPrice': self.gas_price,
+                'chainId': chain_spec.chain_id(),
+                'nonce': self.next_nonce(),
+                })
+        return tx_convert
+
+
+def unpack_convert(data):
+    f = data[2:10]
+    if f != contract_function_signatures['convert2']:
+        raise ValueError('Invalid convert data ({})'.format(f))
+
+    d = data[10:]
+    path = d[384:]
+    source = path[64-40:64]
+    destination = path[-40:]
+
+    amount = int(d[64:128], 16)
+    min_return = int(d[128:192], 16)
+    fee_recipient = d[192:256]
+    fee = int(d[256:320], 16)
+    return {
+        'amount': amount,
+        'min_return': min_return,
+        'source_token': web3.Web3.toChecksumAddress('0x' + source),
+        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
+        'fee_recipient': fee_recipient,
+        'fee': fee,
+        }
+
+
+
+# Kept for historical reference, it unpacks a convert call without fee parameters
+#def _unpack_convert_mint(data):
+#    f = data[2:10]
+#    if f != contract_function_signatures['convert2']:
+#        raise ValueError('Invalid convert data ({})'.format(f))
+#
+#    d = data[10:]
+#    path = d[256:]
+#    source = path[64-40:64]
+#    destination = path[-40:]
+#
+#    amount = int(d[64:128], 16)
+#    min_return = int(d[128:192], 16)
+#    return {
+#        'amount': amount,
+#        'min_return': min_return,
+#        'source_token': web3.Web3.toChecksumAddress('0x' + source),
+#        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
+#        }
+
+
+@celery_app.task(bind=True)
+def convert_with_default_reserve(self, tokens, from_address, source_amount, minimum_return, to_address, chain_str):
+    """Performs a conversion between two liquid tokens using Bancor network.
+
+    :param tokens: Token pair, source and destination respectively
+    :type tokens: list of str, 0x-hex
+    :param from_address: Ethereum address of sender
+    :type from_address: str, 0x-hex
+    :param source_amount: Amount of source tokens to convert
+    :type source_amount: int
+    :param minimum_return: Minimum about of destination tokens to receive
+    :type minimum_return: int
+    """
+
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    queue = self.request.delivery_info['routing_key']
+
+    c = RpcClient(chain_spec, holder_address=from_address)
+
+    cr = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
+    source_token = CICRegistry.get_address(chain_spec, tokens[0]['address'])
+    reserve_address = CICRegistry.get_contract(chain_spec, 'BNTToken', 'ERC20').address()
+
+    tx_factory = TokenTxFactory(from_address, c)
+   
+    tx_approve_zero = tx_factory.approve(source_token.address(), cr.address(), 0, chain_spec)
+    (tx_approve_zero_hash_hex, tx_approve_zero_signed_hex) = sign_and_register_tx(tx_approve_zero, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
+
+    tx_approve = tx_factory.approve(source_token.address(), cr.address(), source_amount, chain_spec)
+    (tx_approve_hash_hex, tx_approve_signed_hex) = sign_and_register_tx(tx_approve, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
+
+    tx_factory = BancorTxFactory(from_address, c)
+    tx_convert = tx_factory.convert(
+            tokens[0]['address'],
+            tokens[1]['address'],
+            reserve_address,
+            source_amount,
+            minimum_return,
+            chain_spec,
+            )
+    (tx_convert_hash_hex, tx_convert_signed_hex) = sign_and_register_tx(tx_convert, chain_str, queue, 'cic_eth.eth.bancor.otx_cache_convert')
+
+    # TODO: consider moving save recipient to async task / chain it before the tx send
+    if to_address != None:
+        save_convert_recipient(tx_convert_hash_hex, to_address, chain_str)
+
+    s = create_check_gas_and_send_task(
+            [tx_approve_zero_signed_hex, tx_approve_signed_hex, tx_convert_signed_hex],
+            chain_str,
+            from_address,
+            tx_approve_zero['gasPrice'] * tx_approve_zero['gas'],
+            tx_hashes_hex=[tx_approve_hash_hex],
+            queue=queue,
+            )
+    s.apply_async()
+    return tx_convert_hash_hex
+
+
+#@celery_app.task()
+#def process_approval(tx_hash_hex):
+#    t = session.query(TxConvertTransfer).query(TxConvertTransfer.approve_tx_hash==tx_hash_hex).first()
+#    c = session.query(Otx).query(Otx.tx_hash==t.convert_tx_hash)
+#    gas_limit = 8000000
+#    gas_price = GasOracle.gas_price()
+#
+#    # TODO: use celery group instead
+#    s_queue = celery.signature(
+#            'cic_eth.queue.tx.create',
+#            [
+#                nonce,
+#                c['address'], # TODO: check that this is in fact sender address
+#                c['tx_hash'],
+#                c['signed_tx'],
+#                ]
+#            )
+#    s_queue.apply_async()
+#
+#    s_check_gas = celery.signature(
+#            'cic_eth.eth.gas.check_gas',
+#            [
+#                c['address'],
+#                [c['signed_tx']],
+#                gas_limit * gas_price,
+#                ]
+#            )
+#    s_send = celery.signature(
+#            'cic_eth.eth.tx.send',
+#            [],
+#            )
+#
+#    s_set_sent = celery.signature(
+#            'cic_eth.queue.state.set_sent',
+#            [False],
+#            )
+#    s_send.link(s_set_sent)
+#    s_check_gas.link(s_send)
+#    s_check_gas.apply_async()
+#    return tx_hash_hex
+
+
+
+@celery_app.task()
+def save_convert_recipient(convert_hash, recipient_address, chain_str):
+    """Registers the recipient target for a convert-and-transfer operation.
+
+    :param convert_hash: Transaction hash of convert operation
+    :type convert_hash: str, 0x-hex
+    :param recipient_address: Address of consequtive transfer recipient
+    :type recipient_address: str, 0x-hex
+    """
+    session = SessionBase.create_session()
+    t = TxConvertTransfer(convert_hash, recipient_address, chain_str)
+    session.add(t)
+    session.commit()
+    session.close()
+
+
+@celery_app.task()
+def save_convert_transfer(convert_hash, transfer_hash):
+    """Registers that the transfer part of a convert-and-transfer operation has been executed.
+
+    :param convert_hash: Transaction hash of convert operation
+    :type convert_hash: str, 0x-hex
+    :param convert_hash: Transaction hash of transfer operation
+    :type convert_hash: str, 0x-hex
+    :returns: transfer_hash,
+    :rtype: list, single str, 0x-hex
+    """
+    session = SessionBase.create_session()
+    t = TxConvertTransfer.get(convert_hash)
+    t.transfer(transfer_hash)
+    session.add(t)
+    session.commit()
+    session.close()
+    return [transfer_hash]
+
+
+# TODO: seems unused, consider removing
+@celery_app.task()
+def resolve_converters_by_tokens(tokens, chain_str):
+    """Return converters for a list of tokens.
+
+    :param tokens: Token addresses to look up
+    :type tokens: list of str, 0x-hex
+    :return: Addresses of matching converters
+    :rtype: list of str, 0x-hex
+    """
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    for t in tokens:
+        c = CICRegistry.get_contract(chain_spec, 'ConverterRegistry')
+        fn = c.function('getConvertersByAnchors')
+        try:
+            converters = fn([t['address']]).call()
+        except Exception as e:
+            raise e
+        t['converters'] = converters
+
+    return tokens
+
+
+@celery_app.task(bind=True)
+def transfer_converted(self, tokens, holder_address, receiver_address, value, tx_convert_hash_hex, chain_str):
+    """Execute the ERC20 transfer of a convert-and-transfer operation.
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param holder_address: Token receiver address
+    :type holder_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    :return: Transaction hash
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    if len(tokens) != 1:
+        raise TokenCountError
+
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+
+    queue = self.request.delivery_info['routing_key']
+
+    c = RpcClient(chain_spec, holder_address=holder_address)
+
+    # get transaction parameters
+    gas_price = c.gas_price()
+    tx_factory = TokenTxFactory(holder_address, c)
+
+    token_address = tokens[0]['address']
+    tx_transfer = tx_factory.transfer(
+        token_address,
+        receiver_address,
+        value,
+        chain_spec,
+            )
+    (tx_transfer_hash_hex, tx_transfer_signed_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.token.otx_cache_transfer')
+
+    # send transaction
+    logg.info('transfer converted token {} from {} to {} value {} {}'.format(token_address, holder_address, receiver_address, value, tx_transfer_signed_hex))
+    s = create_check_gas_and_send_task(
+            [tx_transfer_signed_hex],
+            chain_str,
+            holder_address,
+            tx_transfer['gasPrice'] * tx_transfer['gas'],
+            None,
+            queue,
+            )
+    s_save = celery.signature(
+            'cic_eth.eth.bancor.save_convert_transfer',
+            [
+                tx_convert_hash_hex,
+                tx_transfer_hash_hex,
+                ],
+            queue=queue,
+            )
+    s_save.link(s)
+    s_save.apply_async()
+    return tx_transfer_hash_hex
+
+
+@celery_app.task()
+def otx_cache_convert(
+        tx_hash_hex,
+        tx_signed_raw_hex,
+        chain_str,
+        ):
+
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx_data = unpack_convert(tx['data'])
+    logg.debug('tx data {}'.format(tx_data))
+
+    session = TxCache.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        tx['from'],
+        tx_data['source_token'],
+        tx_data['destination_token'],
+        tx_data['amount'],
+        tx_data['amount'],
+            )
+    session.add(tx_cache)
+    session.commit()
+    session.close()
+    return tx_hash_hex
+

apps/cic-eth/cic_eth/eth/erc20.py

@@ -13,9 +13,9 @@ from chainlib.eth.tx import (
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
+from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
-from chainqueue.sql.tx import cache_tx_dict
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -375,16 +375,19 @@ def cache_transfer_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_dict = {
+    tx_cache = TxCache(
-            'hash': tx_hash_hex,
+        tx_hash_hex,
-            'from': tx['from'],
+        tx['from'],
-            'to': recipient_address,
+        recipient_address,
-            'source_token': tx['to'],
+        tx['to'],
-            'destination_token': tx['to'],
+        tx['to'],
-            'from_value': token_value,
+        token_value,
-            'to_value': token_value,
+        token_value,
-            }
+        session=session,
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -414,16 +417,19 @@ def cache_transfer_from_data(
     token_value = tx_data[2]
 
     session = SessionBase.create_session()
-    tx_dict = {
+    tx_cache = TxCache(
-            'hash': tx_hash_hex,
+        tx_hash_hex,
-            'from': tx['from'],
+        tx['from'],
-            'to': recipient_address,
+        recipient_address,
-            'source_token': tx['to'],
+        tx['to'],
-            'destination_token': tx['to'],
+        tx['to'],
-            'from_value': token_value,
+        token_value,
-            'to_value': token_value,
+        token_value,
-            }
+        session=session,
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -452,16 +458,19 @@ def cache_approve_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_dict = {
+    tx_cache = TxCache(
-            'hash': tx_hash_hex,
+        tx_hash_hex,
-            'from': tx['from'],
+        tx['from'],
-            'to': recipient_address,
+        recipient_address,
-            'source_token': tx['to'],
+        tx['to'],
-            'destination_token': tx['to'],
+        tx['to'],
-            'from_value': token_value,
+        token_value,
-            'to_value': token_value,
+        token_value,
-            }
+        session=session,
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 

apps/cic-eth/cic_eth/eth/gas.py

@@ -9,7 +9,6 @@ from chainlib.chain import ChainSpec
 from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.tx import cache_tx_dict
 from chainlib.eth.gas import (
         balance,
         price,
@@ -134,17 +133,20 @@ def cache_gas_data(
 
     session = SessionBase.create_session()
 
-    tx_dict = {
+    tx_cache = TxCache(
-            'hash': tx_hash_hex,
+        tx_hash_hex,
-            'from': tx['from'],
+        tx['from'],
-            'to': tx['to'],
+        tx['to'],
-            'source_token': ZERO_ADDRESS,
+        ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
+        ZERO_ADDRESS,
-            'from_value': tx['value'],
+        tx['value'],
-            'to_value': tx['value'],
+        tx['value'],
-            }
+        session=session,
+            )
 
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -18,6 +18,7 @@ from hexathon import (
         strip_0x,
         )
 from chainqueue.db.models.tx import Otx
+from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
 from potaahto.symbols import snake_and_camel

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,11 +14,9 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
-from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
-from cic_eth_registry.error import UnknownContractError
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
@@ -116,6 +114,9 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
+                        token = ERC20Token(chain_spec, rpc, tx['to'])
+                        token_symbol = token.symbol
+                        token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
@@ -125,6 +126,12 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
                             'destination_token': tx['to'],
+                            'source_token_symbol': token_symbol,
+                            'destination_token_symbol': token_symbol,
+                            'source_token_decimals': token_decimals,
+                            'destination_token_decimals': token_decimals,
+                            'source_token_chain': chain_str,
+                            'destination_token_chain': chain_str,
                             'nonce': tx['nonce'],
                                 }
                         if times['queue'] != None:
@@ -139,8 +146,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
-@celery_app.task(bind=True, base=BaseTask)
+@celery_app.task()
-def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=True, verify_contracts=True):
+def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
     """Merges transaction data from multiple sources and sorts them in chronological order.
 
     :param tx_batches: Transaction data inputs
@@ -189,32 +196,6 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr
     if newest_first:
         ks.reverse()
     for k in ks:
-        tx = txs_by_block[k]
+        txs.append(txs_by_block[k])
-        if verify_contracts:
-            try:
-                tx = verify_and_expand(tx, chain_spec, sender_address=BaseTask.call_address)
-            except UnknownContractError:
-                logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
-                continue
-        txs.append(tx)
 
     return txs
-
-
-def verify_and_expand(tx, chain_spec, sender_address=ZERO_ADDRESS):
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    registry = CICRegistry(chain_spec, rpc)
-
-    if tx.get('source_token_symbol') == None and tx['source_token'] != ZERO_ADDRESS:
-        r = registry.by_address(tx['source_token'], sender_address=sender_address)
-        token = ERC20Token(chain_spec, rpc, tx['source_token'])
-        tx['source_token_symbol'] = token.symbol
-        tx['source_token_decimals'] = token.decimals
-
-    if tx.get('destination_token_symbol') == None and tx['destination_token'] != ZERO_ADDRESS:
-        r = registry.by_address(tx['destination_token'], sender_address=sender_address)
-        token = ERC20Token(chain_spec, rpc, tx['destination_token'])
-        tx['destination_token_symbol'] = token.symbol
-        tx['destination_token_decimals'] = token.decimals
-
-    return tx

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -27,7 +27,7 @@ def database_engine(
         SessionBase.poolable = False
     dsn = dsn_from_config(load_config)
     #SessionBase.connect(dsn, True)
-    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
+    SessionBase.connect(dsn, debug=load_config.get('DATABASE_DEBUG') != None)
     return dsn
 
 

apps/cic-eth/cic_eth/queue/query.py

@@ -100,7 +100,6 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     q_outer = q_outer.join(Lock, isouter=True)
     q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
 
-
     if not is_alive(status):
         SessionBase.release_session(session)
         raise ValueError('not a valid non-final tx value: {}'.format(status))

apps/cic-eth/cic_eth/runnable/create.py

@@ -80,12 +80,7 @@ def main():
     t = api.create_account(register=register)
 
     ps.get_message()
-    try:
+    o = ps.get_message(timeout=args.timeout)
-        o = ps.get_message(timeout=args.timeout)
-    except TimeoutError as e:
-        sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e))
-        sys.exit(1) 
-    ps.unsubscribe()
     m = json.loads(o['data'])
     print(m['result'])
 

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -90,7 +90,6 @@ class DispatchSyncer:
 
     def __init__(self, chain_spec):
         self.chain_spec = chain_spec
-        self.session = None
 
 
     def chain(self):
@@ -101,18 +100,16 @@ class DispatchSyncer:
         c = len(txs.keys())
         logg.debug('processing {} txs {}'.format(c, list(txs.keys())))
         chain_str = str(self.chain_spec)
-        self.session = SessionBase.create_session()
+        session = SessionBase.create_session()
         for k in txs.keys():
             tx_raw = txs[k]
             tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
             tx = unpack(tx_raw_bytes, self.chain_spec)
             
             try:
-                set_reserved(self.chain_spec, tx['hash'], session=self.session)
+                set_reserved(self.chain_spec, tx['hash'], session=session)
-                self.session.commit()
             except NotLocalTxError as e:
                 logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash']))
-                self.session.rollback()
                 continue
 
             s_check = celery.signature(
@@ -135,25 +132,16 @@ class DispatchSyncer:
             s_check.link(s_send)
             t = s_check.apply_async()
             logg.info('processed {}'.format(k))
-        self.session.close()
-        self.session = None
 
 
-    def loop(self, interval):
+    def loop(self, w3, interval):
         while run:
             txs = {}
             typ = StatusBits.QUEUED
             utxs = get_upcoming_tx(self.chain_spec, typ)
             for k in utxs.keys():
                 txs[k] = utxs[k]
-            try:
+            self.process(w3, txs)
-                conn = RPCConnection.connect(self.chain_spec, 'default')
-                self.process(conn, txs)
-            except ConnectionError as e:
-                if self.session != None:
-                    self.session.close()
-                    self.session = None
-                logg.error('connection to node failed: {}'.format(e))
 
             if len(utxs) > 0:
                 time.sleep(self.yield_delay)
@@ -163,7 +151,8 @@ class DispatchSyncer:
 
 def main(): 
     syncer = DispatchSyncer(chain_spec)
-    syncer.loop(float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    conn = RPCConnection.connect(chain_spec, 'default')
+    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -11,7 +11,6 @@ from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
-from chainlib.eth.address import to_checksum_address
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -48,13 +47,12 @@ class GasFilter(SyncFilter):
 
             SessionBase.release_session(session)
 
-            address = to_checksum_address(r[0])
             logg.info('resuming gas-in-waiting txs for {}'.format(r[0]))
             if len(txs) > 0:
                 s = create_check_gas_task(
                         list(txs.values()),
                         self.chain_spec,
-                        address,
+                        r[0],
                         0,
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -12,24 +12,20 @@ from hexathon import (
 # local imports
 from .base import SyncFilter
 
-logg = logging.getLogger(__name__)
+logg = logging.getLogger().getChild(__name__)
 
 account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 
 
 class RegistrationFilter(SyncFilter):
 
-    def __init__(self, chain_spec, contract_address, queue=None):
+    def __init__(self, chain_spec, queue):
         self.chain_spec = chain_spec
         self.queue = queue
-        self.contract_address = contract_address
 
 
     def filter(self, conn, block, tx, db_session=None): 
-        if self.contract_address != tx.inputs[0]:
+        registered_address = None
-            logg.debug('not an account registry tx; {} != {}'.format(self.contract_address, tx.inputs[0]))
-            return None
-
         for l in tx.logs:
             event_topic_hex = l['topics'][0]
             if event_topic_hex == account_registry_add_log_hash:

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -78,14 +78,6 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
-rpc = RPCConnection.connect(chain_spec, 'default')
-registry = None
-try:
-    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-except UnknownContractError as e:
-    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-    sys.exit(1)
-logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 
 def main():
@@ -93,6 +85,7 @@ def main():
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # Connect to blockchain with chainlib
+    rpc = RPCConnection.connect(chain_spec, 'default')
 
     o = block_latest()
     r = rpc.do(o)
@@ -158,8 +151,7 @@ def main():
 
     tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
 
-    account_registry_address = registry.by_name('AccountRegistry')
+    registration_filter = RegistrationFilter(chain_spec, config.get('_CELERY_QUEUE'))
-    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE'))
 
     gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
 

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         12,
-        2,
+        0,
-        'alpha.3',
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/test/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=0
+DEBUG=

apps/cic-eth/docker/Dockerfile

@@ -1,71 +1,81 @@
-# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster as compile
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+WORKDIR /usr/src
-# do load these all into setup.py later
+
-# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+RUN apt-get update && \
-#COPY cic-eth/requirements.txt .
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+#RUN python -m venv venv && . venv/bin/activate
+
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN /usr/local/bin/python -m pip install --upgrade pip
+RUN pip install semver
+
+COPY cic-eth-aux/  ./cic-eth-aux/
+WORKDIR /usr/src/cic-eth-aux/erc20-demurrage-token
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    	--extra-index-url $EXTRA_INDEX_URL .
+
+WORKDIR /usr/src/cic-eth
+
+COPY cic-eth/ .
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    	--extra-index-url $EXTRA_INDEX_URL .[services]
+
+# --- TEST IMAGE ---
+FROM python:3.8.6-slim-buster as test 
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+WORKDIR /usr/src/cic-eth
+
+RUN /usr/local/bin/python -m pip install --upgrade pip
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+# TODO we could use venv inside container to isolate the system and app deps further
+# COPY --from=compile /usr/src/cic-eth/ .
+# RUN . venv/bin/activate
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+
-#    pip install --index-url https://pypi.org/simple \
+COPY cic-eth/test_requirements.txt .
-#    --force-reinstall \
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    	--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
-#    -r requirements.txt 
+
-COPY *requirements.txt ./
+COPY cic-eth . 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt \
-    -r services_requirements.txt \
-    -r admin_requirements.txt
-    
-COPY . .
-RUN python setup.py install
 
 ENV PYTHONPATH .
 
-COPY docker/entrypoints/* ./ 
+ENTRYPOINT ["pytest"]
+
+# --- RUNTIME ---
+FROM python:3.8.6-slim-buster as runtime 
+
+RUN apt-get update && \
+	apt install -y gnupg libpq-dev procps
+
+WORKDIR /usr/src/cic-eth
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+
+COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
 
+COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
+COPY cic-eth/config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+COPY util/liveness/health.sh /usr/local/bin/health.sh
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-ENTRYPOINT []
-
-## ------------------ PRODUCTION CONTAINER ----------------------
-#FROM python:3.8.6-slim-buster as prod 
-#
-#RUN apt-get update && \
-#	apt install -y gnupg libpq-dev procps
-#
-#WORKDIR /root
-#
-#COPY --from=dev /usr/local/bin/ /usr/local/bin/
-#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
-#                          /usr/local/lib/python3.8/site-packages/
-#
-#COPY docker/entrypoints/* ./ 
-#RUN chmod 755 *.sh
-#
-## # ini files in config directory defines the configurable parameters for the application
-## # they can all be overridden by environment variables
-## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-eth/
-#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-#COPY scripts/ scripts/ 
-#
-## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-##COPY util/liveness/health.sh /usr/local/bin/health.sh
-#
-#ENTRYPOINT []
 

apps/cic-eth/docker/Dockerfile_ci

@@ -1,69 +0,0 @@
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-WORKDIR /usr/src/cic-eth
-
-# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
-# do load these all into setup.py later
-# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
-#COPY cic-eth/requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-#    pip install --index-url https://pypi.org/simple \
-#    --force-reinstall \
-#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-#    -r requirements.txt 
-COPY *requirements.txt . 
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt \
-    -r services_requirements.txt \
-    -r admin_requirements.txt
-    
-COPY . .
-RUN python setup.py install
-
-COPY docker/entrypoints/* ./ 
-RUN chmod 755 *.sh
-
-# # ini files in config directory defines the configurable parameters for the application
-# # they can all be overridden by environment variables
-# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-eth/
-COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-
-# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-#COPY util/liveness/health.sh /usr/local/bin/health.sh
-ENTRYPOINT []
-
-# ------------------ PRODUCTION CONTAINER ----------------------
-#FROM python:3.8.6-slim-buster as prod 
-#
-#RUN apt-get update && \
-#	apt install -y gnupg libpq-dev procps
-#
-#WORKDIR /root
-#
-#COPY --from=dev /usr/local/bin/ /usr/local/bin/
-#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
-#                          /usr/local/lib/python3.8/site-packages/
-#
-#COPY docker/entrypoints/* ./ 
-#RUN chmod 755 *.sh
-#
-## # ini files in config directory defines the configurable parameters for the application
-## # they can all be overridden by environment variables
-## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-eth/
-#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-#COPY scripts/ scripts/ 
-#
-## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-##COPY util/liveness/health.sh /usr/local/bin/health.sh
-#
-#ENTRYPOINT []
-#

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.6a1,<0.1.0
+chainlib~=0.0.5a1
 semver==2.13.0

apps/cic-eth/services_requirements.txt

@@ -1,15 +1,14 @@
-chainqueue>=0.0.3a2,<0.1.0
+chainsyncer[sql]~=0.0.3a3
-chainsyncer[sql]>=0.0.5a1,<0.1.0
+chainqueue~=0.0.2b5
 alembic==1.4.2
-confini>=0.3.6rc4,<0.5.0
+confini~=0.3.6rc4
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.1.3a1,<0.2.0
+eth-address-index~=0.1.2a1
-eth-accounts-index>=0.0.13a1,<0.1.0
+eth-accounts-index~=0.0.12a1
-cic-eth-registry>=0.5.7a1,<0.6.0
+cic-eth-registry~=0.5.6a1
-erc20-faucet>=0.2.3a1,<0.3.0
+erc20-faucet~=0.2.2a1
-erc20-transfer-authorization>=0.3.3a1,<0.4.0
+sarafu-faucet~=0.0.4a1
-sarafu-faucet>=0.0.4a5,<0.1.0
 moolb~=0.1.1b2

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.0.11a1
+eth-erc20~=0.0.10a2

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -18,7 +18,6 @@ def test_filter_bogus(
         cic_registry,
         contract_roles,
         register_lookups,
-        account_registry,
         ):
 
     fltrs = [
@@ -27,7 +26,7 @@ def test_filter_bogus(
         TxFilter(default_chain_spec, None),
         CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
         StragglerFilter(default_chain_spec, None),
-        RegistrationFilter(default_chain_spec, account_registry, queue=None),
+        RegistrationFilter(default_chain_spec, queue=None),
         ]
       
     for fltr in fltrs:

apps/cic-eth/tests/filters/test_register_filter.py

@@ -1,7 +1,3 @@
-# standard imports
-import logging
-import os
-
 # external imports
 from eth_accounts_index.registry import AccountRegistry
 from chainlib.connection import RPCConnection
@@ -18,17 +14,12 @@ from chainlib.eth.block import (
         Block,
         )
 from erc20_faucet import Faucet
-from hexathon import (
+from hexathon import strip_0x
-        strip_0x,
-        add_0x,
-        )
 from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
 
-logg = logging.getLogger()
-
 
 def test_register_filter(
         default_chain_spec,
@@ -69,11 +60,7 @@ def test_register_filter(
     tx = Tx(tx_src, block=block, rcpt=rcpt)
     tx.apply_receipt(rcpt)
 
-    fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None)
+    fltr = RegistrationFilter(default_chain_spec, queue=None)
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    assert t == None
-
-    fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
 
     t.get_leaf()

apps/cic-eth/tests/task/api/test_admin.py

@@ -290,7 +290,6 @@ def test_fix_nonce(
     txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
     ks = txs.keys()
     assert len(ks) == 2
-
     for k in ks:
         hsh = add_0x(k)
         otx = Otx.load(hsh, session=init_database)

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -184,7 +184,7 @@ def test_admin_api_account(
 
 
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=True, include_recipient=True)
+    r = api.account(default_chain_spec, agent_roles['ALICE'])
     assert len(r) == 5
 
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])

apps/cic-eth/tests/task/test_task_list.py

@@ -1,92 +0,0 @@
-# external imports
-import celery
-import pytest
-from chainlib.connection import RPCConnection
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.gas import (
-        RPCGasOracle,
-        )
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from eth_erc20 import ERC20
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
-from chainqueue.db.models.tx import TxCache
-from chainqueue.db.models.otx import Otx
-
-
-def test_ext_tx_collate(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        custodial_roles,
-        agent_roles,
-        foo_token,
-        bar_token,
-        register_tokens,
-        cic_registry,
-        register_lookups,
-        init_celery_tasks,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], eth_rpc)
-    gas_oracle = RPCGasOracle(eth_rpc)
-
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    transfer_value_foo = 1000
-    transfer_value_bar = 1024
-    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], transfer_value_foo, tx_format=TxFormat.RLP_SIGNED)
-    tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
-
-    otx = Otx(
-        tx['nonce'],
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        )
-    init_database.add(otx)
-    init_database.commit()
-
-    txc = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        foo_token,
-        bar_token,
-        transfer_value_foo,
-        transfer_value_bar,
-        666,
-        13,
-        session=init_database,
-            )
-    init_database.add(txc)
-    init_database.commit()
-    
-    s = celery.signature(
-            'cic_eth.ext.tx.tx_collate',
-            [
-                {tx_hash_hex: tx_signed_raw_hex},
-                default_chain_spec.asdict(),
-                0,
-                100,
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    assert len(r) == 1
-
-    tx = r[0]
-    assert tx['source_token_symbol'] == 'FOO'
-    assert tx['source_token_decimals'] == 6
-    assert tx['destination_token_symbol'] == 'BAR'
-    assert tx['destination_token_decimals'] == 9

apps/cic-eth/tests/unit/db/test_nonce_db.py

@@ -1,7 +1,6 @@
-# extended imports
+# third-party imports
 import pytest
 import uuid
-import unittest
 
 # local imports
 from cic_eth.db.models.nonce import (
@@ -56,7 +55,7 @@ def test_nonce_reserve(
     o = q.first()
     assert o.nonce == 43
 
-    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu), session=init_database)
+    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu))
     init_database.commit()
     assert nonce == (str(uu), 42)
 

apps/cic-eth/tests/unit/ext/test_address.py

@@ -1,5 +1,4 @@
 # external imports
-import celery
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
@@ -21,7 +20,6 @@ def test_translate(
         cic_registry,
         init_celery_tasks,
         register_lookups,
-        celery_session_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
@@ -48,20 +46,6 @@ def test_translate(
         'recipient': agent_roles['BOB'],
         'recipient_label': None,
             }
-    
+    tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
-    #tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
+    assert tx['sender_label'] == 'alice'
-    s = celery.signature(
+    assert tx['recipient_label'] == 'bob'
-            'cic_eth.ext.address.translate_tx_addresses',
-            [
-                tx,
-                [contract_roles['CONTRACT_DEPLOYER']],
-                default_chain_spec.asdict(),
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    assert r['sender_label'] == 'alice'
-    assert r['recipient_label'] == 'bob'

apps/cic-eth/tests/unit/queue/test_query.py

@@ -9,11 +9,6 @@ from chainlib.eth.gas import (
         Gas,
         )
 from chainlib.chain import ChainSpec
-from hexathon import (
-        add_0x,
-        strip_0x,
-        uniform as hex_uniform,
-        )
 
 # local imports
 from cic_eth.db.enum import LockEnum
@@ -39,10 +34,7 @@ def test_upcoming_with_lock(
     gas_oracle = RPCGasOracle(eth_rpc)
     c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
 
-    alice_normal = add_0x(hex_uniform(strip_0x(agent_roles['ALICE'])))
+    (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6))
-    bob_normal = add_0x(hex_uniform(strip_0x(agent_roles['BOB'])))
-
-    (tx_hash_hex, tx_rpc) = c.create(alice_normal, bob_normal, 100 * (10 ** 6))
     tx_signed_raw_hex = tx_rpc['params'][0]
 
     register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
@@ -51,12 +43,12 @@ def test_upcoming_with_lock(
     txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
     assert len(txs.keys()) == 1
 
-    Lock.set(str(default_chain_spec), LockEnum.SEND, address=alice_normal)
+    Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE'])
 
-    txs = get_upcoming_tx(default_chain_spec, status=StatusEnum.PENDING)
+    txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
     assert len(txs.keys()) == 0
 
-    (tx_hash_hex, tx_rpc) = c.create(bob_normal, alice_normal, 100 * (10 ** 6))
+    (tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6))
     tx_signed_raw_hex = tx_rpc['params'][0]
 
     register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)

apps/cic-eth/tools_requirements.txt

@@ -1,7 +1,7 @@
-crypto-dev-signer>=0.4.14b7,<=0.4.14
+crypto-dev-signer~=0.4.14b6
-chainqueue~=0.0.2b6
+chainqueue~=0.0.2b5
-confini>=0.3.6rc4,<0.5.0
+confini~=0.3.6rc4
-cic-eth-registry>=0.5.7a1,<0.6.0
+cic-eth-registry~=0.5.6a1
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1

apps/cic-meta/.dockerignore

@@ -1,5 +0,0 @@
-.git
-.cache
-.dot
-**/doc
-**/node_modules

apps/cic-meta/.gitlab-ci.yml

@@ -2,42 +2,43 @@
 .cic_meta_variables:
     variables:
         APP_NAME: cic-meta
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
 
-build-mr-cic-meta:
+.cic_meta_changes_target:
-    extends:
-        - .py_build_merge_request
-        - .cic_meta_variables
     rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
+#        - changes:
-          - apps/cic-meta/**/*
+#            - $CONTEXT/$APP_NAME/*
-      when: always
+        - when: always
+
+cic-meta-build-mr:
+  stage: build
+  extends:
+    - .cic_meta_variables
+    - .cic_meta_changes_target
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
 
 test-mr-cic-meta:
     extends:
         - .cic_meta_variables
+        - .cic_meta_changes_target
     stage: test
-    image: $MR_IMAGE_TAG 
+    image: $IMAGE_TAG 
     script:
-        - cd /root
+        - cd /tmp/src/cic-meta
         - npm install --dev 
         - npm run test
         - npm run test:coverage
-    needs: ["build-mr-cic-meta"]
+    needs: ["cic-meta-build-mr"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-meta/**/*
-      when: always
 
 build-push-cic-meta:
     extends:
         - .py_build_push
         - .cic_meta_variables
-    rules:
+        
-    - if:  $CI_COMMIT_BRANCH == "master"
+
-      changes:
-          - apps/cic-meta/**/*
-      when: always

apps/cic-meta/docker/Dockerfile

@@ -1,33 +1,31 @@
-# syntax = docker/dockerfile:1.2
+FROM node:15.3.0-alpine3.10
-#FROM node:15.3.0-alpine3.10
-FROM node:lts-alpine3.14
 
-WORKDIR /root
+WORKDIR /tmp/src/cic-meta
 
 RUN apk add --no-cache postgresql bash
 
+# required to build the cic-client-meta module
+COPY cic-meta/src/ src/
+COPY cic-meta/scripts/ scripts/
+
 # copy the dependencies
-COPY package.json package-lock.json .
+COPY cic-meta/package.json .
-RUN --mount=type=cache,mode=0755,target=/root/.npm \
+COPY cic-meta/tsconfig.json . 
-    npm set cache /root/.npm && \
+COPY cic-meta/webpack.config.js .
-    npm ci
+     
+RUN npm install
 
-COPY webpack.config.js .
+COPY cic-meta/tests/ tests/
-COPY tsconfig.json . 
+COPY cic-meta/tests/*.asc /root/pgp/
-## required to build the cic-client-meta module
-COPY src/ src/
-COPY scripts/ scripts/
-COPY tests/ tests/
-COPY tests/*.asc /root/pgp/
 
-## copy runtime configs
+# copy runtime configs
-COPY .config/ /usr/local/etc/cic-meta/
+COPY cic-meta/.config/ /usr/local/etc/cic-meta/
-#
+
-## db migrations
+# db migrations
-COPY docker/db.sh ./db.sh
+COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
-#
+
 RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY docker/start_server.sh ./start_server.sh
+COPY cic-meta/docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-#FROM node:15.3.0-alpine3.10
-FROM node:lts-alpine3.14
-
-WORKDIR /root
-
-RUN apk add --no-cache postgresql bash
-
-# copy the dependencies
-COPY package.json package-lock.json .
-RUN npm set cache /root/.npm && \
-    npm ci
-
-COPY webpack.config.js .
-COPY tsconfig.json . 
-## required to build the cic-client-meta module
-COPY src/ src/
-COPY scripts/ scripts/
-COPY tests/ tests/
-COPY tests/*.asc /root/pgp/
-
-## copy runtime configs
-COPY .config/ /usr/local/etc/cic-meta/
-#
-## db migrations
-COPY docker/db.sh ./db.sh
-RUN chmod 755 ./db.sh
-#
-RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY docker/start_server.sh ./start_server.sh
-RUN chmod 755 ./start_server.sh
-ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-notify/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_notify_variables:
     variables:
         APP_NAME: cic-notify
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_notify_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-notify:
     extends:
+        - .cic_notify_changes_target
         - .py_build_merge_request
         - .cic_notify_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-notify/**/*
-      when: always
-
-test-mr-cic-notify:
-    stage: test
-    extends:
-        - .cic_notify_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-notify"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
 
 build-push-cic-notify:
     extends:
         - .py_build_push
         - .cic_notify_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-notify/**/*
-      when: always
         
 

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -34,8 +34,6 @@ elif args.v:
 config = confini.Config(args.c, args.env_prefix)
 config.process()
 config.add(args.q, '_CELERY_QUEUE', True)
-config.censor('API_KEY', 'AFRICASTALKING')
-config.censor('API_USERNAME', 'AFRICASTALKING')
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.10')
+version = (0, 4, 0, 'alpha.7')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/docker/Dockerfile

@@ -1,28 +1,38 @@
-# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
 
-#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+WORKDIR /usr/src/cic-notify
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-COPY requirements.txt .
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-    pip install --index-url https://pypi.org/simple  \
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
 
-COPY . .
+COPY cic-notify/setup.cfg \
-RUN python setup.py install
+	 cic-notify/setup.py \
+     ./
 
-COPY docker/*.sh .
+COPY cic-notify/cic_notify/ ./cic_notify/
-RUN chmod +x *.sh 
+
+COPY cic-notify/requirements.txt \
+	 cic-notify/test_requirements.txt \
+     ./
+
+COPY cic-notify/scripts/ scripts/
+RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog]
+
+COPY cic-notify/tests/ tests/
+COPY cic-notify/docker/db.sh \
+     cic-notify/docker/start_tasker.sh \
+	 /root/
+
+#RUN apk add postgresql-client
+#RUN apk add bash
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY .config/ /usr/local/etc/cic-notify/
+COPY cic-notify/.config/ /usr/local/etc/cic-notify/
-COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+COPY cic-notify/cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
 
-
+WORKDIR /root
-ENTRYPOINT []

apps/cic-notify/docker/Dockerfile_ci

@@ -1,27 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-COPY requirements.txt .
-
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . .
-
-RUN python setup.py install
-
-COPY docker/*.sh .
-RUN chmod +x *.sh 
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY .config/ /usr/local/etc/cic-notify/
-COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
-
-ENTRYPOINT []

apps/cic-notify/docker/db.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-set -e
+
-python scripts/migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv
+migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv

apps/cic-notify/docker/start_tasker.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-. /root/db.sh
+. ./db.sh
 
 /usr/local/bin/cic-notify-tasker -vv $@

apps/cic-notify/requirements.txt

@@ -1,7 +1 @@
-confini~=0.4.1a1
+cic_base[full_graph]==0.1.3a3+build.984b5cff
-africastalking==1.2.3
-SQLAlchemy==1.3.20
-alembic==1.4.2
-psycopg2==2.8.6
-celery==4.4.7
-redis==3.5.3

apps/cic-notify/scripts/migrate.py

@@ -35,10 +35,9 @@ elif args.v:
 
 config = confini.Config(args.c, args.env_prefix)
 config.process()
-config.censor('API_KEY', 'AFRICASTALKING')
-config.censor('API_USERNAME', 'AFRICASTALKING')
 config.censor('PASSWORD', 'DATABASE')
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
+#config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
 
 migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
 if not os.path.isdir(migrations_dir):

apps/cic-notify/setup.cfg

@@ -29,11 +29,18 @@ packages =
 	cic_notify.db
 	cic_notify.db.models
 	cic_notify.ext
-	cic_notify.tasks
 	cic_notify.tasks.sms
 	cic_notify.runnable
 scripts =
-	./scripts/migrate.py
+	scripts/migrate.py
+[options.extras_require]
+africastalking = africastalking==1.2.3
+notifylog = psycopg2==2.8.6
+testing =
+	pytest==6.0.1
+	pytest-celery==0.0.0a1
+	pytest-mock==3.3.1
+	pysqlite3==0.4.3
 
 [options.entry_points]
 console_scripts =

apps/cic-notify/test_requirements.txt

@@ -2,4 +2,3 @@ pytest~=6.0.1
 pytest-celery~=0.0.0a1
 pytest-mock~=3.3.1
 pysqlite3~=0.4.3
-pytest-cov==2.10.1

apps/cic-ussd/.config/app.ini

@@ -1,7 +1,7 @@
 [app]
 ALLOWED_IP=0.0.0.0/0
-LOCALE_FALLBACK=sw
+LOCALE_FALLBACK=en
-LOCALE_PATH=var/lib/locale/
+LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#,*483*061#,*384*96#
@@ -11,13 +11,13 @@ SUPPORT_PHONE_NUMBER=0757628885
 REGION=KE
 
 [ussd]
-MENU_FILE=data/ussd_menu.json
+MENU_FILE=/usr/src/data/ussd_menu.json
 user = 
 pass = 
 
 [statemachine]
-STATES=states/
+STATES=/usr/src/cic-ussd/states/
-TRANSITIONS=transitions/
+TRANSITIONS=/usr/src/cic-ussd/transitions/
 
 [client]
 host =

apps/cic-ussd/.config/pgp.ini

@@ -1,5 +1,5 @@
 [pgp]
-export_dir = pgp/keys/
+export_dir = /usr/src/pgp/keys/
 keys_path = /usr/src/secrets/
 private_keys = privatekeys_meta.asc
 passphrase =

apps/cic-ussd/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-ussd/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_ussd_variables:
     variables:
         APP_NAME: cic-ussd
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+
+.cic_ussd_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-ussd:
     extends:
+        - .cic_ussd_changes_target
         - .py_build_merge_request
         - .cic_ussd_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-ussd/**/*
-      when: always
-
-test-mr-cic-ussd:
-    stage: test
-    extends:
-        - .cic_ussd_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-ussd"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
 
 build-push-cic-ussd:
     extends:
         - .py_build_push
         - .cic_ussd_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-ussd/**/*
-      when: always
 
 

apps/cic-ussd/cic_ussd/balance.py

@@ -15,47 +15,45 @@ from cic_ussd.conversions import from_wei
 logg = logging.getLogger()
 
 
-def get_balances(
+class BalanceManager:
-        address: str,
+
-        chain_str: str,
+    def __init__(self, address: str, chain_str: str, token_symbol: str):
-        token_symbol: str,
+        """
-        asynchronous: bool = False,
+        :param address: Ethereum address of account whose balance is being queried
-        callback_param: any = None,
+        :type address: str, 0x-hex
-        callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Union[celery.Task, dict]:
+        :param chain_str: The chain name and network id.
-    """
+        :type chain_str: str
-    This function queries cic-eth for an account's balances, It provides a means to receive the balance either
+        :param token_symbol: ERC20 token symbol of whose balance is being queried
-    asynchronously or synchronously depending on the provided value for teh asynchronous parameter. It returns a
+        :type token_symbol: str
-    dictionary containing network, outgoing and incoming balances.
+        """
-    :param address: Ethereum address of the recipient
+        self.address = address
-    :type address: str, 0x-hex
+        self.chain_str = chain_str
-    :param chain_str: The chain name and network id.
+        self.token_symbol = token_symbol
-    :type chain_str: str
+
-    :param callback_param:
+    def get_balances(self, asynchronous: bool = False) -> Union[celery.Task, dict]:
-    :type callback_param:
+        """
-    :param callback_task:
+        This function queries cic-eth for an account's balances, It provides a means to receive the balance either
-    :type callback_task:
+        asynchronously or synchronously depending on the provided value for teh asynchronous parameter. It returns a
-    :param token_symbol: ERC20 token symbol of the account whose balance is being queried.
+        dictionary containing network, outgoing and incoming balances.
-    :type token_symbol: str
+        :param asynchronous: Boolean value checking whether to return balances asynchronously
-    :param asynchronous: Boolean value checking whether to return balances asynchronously.
+        :type asynchronous: bool
-    :type asynchronous: bool
+        :return:
-    :return:
+        :rtype:
-    :rtype:
+        """
-    """
+        if asynchronous:
-    logg.debug(f'Retrieving balance for address: {address}')
+            cic_eth_api = Api(
-    if asynchronous:
+                chain_str=self.chain_str,
-        cic_eth_api = Api(
+                callback_queue='cic-ussd',
-            chain_str=chain_str,
+                callback_task='cic_ussd.tasks.callback_handler.process_balances_callback',
-            callback_queue='cic-ussd',
+                callback_param=''
-            callback_task=callback_task,
+            )
-            callback_param=callback_param
+            cic_eth_api.balance(address=self.address, token_symbol=self.token_symbol)
-        )
+        else:
-        cic_eth_api.balance(address=address, token_symbol=token_symbol)
+            cic_eth_api = Api(chain_str=self.chain_str)
-    else:
+            balance_request_task = cic_eth_api.balance(
-        cic_eth_api = Api(chain_str=chain_str)
+                address=self.address,
-        balance_request_task = cic_eth_api.balance(
+                token_symbol=self.token_symbol)
-            address=address,
+            return balance_request_task.get()[0]
-            token_symbol=token_symbol)
-        return balance_request_task.get()[0]
 
 
 def compute_operational_balance(balances: dict) -> float:

apps/cic-ussd/cic_ussd/db/models/base.py

@@ -127,4 +127,3 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-ussd/cic_ussd/error.py

@@ -48,6 +48,3 @@ class InitializationError(Exception):
     pass
 
 
-class UnknownUssdRecipient(Exception):
-    """Raised when a recipient of a transaction is not known to the ussd application."""
-

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -127,19 +127,14 @@ class MetadataRequestsHandler(Metadata):
         if not isinstance(result_data, dict):
             raise ValueError(f'Invalid result data object: {result_data}.')
 
-        if result.status_code == 200:
+        if result.status_code == 200 and self.cic_type == ':cic.person':
-            if self.cic_type == ':cic.person':
+            # validate person metadata
-                # validate person metadata
+            person = Person()
-                person = Person()
+            person_data = person.deserialize(person_data=result_data)
-                person_data = person.deserialize(person_data=result_data)
 
-                # format new person data for caching
+            # format new person data for caching
-                serialized_person_data = person_data.serialize()
+            data = json.dumps(person_data.serialize())
-                data = json.dumps(serialized_person_data)
-            else:
-                data = json.dumps(result_data)
 
             # cache metadata
             cache_data(key=self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
-        return result_data

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -1,7 +1,6 @@
 # standard imports
 
 # external imports
-import celery
 
 # local imports
 from .base import MetadataRequestsHandler

apps/cic-ussd/cic_ussd/processor.py

@@ -12,7 +12,7 @@ from tinydb.table import Document
 
 # local imports
 from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement
-from cic_ussd.balance import compute_operational_balance, get_balances, get_cached_operational_balance
+from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
@@ -182,6 +182,7 @@ def process_transaction_pin_authorization(account: Account, display_key: str, se
     token_symbol = retrieve_token_symbol()
     user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
+    logg.debug('Requires integration to determine user tokens.')
     return process_pin_authorization(
         account=account,
         display_key=display_key,
@@ -379,9 +380,12 @@ def process_start_menu(display_key: str, user: Account):
     token_symbol = retrieve_token_symbol()
     chain_str = Chain.spec.__str__()
     blockchain_address = user.blockchain_address
+    balance_manager = BalanceManager(address=blockchain_address,
+                                     chain_str=chain_str,
+                                     token_symbol=token_symbol)
 
     # get balances synchronously for display on start menu
-    balances_data = get_balances(address=blockchain_address, chain_str=chain_str, token_symbol=token_symbol)
+    balances_data = balance_manager.get_balances()
 
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -28,7 +28,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(args.c, env_prefix=args.env_prefix)
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py

@@ -8,16 +8,13 @@ import tempfile
 import celery
 import i18n
 import redis
-from chainlib.chain import ChainSpec
 from confini import Config
 
 # local imports
-from cic_ussd.chain import Chain
 from cic_ussd.db import dsn_from_config
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.metadata.signer import Signer
 from cic_ussd.metadata.base import Metadata
-from cic_ussd.phone_number import Support
 from cic_ussd.redis import InMemoryStore
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.validator import validate_presence
@@ -85,15 +82,6 @@ Signer.key_file_path = key_file_path
 i18n.load_path.append(config.get('APP_LOCALE_PATH'))
 i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
 
-chain_spec = ChainSpec(
-    common_name=config.get('CIC_COMMON_NAME'),
-    engine=config.get('CIC_ENGINE'),
-    network_id=config.get('CIC_NETWORK_ID')
-)
-
-Chain.spec = chain_spec
-Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER')
-
 # set up celery
 current_app = celery.Celery(__name__)
 

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -45,7 +45,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(args.c, env_prefix=args.env_prefix)
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/cic_ussd/tasks/__init__.py

@@ -1,4 +1,8 @@
 # standard import
+import os
+import logging
+import urllib
+import json
 
 # third-party imports
 # this must be included for the package to be recognized as a tasks package
@@ -10,5 +14,3 @@ from .logger import *
 from .ussd_session import *
 from .callback_handler import *
 from .metadata import *
-from .notifications import *
-from .processor import *

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -7,14 +7,14 @@ from datetime import datetime, timedelta
 import celery
 
 # local imports
-from cic_ussd.balance import compute_operational_balance, get_balances
-from cic_ussd.chain import Chain
 from cic_ussd.conversions import from_wei
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.account import Account
+from cic_ussd.account import define_account_tx_metadata
 from cic_ussd.error import ActionDataNotFoundError
-from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key, get_cached_data
+from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key
 from cic_ussd.tasks.base import CriticalSQLAlchemyTask
+from cic_ussd.transactions import IncomingTransactionProcessor
 
 logg = logging.getLogger(__file__)
 celery_app = celery.current_app
@@ -58,9 +58,9 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
 
             # add phone number metadata lookup
             s_phone_pointer = celery.signature(
-                'cic_ussd.tasks.metadata.add_phone_pointer',
+                    'cic_ussd.tasks.metadata.add_phone_pointer',
-                [result, phone_number]
+                    [result, phone_number]
-            )
+                    )
             s_phone_pointer.apply_async(queue=queue)
 
             # add custom metadata tags
@@ -87,106 +87,59 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
     session.close()
 
 
-@celery_app.task(bind=True)
+@celery_app.task
-def process_transaction_callback(self, result: dict, param: str, status_code: int):
+def process_incoming_transfer_callback(result: dict, param: str, status_code: int):
+    session = SessionBase.create_session()
     if status_code == 0:
-        chain_str = Chain.spec.__str__()
 
-        # collect transaction metadata
+        # collect result data
-        destination_token_symbol = result.get('destination_token_symbol')
-        destination_token_value = result.get('destination_token_value')
         recipient_blockchain_address = result.get('recipient')
         sender_blockchain_address = result.get('sender')
-        source_token_symbol = result.get('source_token_symbol')
+        token_symbol = result.get('destination_token_symbol')
-        source_token_value = result.get('source_token_value')
+        value = result.get('destination_token_value')
 
-        # build stakeholder callback params
+        # try to find users in system
-        recipient_metadata = {
+        recipient_user = session.query(Account).filter_by(blockchain_address=recipient_blockchain_address).first()
-            "token_symbol": destination_token_symbol,
+        sender_user = session.query(Account).filter_by(blockchain_address=sender_blockchain_address).first()
-            "token_value": destination_token_value,
-            "blockchain_address": recipient_blockchain_address,
-            "tag": "recipient",
-            "tx_param": param
-        }
 
-        # retrieve account balances
+        # check whether recipient is in the system
-        get_balances(
+        if not recipient_user:
-            address=recipient_blockchain_address,
+            session.close()
-            callback_param=recipient_metadata,
+            raise ValueError(
-            chain_str=chain_str,
+                f'Tx for recipient: {recipient_blockchain_address} was received but has no matching user in the system.'
-            callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback',
-            token_symbol=destination_token_symbol,
-            asynchronous=True)
-
-        # only retrieve sender if transaction is a transfer
-        if param == 'transfer':
-            sender_metadata = {
-                "blockchain_address": sender_blockchain_address,
-                "token_symbol": source_token_symbol,
-                "token_value": source_token_value,
-                "tag": "sender",
-                "tx_param": param
-            }
-
-            get_balances(
-                address=sender_blockchain_address,
-                callback_param=sender_metadata,
-                chain_str=chain_str,
-                callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback',
-                token_symbol=source_token_symbol,
-                asynchronous=True)
-    else:
-        raise ValueError(f'Unexpected status code: {status_code}.')
-
-
-@celery_app.task(bind=True)
-def process_transaction_balances_callback(self, result: list, param: dict, status_code: int):
-    queue = self.request.delivery_info.get('routing_key')
-    if status_code == 0:
-        # retrieve balance data
-        balances_data = result[0]
-        operational_balance = compute_operational_balance(balances=balances_data)
-
-        # retrieve account's address
-        blockchain_address = param.get('blockchain_address')
-
-        # append balance to transaction metadata
-        transaction_metadata = param
-        transaction_metadata['operational_balance'] = operational_balance
-
-        # retrieve account's preferences
-        s_preferences_metadata = celery.signature(
-            'cic_ussd.tasks.metadata.query_preferences_metadata',
-            [blockchain_address],
-            queue=queue
-        )
-
-        # parse metadata and run validations
-        s_process_account_metadata = celery.signature(
-            'cic_ussd.tasks.processor.process_tx_metadata_for_notification',
-            [transaction_metadata],
-            queue=queue
-        )
-
-        # issue notification of transaction
-        s_notify_account = celery.signature(
-            'cic_ussd.tasks.notifications.notify_account_of_transaction',
-            queue=queue
-        )
-
-        if param.get('tx_param') == 'transfer':
-            celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async()
-
-        if param.get('tx_param') == 'tokengift':
-            s_process_account_metadata = celery.signature(
-                'cic_ussd.tasks.processor.process_tx_metadata_for_notification',
-                [{}, transaction_metadata],
-                queue=queue
             )
-            celery.chain(s_process_account_metadata, s_notify_account).apply_async()
+
+        # process incoming transactions
+        incoming_tx_processor = IncomingTransactionProcessor(phone_number=recipient_user.phone_number,
+                                                             preferred_language=recipient_user.preferred_language,
+                                                             token_symbol=token_symbol,
+                                                             value=value)
+
+        if param == 'tokengift':
+            incoming_tx_processor.process_token_gift_incoming_transactions()
+        elif param == 'transfer':
+            if sender_user:
+                sender_information = define_account_tx_metadata(user=sender_user)
+                incoming_tx_processor.process_transfer_incoming_transaction(
+                    sender_information=sender_information,
+                    recipient_blockchain_address=recipient_blockchain_address
+                )
+            else:
+                logg.warning(
+                    f'Tx with sender: {sender_blockchain_address} was received but has no matching user in the system.'
+                )
+                incoming_tx_processor.process_transfer_incoming_transaction(
+                    sender_information='GRASSROOTS ECONOMICS',
+                    recipient_blockchain_address=recipient_blockchain_address
+                )
+        else:
+            session.close()
+            raise ValueError(f'Unexpected transaction param: {param}.')
     else:
+        session.close()
         raise ValueError(f'Unexpected status code: {status_code}.')
 
+    session.close()
 
 @celery_app.task
 def process_balances_callback(result: list, param: str, status_code: int):
@@ -198,7 +151,6 @@ def process_balances_callback(result: list, param: str, status_code: int):
             salt=':cic.balances_data'
         )
         cache_data(key=key, data=json.dumps(balances_data))
-        logg.debug(f'caching: {balances_data} with key: {key}')
     else:
         raise ValueError(f'Unexpected status code: {status_code}.')
 

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -26,7 +26,6 @@ def query_person_metadata(blockchain_address: str):
     :rtype:
     """
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
-    logg.debug(f'Retrieving person metadata for address: {blockchain_address}.')
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.query()
 
@@ -73,15 +72,3 @@ def add_preferences_metadata(blockchain_address: str, data: dict):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
     custom_metadata_client = PreferencesMetadata(identifier=identifier)
     custom_metadata_client.create(data=data)
-
-
-@celery_app.task()
-def query_preferences_metadata(blockchain_address: str):
-    """This method retrieves preferences metadata based on an account's blockchain address.
-    :param blockchain_address: Blockchain address of an account.
-    :type blockchain_address: str | Ox-hex
-    """
-    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
-    logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
-    person_metadata_client = PreferencesMetadata(identifier=identifier)
-    return person_metadata_client.query()

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -1,70 +0,0 @@
-# standard imports
-import datetime
-import logging
-
-# third-party imports
-import celery
-
-# local imports
-from cic_ussd.notifications import Notifier
-from cic_ussd.phone_number import Support
-
-celery_app = celery.current_app
-logg = logging.getLogger(__file__)
-notifier = Notifier()
-
-
-@celery_app.task
-def notify_account_of_transaction(notification_data: dict):
-    """
-    :param notification_data:
-    :type notification_data:
-    :return:
-    :rtype:
-    """
-
-    account_tx_role = notification_data.get('account_tx_role')
-    amount = notification_data.get('amount')
-    balance = notification_data.get('balance')
-    phone_number = notification_data.get('phone_number')
-    preferred_language = notification_data.get('preferred_language')
-    token_symbol = notification_data.get('token_symbol')
-    transaction_account_metadata = notification_data.get('transaction_account_metadata')
-    transaction_type = notification_data.get('transaction_type')
-
-    timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
-
-    if transaction_type == 'tokengift':
-        support_phone = Support.phone_number
-        notifier.send_sms_notification(
-            key='sms.account_successfully_created',
-            phone_number=phone_number,
-            preferred_language=preferred_language,
-            balance=balance,
-            support_phone=support_phone,
-            token_symbol=token_symbol
-        )
-
-    if transaction_type == 'transfer':
-        if account_tx_role == 'recipient':
-            notifier.send_sms_notification(
-                key='sms.received_tokens',
-                phone_number=phone_number,
-                preferred_language=preferred_language,
-                amount=amount,
-                token_symbol=token_symbol,
-                tx_sender_information=transaction_account_metadata,
-                timestamp=timestamp,
-                balance=balance
-            )
-        else:
-            notifier.send_sms_notification(
-                key='sms.sent_tokens',
-                phone_number=phone_number,
-                preferred_language=preferred_language,
-                amount=amount,
-                token_symbol=token_symbol,
-                tx_recipient_information=transaction_account_metadata,
-                timestamp=timestamp,
-                balance=balance
-            )

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -1,88 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-from i18n import config
-
-# local imports
-from cic_ussd.account import define_account_tx_metadata
-from cic_ussd.db.models.account import Account
-from cic_ussd.db.models.base import SessionBase
-from cic_ussd.error import UnknownUssdRecipient
-from cic_ussd.transactions import from_wei
-
-
-celery_app = celery.current_app
-logg = logging.getLogger(__file__)
-
-
-@celery_app.task
-def process_tx_metadata_for_notification(result: celery.Task, transaction_metadata: dict):
-    """
-    :param result:
-    :type result:
-    :param transaction_metadata:
-    :type transaction_metadata:
-    :return:
-    :rtype:
-    """
-    notification_data = {}
-
-    # get preferred language
-    preferred_language = result.get('preferred_language')
-    if not preferred_language:
-        preferred_language = config.get('fallback')
-    notification_data['preferred_language'] = preferred_language
-
-    # validate account information against present ussd storage data.
-    session = SessionBase.create_session()
-    blockchain_address = transaction_metadata.get('blockchain_address')
-    tag = transaction_metadata.get('tag')
-    account = session.query(Account).filter_by(blockchain_address=blockchain_address).first()
-    if not account and tag == 'recipient':
-        session.close()
-        raise UnknownUssdRecipient(
-            f'Tx for recipient: {blockchain_address} was received but has no matching user in the system.'
-        )
-
-    # get phone number associated with account
-    phone_number = account.phone_number
-    notification_data['phone_number'] = phone_number
-
-    # get account's role in transaction i.e sender / recipient
-    tx_param = transaction_metadata.get('tx_param')
-    notification_data['transaction_type'] = tx_param
-
-    # get token amount and symbol
-    if tag == 'recipient':
-        account_tx_role = tag
-        amount = transaction_metadata.get('token_value')
-        amount = from_wei(value=amount)
-        token_symbol = transaction_metadata.get('token_symbol')
-    else:
-        account_tx_role = tag
-        amount = transaction_metadata.get('token_value')
-        amount = from_wei(value=amount)
-        token_symbol = transaction_metadata.get('token_symbol')
-    notification_data['account_tx_role'] = account_tx_role
-    notification_data['amount'] = amount
-    notification_data['token_symbol'] = token_symbol
-
-    # get account's standard ussd identification pattern
-    if tx_param == 'transfer':
-        tx_account_metadata = define_account_tx_metadata(user=account)
-        notification_data['transaction_account_metadata'] = tx_account_metadata
-
-        if tag == 'recipient':
-            notification_data['notification_key'] = 'sms.received_tokens'
-        else:
-            notification_data['notification_key'] = 'sms.sent_tokens'
-
-    if tx_param == 'tokengift':
-        notification_data['notification_key'] = 'sms.account_successfully_created'
-
-    # get account's balance
-    notification_data['balance'] = transaction_metadata.get('operational_balance')
-
-    return notification_data

apps/cic-ussd/cic_ussd/transactions.py

@@ -7,9 +7,9 @@ from datetime import datetime
 from cic_eth.api import Api
 
 # local imports
-from cic_ussd.balance import get_balances, get_cached_operational_balance
+from cic_ussd.balance import get_cached_operational_balance
 from cic_ussd.notifications import Notifier
-from cic_ussd.phone_number import Support
+
 
 logg = logging.getLogger()
 notifier = Notifier()
@@ -50,6 +50,61 @@ def to_wei(value: int) -> int:
     return int(value * 1e+6)
 
 
+class IncomingTransactionProcessor:
+
+    def __init__(self, phone_number: str, preferred_language: str, token_symbol: str, value: int):
+        """
+        :param phone_number: The recipient's phone number.
+        :type phone_number: str
+        :param preferred_language: The user's preferred language.
+        :type preferred_language: str
+        :param token_symbol: The symbol for the token the recipient receives.
+        :type token_symbol: str
+        :param value: The amount of tokens received in the transactions.
+        :type value: int
+        """
+        self.phone_number = phone_number
+        self.preferred_language = preferred_language
+        self.token_symbol = token_symbol
+        self.value = value
+
+    def process_token_gift_incoming_transactions(self):
+        """This function processes incoming transactions with a "tokengift" param, it collects all appropriate data to
+        send out notifications to users when their accounts are successfully created.
+
+        """
+        balance = from_wei(value=self.value)
+        key = 'sms.account_successfully_created'
+        notifier.send_sms_notification(key=key,
+                                       phone_number=self.phone_number,
+                                       preferred_language=self.preferred_language,
+                                       balance=balance,
+                                       token_symbol=self.token_symbol)
+
+    def process_transfer_incoming_transaction(self, sender_information: str, recipient_blockchain_address: str):
+        """This function processes incoming transactions with the "transfer" param and issues notifications to users
+        about reception of funds into their accounts.
+        :param sender_information: A string with a user's full name and phone number.
+        :type sender_information: str
+        :param recipient_blockchain_address:
+        type recipient_blockchain_address: str
+        """
+        key = 'sms.received_tokens'
+        amount = from_wei(value=self.value)
+        timestamp = datetime.now().strftime('%d-%m-%y, %H:%M %p')
+
+        operational_balance = get_cached_operational_balance(blockchain_address=recipient_blockchain_address)
+
+        notifier.send_sms_notification(key=key,
+                                       phone_number=self.phone_number,
+                                       preferred_language=self.preferred_language,
+                                       amount=amount,
+                                       token_symbol=self.token_symbol,
+                                       tx_sender_information=sender_information,
+                                       timestamp=timestamp,
+                                       balance=operational_balance)
+
+
 class OutgoingTransactionProcessor:
 
     def __init__(self, chain_str: str, from_address: str, to_address: str):
@@ -61,7 +116,6 @@ class OutgoingTransactionProcessor:
         :param to_address: Ethereum address of the recipient
         :type to_address: str, 0x-hex
         """
-        self.chain_str = chain_str
         self.cic_eth_api = Api(chain_str=chain_str)
         self.from_address = from_address
         self.to_address = to_address

apps/cic-ussd/docker/Dockerfile

@@ -1,5 +1,14 @@
-# syntax = docker/dockerfile:1.2
+# FROM python:3.8.5-alpine
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+FROM python:3.8.6-slim-buster
+
+# set working directory
+WORKDIR /usr/src
+
+# add args for installing from self-hosted packages
+ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
 # create secrets directory
 RUN mkdir -vp pgp/keys
@@ -8,25 +17,39 @@ RUN mkdir -vp pgp/keys
 RUN mkdir -vp cic-ussd
 RUN mkdir -vp data
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+COPY cic-ussd/setup.cfg \
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+     cic-ussd/setup.py \
-COPY requirements.txt .
+     cic-ussd/
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+COPY cic-ussd/requirements.txt \
-    pip install --index-url https://pypi.org/simple  \
+     cic-ussd/test_requirements.txt \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+     cic-ussd/
-    -r requirements.txt
 
-COPY . .
+# install requirements
-RUN python setup.py install
+RUN cd cic-ussd && \
+    pip install -r requirements.txt $pip_extra_index_url_flag
 
-COPY cic_ussd/db/ussd_menu.json data/
+# copy all necessary files
+COPY cic-ussd/cic_ussd/ cic-ussd/cic_ussd/
+COPY cic-ussd/cic_ussd/db/ussd_menu.json data/
+COPY cic-ussd/scripts/ cic-ussd/scripts/
+COPY cic-ussd/states/ cic-ussd/states/
+COPY cic-ussd/transitions/ cic-ussd/transitions/
+COPY cic-ussd/var/ cic-ussd/var/
+
+COPY cic-ussd/docker/db.sh \
+     cic-ussd/docker/start_cic_user_tasker.sh \
+     cic-ussd/docker/start_cic_user_ussd_server.sh\
+     cic-ussd/docker/start_cic_user_server.sh\
+     /root/
 
-COPY docker/*.sh .
 RUN chmod +x /root/*.sh
+RUN cd cic-ussd && \
+    pip install $pip_extra_index_url_flag .
+
 
 # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY .config/ /usr/local/etc/cic-ussd/
+COPY cic-ussd/.config/ /usr/local/etc/cic-ussd/
-COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+COPY cic-ussd/cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
 
-ENTRYPOINT []
+WORKDIR /root

apps/cic-ussd/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
-
-
-# create secrets directory
-RUN mkdir -vp pgp/keys
-
-# create application directory
-RUN mkdir -vp cic-ussd
-RUN mkdir -vp data
-
-COPY requirements.txt .
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . .
-RUN python setup.py install
-
-COPY cic_ussd/db/ussd_menu.json data/
-
-COPY docker/*.sh .
-RUN chmod +x /root/*.sh
-
-# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
-COPY .config/ /usr/local/etc/cic-ussd/
-COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
-
-ENTRYPOINT []

apps/cic-ussd/docker/start_cic_user_server.sh

@@ -4,4 +4,4 @@
 
 user_server_port=${SERVER_PORT:-9500}
 
-/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_cic_user_ussd_server.sh

@@ -4,4 +4,4 @@
 
 user_ussd_server_port=${SERVER_PORT:-9000}
 
-/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

apps/cic-ussd/requirements.txt

@@ -1,17 +1,4 @@
-cic-eth~=0.12.2a3
+cic_base[full_graph]==0.1.3a3+build.984b5cff
-cic-notify~=0.4.0a10
+cic-eth~=0.12.0a1
-cic-types~=0.1.0a14
+cic-notify~=0.4.0a7
-confini~=0.4.1a1
+cic-types~=0.1.0a11
-semver==2.13.0
-alembic==1.4.2
-SQLAlchemy==1.3.20
-psycopg2==2.8.6
-tinydb==4.2.0
-phonenumbers==8.12.12
-redis==3.5.3
-celery==4.4.7
-python-i18n[YAML]==0.3.9
-pyxdg==0.27
-bcrypt==3.2.0
-uWSGI==2.0.19.1
-transitions==0.8.4

apps/cic-ussd/scripts/migrate.py

@@ -30,7 +30,7 @@ arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
 args = arg_parser.parse_args()
 
 
-config = Config(args.c, env_prefix=args.env_prefix)
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug(f'config:\n{config}')