add rules to run on mr for test

Fix path to PGP exports directory

See merge request grassrootseconomics/cic-internal-integration!165

.gitlab-ci.yml

@@ -6,6 +6,7 @@ include:
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build

apps/cic-cache/cic_cache/cache.py

@@ -1,22 +1,28 @@
 # standard imports
 import logging
+import datetime
 
-# third-party imports
+# external imports
 import moolb
 
 # local imports
-from cic_cache.db import list_transactions_mined
-from cic_cache.db import list_transactions_account_mined
+from cic_cache.db.list import (
+        list_transactions_mined,
+        list_transactions_account_mined,
+        list_transactions_mined_with_data,
+        )
 
 logg = logging.getLogger()
 
 
-class BloomCache:
+class Cache:
 
     def __init__(self, session):
         self.session = session
 
 
+class BloomCache(Cache):
+
     @staticmethod
     def __get_filter_size(n):
         n = 8192 * 8
@@ -87,3 +93,44 @@ class BloomCache:
             f_blocktx.add(block + tx)
             logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
+
+
+class DataCache(Cache):
+
+    def load_transactions_with_data(self, offset, end):
+        rows = list_transactions_mined_with_data(self.session, offset, end) 
+        tx_cache = []
+        highest_block = -1;
+        lowest_block = -1;
+        date_is_str = None # stick this in startup
+        for r in rows:
+            if highest_block == -1:
+                highest_block = r['block_number']
+            lowest_block = r['block_number']
+            tx_type = 'unknown'
+
+            if r['value'] != None:
+                tx_type = '{}.{}'.format(r['domain'], r['value'])
+
+            if date_is_str == None:
+                date_is_str = type(r['date_block']).__name__ == 'str'
+
+            o = {
+                'block_number': r['block_number'],
+                'tx_hash': r['tx_hash'],
+                'date_block': r['date_block'],
+                'sender': r['sender'],
+                'recipient': r['recipient'],
+                'from_value': int(r['from_value']),
+                'to_value': int(r['to_value']),
+                'source_token': r['source_token'],
+                'destination_token': r['destination_token'],
+                'success': r['success'],
+                'tx_type': tx_type,
+            }
+
+            if date_is_str:
+                o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
+
+            tx_cache.append(o)
+        return (lowest_block, highest_block, tx_cache)

apps/cic-cache/cic_cache/db/list.py

@@ -28,6 +28,26 @@ def list_transactions_mined(
     return r
 
 
+def list_transactions_mined_with_data(
+        session,
+        offset,
+        end,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit.
+
+    :param offset: Offset in data set to return transactions from
+    :type offset: int
+    :param limit: Max number of transactions to retrieve
+    :type limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
+
+    r = session.execute(s)
+    return r
+
+
 def list_transactions_account_mined(
         session,
         address,

apps/cic-cache/cic_cache/runnable/daemons/filters/__init__.py

@@ -1 +1,2 @@
 from .erc20 import *
+from .faucet import *

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -2,7 +2,6 @@
 import logging
 
 # external imports
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.address import (
         to_checksum_address,
         )
@@ -13,6 +12,7 @@ from cic_eth_registry.error import (
         NotAContractError,
         ContractMismatchError,
         )
+from eth_erc20 import ERC20
 
 # local imports
 from .base import TagSyncFilter

apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py

@@ -0,0 +1,73 @@
+# standard imports
+import logging
+
+# external imports
+from erc20_faucet import Faucet
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.status import Status
+from hexathon import strip_0x
+
+# local imports
+import cic_cache.db as cic_cache_db
+from .base import TagSyncFilter
+
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
+
+
+class FaucetFilter(TagSyncFilter):
+
+    def __init__(self, chain_spec, sender_address=ZERO_ADDRESS):
+        super(FaucetFilter, self).__init__('give_to', domain='faucet')
+        self.chain_spec = chain_spec
+        self.sender_address = sender_address
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        try:
+            data = strip_0x(tx.payload)
+        except ValueError:
+            return False
+        logg.debug('data {}'.format(data))
+        if Faucet.method_for(data[:8]) == None:
+            return False
+
+        token_sender = tx.inputs[0]
+        token_recipient = data[64+8-40:]
+        logg.debug('token recipient {}'.format(token_recipient))
+ 
+        f = Faucet(self.chain_spec)
+        o = f.token(token_sender, sender_address=self.sender_address)
+        r = conn.do(o)
+        token = f.parse_token(r)
+
+        f = Faucet(self.chain_spec)
+        o = f.token_amount(token_sender, sender_address=self.sender_address)
+        r = conn.do(o)
+        token_value = f.parse_token_amount(r)
+
+        cic_cache_db.add_transaction(
+                db_session,
+                tx.hash,
+                block.number,
+                tx.index,
+                to_checksum_address(token_sender),
+                to_checksum_address(token_recipient),
+                token,
+                token,
+                token_value,
+                token_value,
+                tx.status == Status.SUCCESS,
+                block.timestamp,
+                )
+        db_session.flush()
+        cic_cache_db.tag_transaction(
+                db_session,
+                tx.hash,
+                self.tag_name,
+                domain=self.tag_domain,
+                )
+        db_session.commit()
+
+        return True

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -0,0 +1,110 @@
+# standard imports
+import logging
+import json
+import re
+import base64
+
+# local imports
+from cic_cache.cache import (
+        BloomCache,
+        DataCache,
+    )
+
+logg = logging.getLogger(__name__)
+
+re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
+re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
+
+DEFAULT_LIMIT = 100
+
+
+def process_transactions_account_bloom(session, env):
+    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    address = r[1]
+    if r[2] == None:
+        address = '0x' + address
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 2:
+        offset = r[3]
+    limit = 0
+    if r.lastindex > 3:
+        limit = r[4]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_all_bloom(session, env):
+    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 0:
+        offset = r[1]
+    limit = 0
+    if r.lastindex > 1:
+        limit = r[2]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_all_data(session, env):
+    r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
+    if not r:
+        return None
+    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+        return None
+
+    offset = r[1]
+    end = r[2]
+    if r[2] < r[1]:
+        raise ValueError('cart before the horse, dude')
+
+    c = DataCache(session)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
+
+    for r in tx_cache:
+        r['date_block'] = r['date_block'].timestamp()
+
+    o = {
+        'low': lowest_block,
+        'high': highest_block,
+        'data': tx_cache,
+    }
+
+    
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -1,18 +1,20 @@
 # standard imports
 import os
-import re
 import logging
 import argparse
-import json
 import base64
 
-# third-party imports
+# external imports
 import confini
 
 # local imports
-from cic_cache import BloomCache
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
+from cic_cache.runnable.daemons.query import (
+        process_transactions_account_bloom,
+        process_transactions_all_bloom,
+        process_transactions_all_data,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -44,72 +46,6 @@ logg.debug('config:\n{}'.format(config))
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 
-re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-
-DEFAULT_LIMIT = 100
-
-
-def process_transactions_account_bloom(session, env):
-    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    address = r[1]
-    if r[2] == None:
-        address = '0x' + address
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 2:
-        offset = r[3]
-    limit = 0
-    if r.lastindex > 3:
-        limit = r[4]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_bloom(session, env):
-    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 0:
-        offset = r[1]
-    limit = 0
-    if r.lastindex > 1:
-        limit = r[2]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
 
 # uwsgi application
 def application(env, start_response):
@@ -119,10 +55,16 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
+            process_transactions_all_data,
             process_transactions_all_bloom,
             process_transactions_account_bloom,
             ]:
-        r = handler(session, env)
+        r = None
+        try:
+            r = handler(session, env)
+        except ValueError as e:
+            start_response('400 {}'.format(str(e)))
+            return []
         if r != None:
             (mime_type, content) = r
             break

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -41,16 +41,26 @@ from cic_cache.db import (
         )
 from cic_cache.runnable.daemons.filters import (
         ERC20TransferFilter,
+        FaucetFilter,
         )
 
 script_dir = os.path.realpath(os.path.dirname(__file__))
 
+def add_block_args(argparser):
+    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+    return argparser
+
+
 logg = cic_base.log.create()
 argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-#argparser = cic_base.argparse.add(argparser, add_traffic_args, 'traffic')
+argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
 args = cic_base.argparse.parse(argparser, logg)
 config = cic_base.config.create(args.c, args, args.env_prefix)
 
+config.add(args.history_start, 'SYNCER_HISTORY_START', True)
+config.add(args.no_history, '_NO_HISTORY', True)
+
 cic_base.config.log(config)
 
 dsn = dsn_from_config(config)
@@ -59,7 +69,6 @@ SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
@@ -71,6 +80,7 @@ def register_filter_tags(filters, session):
             session.commit()
             logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
         except sqlalchemy.exc.IntegrityError:
+            session.rollback()
             logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
 
 
@@ -82,7 +92,7 @@ def main():
     r = rpc.do(o)
     block_offset = int(strip_0x(r), 16) + 1
 
-    logg.debug('starting at block {}'.format(block_offset))
+    logg.debug('current block height {}'.format(block_offset))
 
     syncers = []
 
@@ -91,8 +101,13 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        logg.info('found no backends to resume')
-        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
+        initial_block_start = config.get('SYNCER_HISTORY_START')
+        initial_block_offset = block_offset
+        if config.get('_NO_HISTORY'):
+            initial_block_start = block_offset
+            initial_block_offset += 1
+        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
+        logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
@@ -112,9 +127,11 @@ def main():
         logg.info('using trusted address {}'.format(address))
 
     erc20_transfer_filter = ERC20TransferFilter(chain_spec)
+    faucet_filter = FaucetFilter(chain_spec)
 
     filters = [
         erc20_transfer_filter,
+        faucet_filter,
             ]
 
     session = SessionBase.create_session()

apps/cic-cache/config/docker/eth.ini

@@ -1,2 +1,2 @@
 [eth]
-provider = ws://localhost:63546
+provider = http://localhost:63545

apps/cic-cache/config/docker/syncer.ini

@@ -1,2 +1,3 @@
 [syncer]
 loop_interval = 1
+history_start = 0

apps/cic-cache/config/syncer.ini

@@ -1,2 +1,3 @@
 [syncer]
 loop_interval = 5
+history_start = 0

apps/cic-cache/config/test/syncer.ini

@@ -0,0 +1,2 @@
+[syncer]
+loop_interval = 1

apps/cic-cache/docker/Dockerfile

@@ -17,7 +17,7 @@ RUN apt-get update && \
 
 # Copy shared requirements from top of mono-repo
 RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
 COPY cic-cache/requirements.txt ./
 COPY cic-cache/setup.cfg \

apps/cic-cache/requirements.txt

@@ -1,12 +1,12 @@
-cic-base~=0.1.2b6
+cic-base~=0.1.2b10
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a1
+cic-eth-registry~=0.5.5a4
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a2
+chainsyncer[sql]~=0.0.2a4

apps/cic-cache/test_requirements.txt

@@ -8,6 +8,4 @@ eth_tester==0.5.0b3
 py-evm==0.3.0a20
 web3==5.12.2
 cic-eth-registry~=0.5.5a3
-giftable-erc20-token~=0.0.8a10
-eth-address-index~=0.1.1a10
-sarafu-faucet~=0.0.3a1
+cic-base[full]==0.1.2b8

apps/cic-cache/tests/conftest.py

@@ -88,3 +88,16 @@ def txs(
             tx_hash_first,
             tx_hash_second,
             ]
+
+
+@pytest.fixture(scope='function')
+def tag_txs(
+        init_database,
+        txs,
+        ):
+
+    db.add_tag(init_database, 'taag', domain='test')
+    init_database.commit()
+
+    db.tag_transaction(init_database, txs[1], 'taag', domain='test')
+

apps/cic-cache/tests/filters/test_erc20.py

@@ -22,7 +22,7 @@ from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
 logg = logging.getLogger()
 
 
-def test_cache(
+def test_erc20_filter(
         eth_rpc,
         foo_token,
         init_database,

apps/cic-cache/tests/filters/test_faucet.py

@@ -0,0 +1,71 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.chain import ChainSpec
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.block import (
+        block_by_hash,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        unpack,
+        transaction,
+        Tx,
+        )
+from hexathon import strip_0x
+from erc20_faucet.faucet import SingleShotFaucet
+from sqlalchemy import text
+
+# local imports
+from cic_cache.db import add_tag
+from cic_cache.runnable.daemons.filters.faucet import FaucetFilter
+
+logg = logging.getLogger()
+
+
+def test_filter_faucet(
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        faucet_noregistry,
+        init_database,
+        list_defaults,
+        contract_roles,
+        agent_roles,
+        tags,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = FaucetFilter(chain_spec, contract_roles['CONTRACT_DEPLOYER'])
+
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
+    c = SingleShotFaucet(chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex, o) = c.give_to(faucet_noregistry, agent_roles['ALICE'], agent_roles['ALICE'])
+    r = eth_rpc.do(o)
+
+    tx_src = unpack(bytes.fromhex(strip_0x(o['params'][0])), chain_spec)
+
+    o = receipt(r)
+    r = eth_rpc.do(o)
+    rcpt = Tx.src_normalize(r)
+
+    assert r['status'] == 1
+
+    o = block_by_hash(r['block_hash'])
+    r = eth_rpc.do(o)
+    block_object = Block(r)
+
+    tx = Tx(tx_src, block_object)
+    tx.apply_receipt(rcpt)
+
+    r = fltr.filter(eth_rpc, block_object, tx, init_database)
+    assert r
+
+    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
+    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
+    assert r[0] == tx.hash

apps/cic-cache/tests/test_api.py

@@ -0,0 +1,31 @@
+# standard imports
+import json
+
+# external imports
+import pytest
+
+# local imports
+from cic_cache.runnable.daemons.query import process_transactions_all_data
+
+
+def test_api_all_data(
+        init_database,
+        txs,
+        ):
+
+    env = {
+        'PATH_INFO': '/txa/410000/420000',
+        'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+    j = process_transactions_all_data(init_database, env)
+    o = json.loads(j[1])
+    
+    assert len(o['data']) == 2
+
+    env = {
+        'PATH_INFO': '/txa/420000/410000',
+        'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+   
+    with pytest.raises(ValueError):
+        j = process_transactions_all_data(init_database, env)

apps/cic-cache/tests/test_cache.py

@@ -9,6 +9,7 @@ import pytest
 
 # local imports
 from cic_cache import BloomCache
+from cic_cache.cache import DataCache
 
 logg = logging.getLogger()
 
@@ -33,3 +34,23 @@ def test_cache(
 
     assert b[0] == list_defaults['block'] - 1
 
+
+def test_cache_data(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        tag_txs,
+        ):
+
+    session = init_database
+
+    c = DataCache(session)
+    b = c.load_transactions_with_data(410000, 420000)
+
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == txs[1]
+    assert b[2][1]['tx_type'] == 'unknown'
+    assert b[2][0]['tx_type'] == 'test.taag'
+    

apps/cic-eth/.gitlab-ci.yml

@@ -5,14 +5,27 @@
 
 .cic_eth_changes_target:
     rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+      - $CONTEXT/$APP_NAME/**/*
+      when: always
 
 build-mr-cic-eth:
     extends:
-        - .cic_eth_changes_target
-        - .py_build_merge_request
         - .cic_eth_variables
+        - .cic_eth_changes_target
+        - .py_build_target_test
+
+test-mr-cic-eth:
+    extends:
+        - .cic_eth_variables
+        - .cic_eth_changes_target
+        - .cic_eth_changes_target
+    stage: test
+    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
+    script:
+        - cd apps/$APP_NAME/
+        - pytest tests/unit/ 
 
 build-push-cic-eth:
     extends:

apps/cic-eth/cic_eth/admin/token.py

@@ -16,4 +16,6 @@ def default_token(self):
     return {
             'symbol': self.default_token_symbol,
             'address': self.default_token_address,
+            'name': self.default_token_name,
+            'decimals': self.default_token_decimals,
             }

apps/cic-eth/cic_eth/eth/account.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from erc20_faucet import Faucet
 from hexathon import (
         strip_0x,
         )
@@ -20,8 +20,9 @@ from chainlib.eth.tx import (
         )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
-from eth_accounts_index import AccountRegistry
-from sarafu_faucet import MinterFaucet as Faucet
+from eth_accounts_index.registry import AccountRegistry
+from eth_accounts_index import AccountsIndex 
+from sarafu_faucet import MinterFaucet
 from chainqueue.db.models.tx import TxCache
 
 # local import
@@ -133,7 +134,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
     gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
-    account_registry = AccountRegistry(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
 
@@ -185,7 +186,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -338,7 +339,7 @@ def cache_account_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
     tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = AccountRegistry.parse_add_request(tx['data'])
+    tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_cache = TxCache(

apps/cic-eth/cic_eth/eth/erc20.py

@@ -6,7 +6,6 @@ import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -16,6 +15,7 @@ from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
 from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase

apps/cic-eth/cic_eth/ext/address.py

@@ -7,7 +7,7 @@ from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth_registry import CICRegistry
-from eth_address_declarator import AddressDeclarator
+from eth_address_declarator import Declarator
 
 # local imports
 from cic_eth.task import BaseTask
@@ -23,12 +23,12 @@ def translate_address(address, trusted_addresses, chain_spec, sender_address=ZER
     registry = CICRegistry(chain_spec, rpc)
     
     declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    c = AddressDeclarator(chain_spec)
+    c = Declarator(chain_spec)
 
     for trusted_address in trusted_addresses:
         o = c.declaration(declarator_address, trusted_address, address, sender_address=sender_address)
         r = rpc.do(o)
-        declaration_hex = AddressDeclarator.parse_declaration(r)
+        declaration_hex = Declarator.parse_declaration(r)
         declaration_hex = declaration_hex[0].rstrip('0')
         declaration_bytes = bytes.fromhex(declaration_hex)
         declaration = None

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,13 +14,13 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
-from chainlib.eth.erc20 import ERC20
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.query import get_tx_cache
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.queue.time import tx_times

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -3,19 +3,20 @@ import logging
 
 # external imports
 import celery
-from cic_eth_registry.error import UnknownContractError
+from cic_eth_registry.error import (
+    UnknownContractError,
+    NotAContractError,
+    )
 from chainlib.status import Status as TxStatus
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.erc20 import ERC20
 from hexathon import (
         strip_0x,
         add_0x,
         )
-# TODO: use sarafu_Faucet for both when inheritance has been implemented
-from erc20_single_shot_faucet import SingleShotFaucet
-from sarafu_faucet import MinterFaucet as Faucet
+from eth_erc20 import ERC20
+from erc20_faucet import Faucet
 
 # local imports
 from .base import SyncFilter
@@ -71,14 +72,15 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        c = SingleShotFaucet(self.chain_spec)
+        c = Faucet(self.chain_spec)
+
         o = c.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 
-        o = c.amount(faucet_contract, sender_address=self.caller_address)
+        o = c.token_amount(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
-        transfer_data['value'] = c.parse_amount(r)
+        transfer_data['value'] = c.parse_token_amount(r)
 
         return ('tokengift', transfer_data)
 
@@ -127,8 +129,7 @@ class CallbackFilter(SyncFilter):
                     (transfer_type, transfer_data) = parser(tx, conn)
                     if transfer_type == None:
                         continue
-                else:
-                    pass
+                break
             except RequestMismatchException:
                 continue
 
@@ -171,7 +172,9 @@ class CallbackFilter(SyncFilter):
                 t = self.call_back(transfer_type, result)
                 logg.info('callback success task id {} tx {} queue {}'.format(t, tx.hash, t.queue))
             except UnknownContractError:
-                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tx.queue, tx.method, transfer_data['to'], tx.hash))
+                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(self.queue, self.method, transfer_data['to'], tx.hash))
+            except NotAContractError:
+                logg.debug('callback filter {}:{} skipping "transfer" on non-contract address {} tx {}'.format(self.queue, self.method, transfer_data['to'], tx.hash))
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -14,7 +14,7 @@ from .base import SyncFilter
 
 logg = logging.getLogger().getChild(__name__)
 
-account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd'
+account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 
 
 class RegistrationFilter(SyncFilter):

apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py

@@ -30,7 +30,7 @@ class TxFilter(SyncFilter):
         if otx == None:
             logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
             return None
-        logg.info('tx filter match on {}'.format(otx.tx_hash))
+        logg.debug('otx filter match on {}'.format(otx.tx_hash))
         db_session.flush()
         SessionBase.release_session(db_session)
         s_final_state = celery.signature(

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -22,6 +22,7 @@ from chainlib.eth.connection import (
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
 from cic_eth_registry.error import UnknownContractError
+from cic_eth_registry.erc20 import ERC20Token
 import liveness.linux
 
 
@@ -36,7 +37,7 @@ from cic_eth.eth import (
 from cic_eth.admin import (
         debug,
         ctrl,
-        token
+        token,
         )
 from cic_eth.queue import (
         query,
@@ -75,7 +76,6 @@ argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
 argparser.add_argument('-r', type=str, help='CIC registry address')
 argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
@@ -121,20 +121,25 @@ broker = config.get('CELERY_BROKER_URL')
 if broker[:4] == 'file':
     bq = tempfile.mkdtemp()
     bp = tempfile.mkdtemp()
-    current_app.conf.update({
+    conf_update = {
             'broker_url': broker,
             'broker_transport_options': {
                 'data_folder_in': bq,
                 'data_folder_out': bq,
                 'data_folder_processed': bp,
             },
-            },
-            )
+            }
+    if config.true('CELERY_DEBUG'):
+        conf_update['result_extended'] = True
+    current_app.conf.update(conf_update)
     logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
 else:
-    current_app.conf.update({
-        'broker_url': broker,
-        })
+    conf_update = {
+            'broker_url': broker,
+            }
+    if config.true('CELERY_DEBUG'):
+        conf_update['result_extended'] = True
+    current_app.conf.update(conf_update)
 
 result = config.get('CELERY_RESULT_URL')
 if result[:4] == 'file':
@@ -203,6 +208,11 @@ def main():
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
+    default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)
+    default_token.load(rpc)
+    BaseTask.default_token_decimals = default_token.decimals
+    BaseTask.default_token_name = default_token.name
+
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
    

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -51,15 +51,23 @@ from cic_eth.registry import (
 
 script_dir = os.path.realpath(os.path.dirname(__file__))
 
+def add_block_args(argparser):
+    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+    return argparser
+
+
 logg = cic_base.log.create()
 argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-#argparser = cic_base.argparse.add(argparser, add_traffic_args, 'traffic')
+argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
 args = cic_base.argparse.parse(argparser, logg)
+
 config = cic_base.config.create(args.c, args, args.env_prefix)
 
 config.add(args.y, '_KEYSTORE_FILE', True)
-
 config.add(args.q, '_CELERY_QUEUE', True)
+config.add(args.history_start, 'SYNCER_HISTORY_START', True)
+config.add(args.no_history, '_NO_HISTORY', True)
 
 cic_base.config.log(config)
 
@@ -69,9 +77,9 @@ SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -89,7 +97,7 @@ def main():
         stat = init_chain_stat(rpc, block_start=block_current)
         loop_interval = stat.block_average()
 
-    logg.debug('starting at block {}'.format(block_offset))
+    logg.debug('current block height {}'.format(block_offset))
 
     syncers = []
 
@@ -98,8 +106,13 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        logg.info('found no backends to resume')
-        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
+        initial_block_start = config.get('SYNCER_HISTORY_START')
+        initial_block_offset = block_offset
+        if config.get('_NO_HISTORY'):
+            initial_block_start = block_offset
+            initial_block_offset += 1
+        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
+        logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
@@ -155,7 +168,6 @@ def main():
         for cf in callback_filters:
             syncer.add_filter(cf)
 
-        #r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
         r = syncer.loop(int(loop_interval), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 

apps/cic-eth/cic_eth/runnable/info.py

@@ -12,7 +12,10 @@ import confini
 import celery
 
 # local imports
-from cic_eth.api import Api
+from cic_eth.api import (
+        Api,
+        AdminApi,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -53,13 +56,20 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
 queue = args.q
 
 api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
+admin_api = AdminApi(None)
 
 def main():
+    t = admin_api.registry()
+    registry_address = t.get()
+    print('Registry: {}'.format(registry_address))
+
     t = api.default_token()
     token_info = t.get()
     print('Default token symbol: {}'.format(token_info['symbol']))
     print('Default token address: {}'.format(token_info['address']))
+    logg.debug('Default token name: {}'.format(token_info['name']))
+    logg.debug('Default token decimals: {}'.format(token_info['decimals']))
 
-
+    
 if __name__ == '__main__':
     main()

apps/cic-eth/cic_eth/task.py

@@ -33,6 +33,8 @@ class BaseTask(celery.Task):
     create_gas_oracle = RPCGasOracle
     default_token_address = None
     default_token_symbol = None
+    default_token_name = None
+    default_token_decimals = None
     run_dir = '/run'
 
     def create_session(self):

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.11',
+        'beta.14',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/celery.ini

@@ -1,3 +1,4 @@
 [celery]
 broker_url = redis://
 result_url = redis://
+debug = 0

apps/cic-eth/config/docker/celery.ini

@@ -1,3 +1,4 @@
 [celery]
 broker_url = redis://localhost:63379
 result_url = redis://localhost:63379
+debug = 0

apps/cic-eth/config/docker/syncer.ini

@@ -1,2 +1,3 @@
 [SYNCER]
 loop_interval = 
+history_start = 0

apps/cic-eth/config/syncer.ini

@@ -1,2 +1,3 @@
 [SYNCER]
 loop_interval = 
+history_start = 0

apps/cic-eth/doc/texinfo/accounts.texi

@@ -0,0 +1,22 @@
+@node cic-eth-accounts
+@section Accounts
+
+Accounts are private keys in the signer component keyed by "addresses," a one-way transformation of a public key. Data can be signed by using the account as identifier for corresponding RPC requests.
+
+Any account to be managed by @code{cic-eth} must be created by the corresponding task. This is because @code{cic-eth} creates a @code{nonce} entry for each newly created account, and guarantees that every nonce will only be used once in its threaded environment.
+
+The calling code receives the account address upon creation. It never receives or has access to the private key.
+
+
+@subsection Signer RPC 
+
+The signer is expected to handle a subset of the standard JSON-RPC:
+
+@table @code
+@item personal_newAccount(password)
+Creates a new account, returning the account address.
+@item eth_signTransactions(tx_dict)
+Sign the transaction represented as a dictionary.
+@item eth_sign(address, message)
+Signs an arbtirary message with the standard Ethereum prefix.
+@end table

apps/cic-eth/doc/texinfo/admin.texi

@@ -0,0 +1,60 @@
+@node cic-eth system maintenance
+@appendix Admin API
+
+The admin API is still in an early stage of refinement. User friendliness can be considerably improved.
+
+All of the API calls are celery task proxies, and return @code{Celery.AsyncResult} unless otherwise noted.
+
+In contrast to the client API module, this API does not currently implement a pluggable callback.
+
+@appendixsection registry
+
+Returns the @code{ContractRegistry} this instance of @code{cic-eth-tasker} is running on.
+
+@appendixsection proxy-do
+
+Execute an arbitary JSON-RPC request using the @code{cic-eth-tasker} blockchain node RPC connection.
+
+@appendixsection default_token
+
+Returns the default token symbol and address.
+
+@appendixsection lock
+
+Set lock bits, globally or per address
+
+@appendixsection unlock
+
+Opposite of lock
+
+@appendixsection get_lock
+
+Get the current state of a lock
+
+@appendixsection tag_account
+
+Associate an identifier with an account address (@xref{cic-eth system accounts})
+
+@appendixsection have_account
+
+Check whether a private key exists in the keystore able to sign on behalf of the given account (it actually performs a signature).
+
+@appendixsection resend
+
+Clone or resend a transaction
+
+@appendixsection check_nonce
+
+Returns diagnostics for nonce sequences per account, e.g. detect nonce gaps that block execution of further transactions.
+
+@appendixsection fix_nonce
+
+Re-orders all nonces by shifting all transaction nonces after the given transaction down by one. This has the additional effect of obsoleting the given transaction. Can be used to close gaps in the nonce sequencing. Use with care!
+
+@appendixsection account
+
+Return brief transaction info lists per account
+
+@appendixsection tx
+
+Return a complex transaction metadata object for a single transaction. The object assembles state from both the blockchain node and the custodial queue system.

apps/cic-eth/doc/texinfo/all.texi

@@ -0,0 +1,18 @@
+\input texinfo
+@setfilename index.html
+@settitle CIC custodial services reference deployment
+
+@copying
+Released 2021 under GPL3
+@end copying
+
+@titlepage
+@title CIC custodial services reference deployment
+@author Louis Holbrook
+@end titlepage
+
+@c
+@contents
+
+@include index.texi
+

apps/cic-eth/doc/texinfo/chains.texi

@@ -0,0 +1,4 @@
+@node cic-eth Appendix Task chains
+@appendix Task chains
+
+TBC - explain here how to generate these chain diagrams

apps/cic-eth/doc/texinfo/configuration.texi

@@ -0,0 +1,108 @@
+@node cic-eth configuration
+@section Configuration
+
+(refer to @code{cic-base} for a general overview of the config pipeline)
+
+Configuration parameters are grouped by configuration filename.
+
+
+@subsection cic
+
+@table @var
+@item registry_address
+Ethereum address of the @var{ContractRegistry} contract
+@item chain_spec
+String representation of the connected blockchain according to the @var{chainlib} @var{ChainSpec} format.
+@item tx_retry_delay
+Minimum time in seconds to wait before retrying a transaction
+@item trust_address
+Comma-separated list of one or more ethereum addresses regarded as trusted for describing other resources, Used by @var{cic-eth-registry} in the context of the @var{AddressDeclarator}.
+@item defalt_token_symbol
+Fallback token to operate on when no other context is given.
+@item health_modules
+Comma-separated list of methods to execute liveness tests against. (see ...)
+@item run_dir
+Directory to use for session-scoped variables for @var{cic-eth} daemon parent processes.
+@end table
+
+
+@subsection celery
+
+@table @var
+@item broker_url
+Message broker URL
+@item result_url
+Result backend URL
+@item debug
+Boolean value. If set, the amount of available context for a task in the result backend will be maximized@footnote{This is a @emph{required} setting for the task graph documenter to enabled it to display task names in the graph}.
+@end table
+
+
+@subsection database
+
+See ref cic-base when ready
+
+
+@subsection eth
+
+@table @var
+@item provider
+Address of default RPC endpoint for transactions and state queries.
+@item gas_gifter_minimum_balance
+The minimum gas balance that must be held by the @code{GAS GIFTER} token before the queue processing shuts down@footnote{You should really make sure that this threshold is never hit}
+@end table
+
+
+@subsection redis
+
+Defines connection to the redis server used outside of the context of @var{celery}. This is usually the same server, but should be a different db.
+
+@table @var
+@item host
+Redis hostname
+@item port
+Redis port
+@item db
+Redis db
+@end table
+
+
+@subsection signer
+
+Parameters 
+
+@table @var
+@item socket_path
+The connection string for the signer JSON-RPC service.@footnote{The @var{crypto-dev-signer} supports UNIX socket or a HTTP(S) connections}
+@item secret
+If set, this password is used to add obfuscation on top of the encryption already applied by the signer for the keystore.
+@end table
+
+
+
+@subsection ssl
+
+Certificate information for https api callbacks.
+
+@table @var
+@item enable_client
+Boolean value. If set, client certificate will be used to authenticate the callback request.
+@item cert_file
+Client certificate file in PEM or DER format
+@item key_file
+Client key file in PEM or DER format
+@item password
+Password for unlocking the client key
+@item ca_file
+Certificate authority bundle, to verify the certificate sent by the callback server.
+@end table
+
+
+@subsection syncer
+
+@table @var
+@item loop_interval
+Seconds to pause before each execution of the @var{chainsyncer} poll loop.
+@end table
+
+

apps/cic-eth/doc/texinfo/dependencies.texi

@@ -0,0 +1,46 @@
+@node cic-eth-dependencies
+@section Dependencies
+
+This application is written in Python 3.8. It is tightly coupled with @code{python-celery}, which provides the task worker ecosystem. It also uses @code{SQLAlchemy} which provides useful abstractions for persistent storage though SQL, and @code{alembic} for database schema migrations.
+
+There is currently also a somewhat explicit coupling with @code{Redis}, which is used as message broker for @code{python-celery}. @code{Redis} is also explicitly used by some CLI tools to retrieve results from command execution. This coupling may be relaxed in the future to allow other key-value pubsub solutions instead. 
+
+
+@subsection Generalized project dependencies
+
+The core features are built around four main independent components that have been developed for the purpose of this project, but are separated and maintained as general-purpose libraries.
+
+@table @samp
+@item chainlib
+A cross-chain library prototype that can provide encodings for transactions on a Solidity-based EVM contract network.
+@item chainqueue
+Queue manager that guarantees delivery of outgoing blockchain transactions.
+@item chainsyncer
+Monitors blockchains and guarantees execution of an arbitrary count of pluggable code objects for each block transaction.
+@item crypto-dev-signer
+An keystore capable of signing for the EVM chain through a standard Ethereum JSON-RPC interface.
+@end table
+
+@anchor{cic-eth-dependencies-smart-contracts}
+@subsection Smart contract dependencies
+
+The Smart contracts needed by the network must be discoverable through a single entry point called the Contract Registry. The contract registry is expected to reference itself in its records. The authenticity of the contract registry must be guaranteed by external sources of trust.
+
+The contract registry maps contract addresses to well-known identifiers. The contracts are as follows:
+
+@table @code
+@item ContractRegistry (points to self)
+Resolves plaintext identifiers to contract addresses.
+@item AccountRegistry
+An append-only store of accounts hosted by the custodial system
+@item TokenRegistry
+Unique symbol-to-address mappings for token contracts
+@item AddressDeclarator
+Reverse address to resource lookup
+@item TokenAuthorization
+Escrow contract for external spending on behalf of custodial users
+@item Faucet
+Called by newly created accounts to receive initial token balance
+@end table
+
+The dependency @code{cic-eth-registry} abstracts and facilitates lookups of resources on the blockchain network. In its current state it resolves tokens by symbol or address, and contracts by common-name identifiers. In the @code{cic-eth} code all lookups for EVM network resources will be performed through this dependency.

apps/cic-eth/doc/texinfo/incoming.texi

@@ -0,0 +1,49 @@
+@node cic-eth-incoming
+@section Incoming transactions
+
+All transactions in mined blocks will be passed to a selection of plugin filters to the @code{chainsyncer} component. Each of these filters are individual python module files in @code{cic_eth.runnable.daemons.filters}. This section describes their function.
+
+The status bits refer to the bits definining the @code{chainqueue} state.
+
+
+@subsection tx
+
+Looks up the transaction in the local queue, and if found it sets the @code{FINAL} state bit. If the contract code execution was unsuccessful, the @code{NETWORK ERROR} state bit is also set.
+
+
+@subsection gas
+
+If the transaction is a gas token transfer, it checks if the recipient is a custodial account awaiting gas refill to execute a transaction (the queue item will have the @code{GAS ISSUES} bit set). If this is the case, the transaction will be activated by setting the @code{QUEUED} bit.
+
+
+@subsection register
+
+If the transaction is an account registration@footnote{The contract keyed by @var{AccountRegistry} in the @var{ContractRegistry} contract}, a Faucet transaction will be triggered for the registered account@footnote{The faucet contract used in the reference implementation will verify whether the account calling it is registered in the @var{AccountRegistry}. Thus it cannot be called before the account registration has succeeded.}
+
+
+@subsection callback
+
+Executes, in order, Celery tasks defined in the configuration variable @var{TASKS_TRANSFER_CALLBACKS}. Each of these tasks are registered as individual filters in the @code{chainsyncer} component, with the corresponding execution guarantees.
+
+The callbacks will receive the following arguments
+
+@enumerate
+@item @strong{result}
+A complex representation of the transaction (see section ?)
+@item @strong{transfertype}
+A string describing the type of transaction found@footnote{See appendix ? for an overview of possible values}
+@item @strong{status}
+0 if contract code executed successfully. Any other value is an error@footnote{The values 1-1024 are reserved for system specific errors. In the current implementation only a general error state with value 1 is defined. See appendix ?.}
+@end enumerate
+
+
+@subsection transferauth
+If a valid transfer authorization request has been made, a token @emph{allowance}@footnote{@code{approve} for ERC20 tokens} transaction is executed on behalf of the custodial account, with the @var{TransferAuthorization} contract as spender.
+
+
+
+@subsection convert
+If the transaction is a token conversion, @emph{and} there is a pending transfer registered for the conversion, the corresponding token transfer transaction will be executed. Not currently implemented
+
+
+

apps/cic-eth/doc/texinfo/index.texi

@@ -0,0 +1,14 @@
+@top cic-eth
+
+@include intro.texi
+@include dependencies.texi
+@include configuration.texi
+@include system.texi
+@include interacting.texi
+@include outgoing.texi
+@include incoming.texi
+@include services.texi
+@include tools.texi
+@include admin.texi
+@include chains.texi
+@include transfertypes.texi

apps/cic-eth/doc/texinfo/interacting.texi

@@ -0,0 +1,109 @@
+@node cic-eth-interacting
+@section Interacting with the system
+
+The API to the @var{cic-eth} component is a proxy for executing @emph{chains of Celery tasks}. The tasks that compose individual chains are documented in @ref{cic-eth Appendix Task chains,the Task Chain appendix}, which also describes a CLI tool that can generate graph representationso of them.
+
+There are two API classes, @var{Api} and @var{AdminApi}. The former is described later in this section, the latter described in @ref{cic-eth system maintenance,the Admin API appendix}.
+
+
+@subsection Interface
+
+API calls are constructed by creating @emph{Celery task signatures} and linking them together, sequentially and/or in parallell. In turn, the tasks themselves may spawn other asynchronous tasks. This means that the code in @file{cic_eth.api.*} does not necessarily specify the full task graph that will be executed for any one command.
+
+The operational guarantee that tasks will be executed, not forgotten, and retried under certain circumstances is deferred to @var{Celery}. On top of this, the @var{chainqueue} component ensures that semantic state changes that the @code{Celery} tasks ask of it are valid.
+
+
+@anchor{cic-eth-locking}
+@subsection Locking
+
+All methods that make a change to the blockchain network must pass @emph{locking layer checks}. Locks may be applied on a global or per-address basis. Lock states are defined by a combination of bit flags. The implemented lock bits are:
+
+@table @var
+@item INIT
+The system has not yet been initialized. In this state, writes are limited to creating unregistered accounts only.
+@item QUEUE
+Items may not be added to the queue
+@item SEND
+Queued items may not be attempted sent to the network
+@item CREATE (global-only)
+New accounts may not be created
+@item STICKY
+Until reset, no other part of the locking state can be reset
+@end table 
+
+
+@subsection Callback
+
+All API calls provide the option to attach a callback to the end of the task chain. This callback will be executed regardless of whether task chain execution succeeded or not.
+
+Refer to @file{cic-eth.callbacks.noop.noop} for the expected callback signature.
+
+
+@subsection API Methods that change state
+
+
+@subsubsection create_account
+
+Creates a new account in the keystore, optionally registering the account with the @var{AccountRegistry} contract.
+
+
+@subsubsection transfer
+
+Attempts to execute a token transaction between two addresses. It is the caller's responsibility to check whether the token balance is sufficient for the transactions.
+
+
+@subsubsection refill_gas
+
+Executes a gas token transfer to a custodial address from the @var{GAS GIFTER} system account.
+
+
+@subsubsection convert
+
+Converts a token to another token for the given custodial account. Currently not implemented.
+
+
+@anchor{cic-eth-convert-and-transfer}
+@subsubsection convert_and_transfer
+
+Same as convert, but will automatically execute a token transfer to another custodial account when conversion has been completed. Currently not implemented.
+
+
+@subsection Read-only API methods
+
+@subsubsection balance
+
+Retrieves a complex balance statement of a single account, including:
+
+@itemize
+@item The network balance at the current block height
+@item Value reductions due to by pending outgoing transactions
+@item Value increments due to by pending incoming transactions
+@end itemize
+
+Only the first of these balance items has guaranteed finality. The reduction by outgoing transaction can be reasonably be assumed to eventually become final. The same applies for the increment by incoming transaction, @emph{unless} the transfer is part of a multiple-transaction operation. For example, a @ref{cic-eth-convert-and-transfer,convert_and_transfer} operation may fail in the convert stage and/or may yield less tokens then expected after conversion.
+
+
+@subsubsection list
+
+Returns an aggregate iist of all token value changes for a given address. As not all value transfers are a result of literal value transfer contract calls (e.g. @var{transfer} and @var{transferFrom} in @var{ERC20}), this data may come from a number of sources, including:
+
+@itemize
+@item Literal value transfers within the custodial system
+@item Literal value transfers from or to an external address
+@item Faucet invocations (token minting)
+@item Demurrage and redistribution built into the token contract
+@end itemize
+
+
+@subsubsection default_token
+
+Return the symbol and address of the token used by default in the network.
+
+
+@subsubsection ping
+
+Convenience method for the caller to check whether the @var{cic-eth} engine is alive.
+
+
+
+

apps/cic-eth/doc/texinfo/outgoing.texi

@@ -0,0 +1,74 @@
+@node cic-eth-outgoing
+@section Outgoing transactions
+
+@strong{Important! A pre-requisite for proper functioning of the component is that no other agent is sending transactions to the network for any of the keys in the keystore.}
+
+The term @var{state bit} refers to the bits definining the @code{chainqueue} state.
+
+@subsection Lock
+
+Any task that changes blockchain state @strong{must} apply a @code{QUEUE} lock for the address it operates on. This is to ensure that transactions are sent to the network in order.@footnote{If too many transactions arrive out of order to the blockchain node, it may arbitrarily prune those that cannot directly be included in a block. This puts unnecessary strain (and reliance) on the transaction retry mechanism.}
+
+This lock will be released once the blockchain node confirms handover of the transaction.@footnote{This is the responsibility of the @var{dispatcher} service}
+
+
+@subsection Nonce
+
+A separate task step is executed for binding a transaction nonce to a Celery task root id, which uniquely identifies the task chain. This provides atomicity of the nonce across the parallell task environment, and also recoverability in case unexpected program interruption.
+
+The nonce of a permanently failed task must be @emph{manually} unlocked. Celery tasks that involve nonces who permanently fail are to be considered @emph{critical anomalies} and should not happen. The queue locking mechanism is designed to prevent the amount of out-of-sequence transactions for an account to escalate.
+
+
+@subsection Choosing fee prices
+
+@code{cic-eth} uses the @code{chainlib} module to resolve gas price lookups.
+
+Optimizing gas price discovery should be the responsibility of the chainlib layer. It already accommodates using an separate RPC for the @code{eth_gasPrice} call.@footnote{A sample implementation of a gas price tracker speaking JSON-RPC (also built using chainlib/chainsyncer) can be found at @url{https://gitlab.com/nolash/eth-stat-syncer}.}
+
+
+@subsection Choosing gas limits
+
+To determine the gas limit of a transaction, normally the EVM node will be used to perform a dry-run exection of the inputs against the current chain state.
+
+As the current state of the custodial system should only rely on known, trusted contract bytecode, there is no real need for this mechanism. The @code{chainlib}-based contract interfaces are expected to provide a method call that return safe gas limit values for contract interactions.@footnote{Of course, this method call may in turn conceal more sophisticated gas limit heuristics.}
+
+Note that it is still the responsibility of @code{cic-eth} to make sure that the gas limit of the network is sufficient to allow execution of all needed contracts.
+
+
+@subsection Gas refills
+
+If the gas balance of a custodial account is below a certain threshold, a gas refill task will be spawned. The gas will be transferred from the @code{GAS GIFTER} system account.
+
+In the event that the balance is insufficient even for the imminent transaction@footnote{This will of course be the case when an account is first created, whereupon it has a balance of 0. The subsequent faucet call will spawn a gas refill task.}, execution of the transaction will be deferred until the gas refill transaction is completed. In this case the transaction will be marked with the @code{GAS ISSUES} state bit.
+
+The value chosen for the gas refill threshold should ideally allow enough of a margin to avoid the need of deferring transactions in the future.
+
+
+@subsection Queueing transactions
+
+Once the lock, nonce and gas processing parts has been completed, the transaction will be queued for sending. This means that the @code{QUEUED} state bit is set. From here the @ref{cic-eth-services-dispatcher,dispatcher service} takes over responsibility.
+
+
+@subsection Retrying transactions
+
+There are three conditions create the need to defer and retry transactions.
+
+The first is communication problems with the blockchain node itself, for example if it is overloaded or being restarted. As far as possible, retries of this nature will be left to the Celery task workers. There may be cases, however, where it is appropriate to hand the responsibility to the @code{chainqueue} instead. In this case, the queue item will have the @code{NODE ERROR} state bit set.
+
+The second condition occurs when transactions take too long to be confirmed by the network. In this case, the transaction will be re-submitted, but with a higher gas price.
+
+The third condition occurs when the blockchain node purges the transaction from the mempool before it is sent to the network. @code{cic-eth} does not distinguish this case from the second, as the issue is solved using the same mechanism.
+
+
+@subsubsection Transaction obsoletion
+
+"Re-submitting" a transaction means creating a transaction with a previously used nonce for an account address.
+
+When this happens, The @code{chainqueue} will still contain all previous transactions with the same nonce. The transaction being superseded will have the @code{OBSOLETED} state bit set.
+
+Once a transaction has been mined, all other transactions with the same node will have the @code{OBSOLETED} and @code{FINAL} state bits set.
+
+
+@subsection Unexpected conditions
+
+Any unexpected condition exposing the need for urgent code improvement and/or manual intervention will be signalled by marking the transaction with the @code{FUBAR} state bit set.

apps/cic-eth/doc/texinfo/overview.texi

@@ -1,24 +0,0 @@
-@node cic-eth
-@chapter cic-eth
-
-@section Overview
-
-@code{cic-eth} is the heart of the custodial account component. It is a combination of python-celery task queues and daemons that sign, dispatch and monitor blockchain transactions, aswell as triggering tasks contingent on other transactions.
-
-@subsection Dependencies
-
-The @code{cic-registry} module is used as a cache for contracts and tokens on the network.
-
-A web3 JSON-RPC service that transparently proxies a keystore and provides transaction and message signing. The current development version uses the python web3 middleware feature to route methodsi involving the keystore to the module @code{crypto-dev-signer}, which is hosted on @file{pypi.org}.
-
-@subsection What does it do
-
-
-
-@subsection Tasks
-
-Two main categories exist for tasks, @code{eth} and @code{queue}.
-
-The @code{eth} tasks provide means to construct and decode Ethereum transactions, as well as interfacing the underlying key store.
-
-Tasks in the @code{queue} module operate on the state of transactions queued for processing by @code{cic-eth}.

apps/cic-eth/doc/texinfo/services.texi

@@ -0,0 +1,50 @@
+@node cic-eth-services
+@section Services
+
+There are four daemons that together orchestrate all of the aforementioned recipes. This section will provide a high level description of them. 
+
+Each of them have their own set of command line flags. These are available in the CLI help text provided by @kbd{-h} @kbd{--help} and are not recited here.
+
+Daemon executable scripts are located in the @file{cic_eth.runnable.daemons} package. If @var{cic-eth} is installed as a python package, they are installed as executables in @var{PATH}.
+
+
+@subsection tasker
+
+This is the heart of the custodial system. Tasker is the parent process for the celery workers executing all tasks interacting with and changing the state of the queue and the chain. It is also the only service that interfaces with the signer/keystore.
+
+The other @var{cic-eth} daemons all interface with this component, along with any client adapter bridging an end-user gateway (e.g. @var{cic-ussd}). However, the service itself does not have to be actively running for the other services to run; @var{Celery} handles queueing up the incoming tasks until the @var{tasker} comes back online.@footnote{Whereas this is true, there is currently no fail-safe implemented to handles the event of task backlog overflow in Celery. Furthermore, no targeted testing has yet been performed to asses the stability of the system over time if a sudden, sustained surge of resumed task executions occurs. It may be advisable to suspend activity that adds new queue items to the system if volume is high and/or the @var{cic-eth} outage endures. However, there is no panacea for this condition, as every usage scenario is different}
+
+The tasker has a set of pre-requisites that must be fulfilled before it will start
+
+@itemize
+@item It must be given a valid @var{ContractRegistry} address, which must include valid references to all contracts specified in @ref{cic-eth-dependencies-smart-contracts,Smart contract dependencies}
+@item The gas gifter balance must be above the minimum threshold (See "eth" section in configurations).
+@item There must be a valid alembic migration record in the storage database
+@item The redis backend must be reachable and writable
+@item There must be a reachable JSON-RPC server at the other end of the signer socket path (see "signer" section in configurations)
+@end itemize
+
+
+@subsection tracker
+
+Implements the @var{chainsyncer}, and registers the filters described in @ref{cic-eth-incoming,Incoming Transactions} to be executed for every transaction. It consumes the appropriate @var{TASKS_TRANSFER_CALLBACKS} configuration setting to add externally defined filters at without having to change the daemon code.
+
+The @var{tracker} has the same requisities for the @var{ContractRegistry} as the @var{tasker}.
+
+@strong{Important! Guarantees of filter executions has some caveats. Refer to the @var{chainsyncer} documentation for more details.}
+
+
+@anchor{cic-eth-services-dispatcher}
+@subsection dispatcher
+
+Uses the @code{get_upcoming_tx} method call from @var{chainqueue} to receive batches of queued transactions that are ready to send to the blockchain node. Every batch will only contain a single transaction by any one address, which will be the transaction with the next nonce not previously seen by the network. There is no limit currently set to how many transactions that will be included in a single batch.
+
+
+@subsection retrier
+
+The responsibility of the @var{retrier} is to re-queue transactions that failed to be sent to the blockchain node, as well as create @emph{replacements} for transactions whose processing by the network has been delayed. @strong{[refer transaction obolestion]}.
+
+It is in turn the responsiblity of the @var{dispatcher} to send these (re-)queued transactions to the blockchain node.
+
+
+

apps/cic-eth/doc/texinfo/system.texi

@@ -0,0 +1,17 @@
+@node cic-eth system accounts
+@section System initialization
+
+When the system starts for the first time, it is locked for any state change request other than account creation@footnote{Specifically, the @code{INIT}, @code{SEND} and @code{QUEUE} lock bits are set.}. These locks should be @emph{reset} once system initialization has been completed. Currently, system initialization only involves creating and tagging required system accounts, as specified below.
+
+See @ref{cic-eth-locking,Locking} and @ref{cic-eth-tools-ctrl,ctrl in Tools} for details on locking.
+
+@subsection System accounts
+
+Certain accounts in the system have special roles. These are defined by @emph{tagging} certain accounts addresses with well-known identifiers.
+
+@table @var
+@item GAS_GIFTER
+This account @strong{must} at all times have enough gas token to fund any custodial account address in need.
+@item ACCOUNT_REGISTRY_WRITER
+This account @strong{must} have access to add newly created account addresses to the (@xref{cic-eth-dependencies-smart-contracts,Smart contract dependencies})
+@end table

apps/cic-eth/doc/texinfo/tools.texi

@@ -0,0 +1,51 @@
+@node cic-eth-tools
+@section Tools
+
+A collection of CLI tools have been provided to help with diagnostics and other administrative tasks. These use the same configuration infrastructure as the daemons.
+
+Tool scripts are located in the @file{cic_eth.runnable} package. If @var{cic-eth} is installed as a python package, they are installed as executables in @var{PATH}.
+
+
+@subsection info (cic-eth-info)
+
+Returns self-explanatory metadata for the blockchain network, and optionally an address.
+
+
+@subsection inspect (cic-eth-inspect)
+
+Returns information about a specific resource related to the tranasaction queue. The results returned depend on the type of the argument.
+
+@table @var
+@item lock
+If the argument is the literal string @kbd{lock}, it will list all active lock settings currently in effect. (@xref{cic-eth-locking})
+@item <address>
+If the argument is a 0x-prefixed hex string of 42 characters, it returns all transactions where the specified address is a sender or recipient@footnote{If the address is the gas gifter or the accounts index writer, this may be a @emph{lot} of transactions. Use with care!}
+@item <tx_hash>
+If the argument is a 0x-prefixed hex string of 66 characters, it returns data from the custodial queueing system aswell as the network for a single transaction whose hash matches the input. Fails if the transaction does not exist in the queue
+@item <code>
+If the argument is a 0x-prefixed hex string longer than 66 bytes, the argument will be interpreted as raw RLP serialized transaction data, and attempt to match this with an entry in the queue. If a match is found, the result is the same as for @var{<tx_hash>}
+@end table
+
+
+@subsection create (cic-eth-create)
+
+Create a new account, optionally registering the account in the accounts registry, and optionally receiving the newly created address through a redis subscription.
+
+@subsection transfer (cic-eth-transfer)
+
+Execute a token transfer on behalf of a custodial account.
+
+@subsection tag (cic-eth-tag)
+
+Associate an account address with a string identifier. @xref{cic-eth system accounts}
+
+
+@anchor{cic-eth-tools-ctrl}
+@subsection ctrl (cic-eth-ctrl)
+
+Set or reset lock bits, globally or per account address.
+
+@subsection resend (cic-eth-resend)
+
+Resend a transaction. This can either be done "in-place," which means increasing the gas price and re-queueing@footnote{this is the same thing that the retrier does}. It can also be used to @emph{clone} a transaction, which obviously will duplicate the effect of the cloned transaction on the blockchain network.
+

apps/cic-eth/doc/texinfo/transfertypes.texi

@@ -0,0 +1,11 @@
+@node cic-eth Appendix Transaction types
+@appendix Transfer types
+
+@table @var
+@item transfer
+A regular token transfer, e.g. ERC20 @code{transfer}
+@item transferfrom
+A token transfer performed on behalf of another party, e.g. ERC20 @code{transferFrom}
+@item tokengift
+Result of a successful faucet request.
+@end table

apps/cic-eth/docker/Dockerfile

@@ -1,48 +1,63 @@
-# FROM grassrootseconomics:cic
-
-#FROM python:3.8.6-alpine
-FROM python:3.8.6-slim-buster
-
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
+FROM python:3.8.6-slim-buster as compile
 
 WORKDIR /usr/src/cic-eth
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
-
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
-# Copy shared requirements from top of mono-repo
-RUN echo "copying root req file ${root_requirement_file}"
-#COPY $root_requirement_file .
-#RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
-RUN /usr/local/bin/python -m pip install --upgrade pip
-#RUN git clone https://gitlab.com/grassrootseconomics/cic-base.git && \
-#	cd cic-base && \
-#	git checkout 7ae1f02efc206b13a65873567b0f6d1c3b7f9bc0 && \
-#	python merge_requirements.py | tee merged_requirements.txt 
-#RUN cd cic-base && \
-#	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
+#RUN python -m venv venv && . venv/bin/activate
 
-COPY cic-eth/scripts/ scripts/
-COPY cic-eth/setup.cfg cic-eth/setup.py ./
-COPY cic-eth/cic_eth/ cic_eth/
-# Copy app specific requirements
-COPY cic-eth/requirements.txt .
-COPY cic-eth/test_requirements.txt .
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+RUN /usr/local/bin/python -m pip install --upgrade pip
+RUN pip install semver
+
+# TODO use a packaging style that lets us copy requirments only ie. pip-tools 
+COPY cic-eth/ .
 RUN	pip install $pip_extra_index_url_flag .
+RUN	pip install $pip_extra_index_url_flag pycryptodome==3.10.1
+
+# --- TEST IMAGE ---
+FROM python:3.8.6-slim-buster as test 
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+WORKDIR /usr/src/cic-eth
+
+RUN /usr/local/bin/python -m pip install --upgrade pip
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+# TODO we could use venv inside container to isolate the system and app deps further
+# COPY --from=compile /usr/src/cic-eth/ .
+# RUN . venv/bin/activate
+
+COPY cic-eth/test_requirements.txt .
+RUN	pip install $pip_extra_index_url_flag -r test_requirements.txt 
+
+COPY cic-eth . 
+
+ENV PYTHONPATH .
+
+ENTRYPOINT ["pytest"]
+
+# --- RUNTIME ---
+FROM python:3.8.6-slim-buster as runtime 
+
+RUN apt-get update && \
+	apt install -y gnupg libpq-dev procps
+
+WORKDIR /usr/src/cic-eth
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
 
 COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
-COPY cic-eth/tests/ tests/
 
+COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
@@ -51,3 +66,4 @@ COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
 COPY util/liveness/health.sh /usr/local/bin/health.sh
+

apps/cic-eth/requirements.txt

@@ -1,25 +1,24 @@
-cic-base==0.1.2b5
+cic-base~=0.1.2b11
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.4a16
-#cic-bancor~=0.0.6
+cic-eth-registry~=0.5.5a4
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a9
-erc20-transfer-authorization~=0.3.1a5
+eth_accounts_index~=0.0.11a12
+erc20-transfer-authorization~=0.3.1a6
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a9
-chainlib~=0.0.2a20
+eth-address-index~=0.1.1a11
+chainlib~=0.0.3a2
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a2
+chainsyncer[sql]~=0.0.2a4
 chainqueue~=0.0.2a2
-pysha3==1.0.2
+sarafu-faucet==0.0.3a3
+erc20-faucet==0.2.1a4
 coincurve==15.0.0
-sarafu-faucet==0.0.2a28
-potaahto~=0.0.1a1
+potaahto~=0.0.1a2

apps/cic-eth/setup.py

@@ -11,17 +11,6 @@ while True:
     requirements.append(l.rstrip())
 f.close()
 
-test_requirements = []
-f = open('test_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    test_requirements.append(l.rstrip())
-f.close()
-
-
 setup(
-    install_requires=requirements,
-    tests_require=test_requirements,
+    install_requires=requirements
         )

apps/cic-eth/test_requirements.txt

@@ -4,4 +4,3 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-giftable-erc20-token==0.0.8a9

apps/cic-eth/tests/conftest.py

@@ -4,7 +4,7 @@ import sys
 import logging
 
 # external imports
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.api import Api

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -14,9 +14,9 @@ from chainlib.eth.tx import (
         Tx,
         )
 from chainlib.eth.block import Block
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from sarafu_faucet import MinterFaucet
-from eth_accounts_index import AccountRegistry
+from eth_accounts_index.registry import AccountRegistry
 from potaahto.symbols import snake_and_camel
 from hexathon import add_0x
 
@@ -26,7 +26,6 @@ from cic_eth.runnable.daemons.filters.callback import CallbackFilter
 logg = logging.getLogger()
 
 
-@pytest.mark.skip()
 def test_transfer_tx(
         default_chain_spec,
         init_database,
@@ -66,7 +65,6 @@ def test_transfer_tx(
     assert transfer_type == 'transfer'
 
 
-@pytest.mark.skip()
 def test_transfer_from_tx(
         default_chain_spec,
         init_database,

apps/cic-eth/tests/task/api/test_list.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import pytest
 from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from chainlib.eth.tx import receipt
 
 # local imports

apps/cic-eth/tests/task/test_task_account.py

@@ -9,7 +9,7 @@ import celery
 from chainlib.connection import RPCConnection
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import receipt
-from eth_accounts_index import AccountRegistry
+from eth_accounts_index.registry import AccountRegistry
 from hexathon import strip_0x
 from chainqueue.db.enum import StatusEnum
 from chainqueue.db.models.otx import Otx

apps/cic-eth/tests/task/test_task_erc20.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import pytest
 import celery
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,

apps/cic-eth/tests/unit/ext/test_address.py

@@ -3,7 +3,7 @@ from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
         )
-from eth_address_declarator import AddressDeclarator
+from eth_address_declarator import Declarator
 from hexathon import add_0x
 
 # local imports
@@ -23,7 +23,7 @@ def test_translate(
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
 
-    c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    c = Declarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
 
     description = 'alice'.encode('utf-8').ljust(32, b'\x00').hex()
     (tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], add_0x(description))

apps/cic-eth/tests/unit/ext/test_ext_tx.py

@@ -8,7 +8,7 @@ from chainlib.eth.tx import (
         count,
         receipt,
         )
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
 
 # local imports

apps/cic-meta/bin/get.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+const colors = require('colors');
+const {Meta} = require("../dist");
+
+let { argv } = require('yargs')
+    .usage('Usage: $0 -m http://localhost:63380 -n publickeys')
+    .example(
+        '$0 -m http://localhost:63380 -n publickeys',
+        'Fetches the public keys blob from the meta server'
+    )
+    .option('m', {
+        alias: 'metaurl',
+        describe: 'The URL for the meta service',
+        demandOption: 'The meta url is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('n', {
+        alias: 'name',
+        describe: 'The name of the resource to be fetched from the meta service',
+        demandOption: 'The name of the resource is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('t', {
+        alias: 'type',
+        describe: 'The type of resource to be fetched from the meta service\n' +
+            'Options: `user`, `phone` and `custom`\n' +
+            'Defaults to `custom`',
+        type: 'string',
+        nargs: 1,
+    })
+    .epilog('Grassroots Economics (c) 2021')
+    .wrap(null);
+
+const metaUrl = argv.m;
+const resourceName = argv.n;
+let type = argv.t;
+if (type === undefined) {
+    type = 'custom'
+}
+
+(async () => {
+    const identifier = await Meta.getIdentifier(resourceName, type);
+    console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
+    const metaResponse = await Meta.get(identifier, metaUrl);
+    if (typeof metaResponse !== "object") {
+        console.error(colors.red('Metadata get failed!'));
+    }
+    console.log(colors.green(metaResponse));
+})();

apps/cic-meta/bin/set.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const colors = require('colors');
+const {Meta} = require("../dist");
+
+let { argv } = require('yargs')
+    .usage('Usage: $0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc')
+    .example(
+        '$0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc',
+        'Updates the public keys blob to the meta server'
+    )
+    .option('m', {
+        alias: 'metaurl',
+        describe: 'The URL for the meta service',
+        demandOption: 'The meta url is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('k', {
+        alias: 'privatekey',
+        describe: 'The PGP private key blob file used to sign the changes to the meta service',
+        demandOption: 'The private key file is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('n', {
+        alias: 'name',
+        describe: 'The name of the resource to be set or updated to the meta service',
+        demandOption: 'The name of the resource is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('r', {
+        alias: 'resource',
+        describe: 'The resource file to be set or updated to the meta service',
+        demandOption: 'The resource file is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('t', {
+        alias: 'type',
+        describe: 'The type of resource to be set or updated to the meta service\n' +
+            'Options: `user`, `phone` and `custom`\n' +
+            'Defaults to `custom`',
+        type: 'string',
+        nargs: 1,
+    })
+    .epilog('Grassroots Economics (c) 2021')
+    .wrap(null);
+
+const metaUrl = argv.m;
+const privateKeyFile = argv.k;
+const resourceName = argv.n;
+const resourceFile = argv.r;
+let type = argv.t;
+if (type === undefined) {
+    type = 'custom'
+}
+
+const privateKey = readFile(privateKeyFile);
+const resource = readFile(resourceFile);
+
+(async () => {
+    if (privateKey && resource) {
+        const identifier = await Meta.getIdentifier(resourceName, type);
+        console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
+        const meta = new Meta(metaUrl, privateKey);
+        meta.onload = async (status) => {
+            const response = await meta.set(identifier, resource)
+            console.log(colors.green(response));
+        }
+    }
+})();
+
+function readFile(filename) {
+    if(!fs.existsSync(filename)) {
+        console.log(colors.red(`File ${filename} not found`));
+        return;
+    }
+    return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
+}

apps/cic-meta/package.json

@@ -1,9 +1,14 @@
 {
-	"name": "cic-client-meta",
-	"version": "0.0.7-alpha.8",
+	"name": "@cicnet/cic-client-meta",
+	"version": "0.0.11",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
+	"bin": {
+		"meta-set": "bin/set.js",
+		"meta-get": "bin/get.js"
+	},
+	"preferGlobal": true,
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
 		"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
@@ -11,12 +16,14 @@
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
 		"clean": "rm -rf dist",
 		"prepare": "npm run build && npm run build-server",
-                "start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts" 
+		"start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts",
+		"publish": "npm publish --access public"
 	},
 	"dependencies": {
+		"@cicnet/crdt-meta": "^0.0.10",
 		"@ethereumjs/tx": "^3.0.0-beta.1",
 		"automerge": "^0.14.1",
-		"crdt-meta": "0.0.8",
+		"colors": "^1.4.0",
 		"ethereumjs-wallet": "^1.0.1",
 		"ini": "^1.3.8",
 		"openpgp": "^4.10.8",

apps/cic-meta/scripts/dumpconfig.js

@@ -1,4 +1,4 @@
-import { Config } from 'crdt-meta';
+import { Config } from '@cicnet/crdt-meta';
 const fs = require('fs');
 
 if (process.argv[2] === undefined) {

apps/cic-meta/scripts/server/handlers.ts

@@ -1,7 +1,7 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
 
-import { Envelope, Syncable } from 'crdt-meta';
+import { Envelope, Syncable } from '@cicnet/crdt-meta';
 
 
 function handleNoMergeGet(db, digest, keystore) {

apps/cic-meta/scripts/server/server.ts

@@ -3,7 +3,8 @@ import * as fs from 'fs';
 import * as path from 'path';
 
 import * as handlers from './handlers';
-import { PGPKeyStore, PGPSigner, Config, SqliteAdapter, PostgresAdapter } from 'crdt-meta';
+import { PGPKeyStore, PGPSigner, Config } from '@cicnet/crdt-meta';
+import { SqliteAdapter, PostgresAdapter } from '../../src/db';
 
 import { standardArgs } from './args';
 

apps/cic-meta/src/custom.ts

@@ -0,0 +1,27 @@
+import {Addressable, mergeKey, Syncable} from "@cicnet/crdt-meta";
+
+class Custom extends Syncable implements Addressable {
+
+    name:	string
+    value:	Object
+
+    constructor(name:string, v:Object={}) {
+        super('', v);
+        Custom.toKey(name).then((cid) => {
+            this.id = cid;
+            this.value = v;
+        });
+    }
+
+    public static async toKey(item:string, identifier: string = ':cic.custom') {
+        return await mergeKey(Buffer.from(item), Buffer.from(identifier));
+    }
+
+    public key(): string {
+        return this.id;
+    }
+}
+
+export {
+    Custom,
+}

apps/cic-meta/src/db.ts

@@ -0,0 +1,90 @@
+import * as pg from 'pg';
+import * as sqlite from 'sqlite3';
+
+type DbConfig = {
+	name:		string
+	host:		string
+	port:		number
+	user:		string
+	password:	string	
+}
+
+interface DbAdapter {
+	query:	(s:string, callback:(e:any, rs:any) => void) => void
+	close: 	() => void
+}
+
+const re_creatematch = /^(CREATE)/i
+const re_getmatch = /^(SELECT)/i;
+const re_setmatch = /^(INSERT|UPDATE)/i;
+
+class SqliteAdapter implements DbAdapter {
+
+	db:		any
+
+	constructor(dbConfig:DbConfig, callback?:(any) => void) {
+		this.db = new sqlite.Database(dbConfig.name); //, callback);
+	}
+
+	public query(s:string, callback:(e:any, rs?:any) => void): void {
+		const local_callback = (e, rs) => {
+			let r = undefined;
+			if (rs !== undefined) {
+				r = {
+					rowCount: rs.length,
+					rows: rs,
+				}
+			}
+			callback(e, r);
+		};
+		if (s.match(re_getmatch)) {
+			this.db.all(s, local_callback);
+		} else if (s.match(re_setmatch)) {
+			this.db.run(s, local_callback);
+		} else if (s.match(re_creatematch)) {
+			this.db.run(s, callback);
+		} else {
+			throw 'unhandled query';
+		}
+	}
+
+	public close() {
+		this.db.close();
+	}
+}
+
+class PostgresAdapter implements DbAdapter {
+
+	db:		any
+
+	constructor(dbConfig:DbConfig) {
+		let o = dbConfig;
+		o['database'] = o.name;
+		this.db = new pg.Pool(o);
+		return this.db;
+	}
+
+	public query(s:string, callback:(e:any, rs:any) => void): void {
+		this.db.query(s, (e, rs) => {
+			let r = {
+				length: rs.rowCount,
+			}
+			rs.length = rs.rowCount;
+			if (e === undefined) {
+				e = null;
+			}
+			console.debug(e, rs);
+			callback(e, rs);
+		});
+	}
+
+	public close() {
+		this.db.end();
+	}
+}
+
+export {
+	DbConfig,
+	SqliteAdapter,
+	PostgresAdapter,
+}

apps/cic-meta/src/index.ts

@@ -1,2 +1,4 @@
 export { User } from './user';
 export { Phone } from './phone';
+export { Custom } from './custom';
+export { Meta } from './meta';

apps/cic-meta/src/meta.ts

@@ -0,0 +1,126 @@
+import {ArgPair, Envelope, Syncable, MutablePgpKeyStore, PGPSigner} from "@cicnet/crdt-meta";
+import {User} from "./user";
+import {Phone} from "./phone";
+import {Custom} from "./custom";
+const fetch = require("node-fetch");
+
+const headers =  {
+    'Content-Type': 'application/json;charset=utf-8',
+    'x-cic-automerge': 'client'
+};
+const options = {
+    headers: headers,
+};
+
+class Meta {
+    keystore: MutablePgpKeyStore = new MutablePgpKeyStore();
+    signer: PGPSigner = new PGPSigner(this.keystore);
+    metaUrl: string;
+    private privateKey: string;
+    onload: (status: boolean) => void;
+
+    constructor(metaUrl: string, privateKey: any) {
+        this.metaUrl = metaUrl;
+        this.privateKey = privateKey;
+        this.keystore.loadKeyring().then(() => {
+            this.keystore.importPrivateKey(privateKey).then(() => this.onload(true));
+        });
+    }
+
+    async set(identifier: string, data: Object): Promise<any> {
+        let syncable: Syncable;
+        const response = await Meta.get(identifier, this.metaUrl);
+        if (response === `Request to ${this.metaUrl}/${identifier} failed. Connection error.`) {
+            return response;
+        } else if (typeof response !== "object" || typeof data !== "object") {
+            syncable = new Syncable(identifier, data);
+            const res = await this.updateMeta(syncable, identifier);
+            return `${res.status}: ${res.statusText}`;
+        } else {
+            syncable = await Meta.get(identifier, this.metaUrl);
+            let update: Array<ArgPair> = [];
+            for (const prop in data) {
+                update.push(new ArgPair(prop, data[prop]));
+            }
+            syncable.update(update, 'client-branch');
+            const res = await this.updateMeta(syncable, identifier);
+            return `${res.status}: ${res.statusText}`;
+        }
+    }
+
+    async updateMeta(syncable: Syncable, identifier: string): Promise<any> {
+        const envelope: Envelope = await this.wrap(syncable);
+        const reqBody: string = envelope.toJSON();
+        const putOptions = {
+            method: 'PUT',
+            headers: headers,
+            body: reqBody
+        };
+        return await fetch(`${this.metaUrl}/${identifier}`, putOptions).then(async response => {
+            if (response.ok) {
+                return Promise.resolve({
+                    status: response.status,
+                    statusText: response.statusText + ', Metadata updated successfully!'
+                });
+            } else {
+                return Promise.reject({
+                    status: response.status,
+                    statusText: response.statusText
+                });
+            }
+        });
+    }
+
+    static async get(identifier: string, metaUrl: string): Promise<any> {
+        const response = await fetch(`${metaUrl}/${identifier}`, options).then(response => {
+            if (response.ok) {
+                return (response.json());
+            } else {
+                return Promise.reject({
+                    status: response.status,
+                    statusText: response.statusText
+                });
+            }
+        }).catch(error => {
+            if (error.code === 'ECONNREFUSED') {
+                return `Request to ${metaUrl}/${identifier} failed. Connection error.`
+            }
+            return `${error.status}: ${error.statusText}`;
+        });
+        if (typeof response !== "object") {
+            return response;
+        }
+        return Envelope.fromJSON(JSON.stringify(response)).unwrap();
+    }
+
+    static async getIdentifier(name: string, type: string = 'custom'): Promise<string> {
+        let identifier: string;
+        type = type.toLowerCase();
+        if (type === 'user') {
+            identifier = await User.toKey(name);
+        } else if (type === 'phone') {
+            identifier = await Phone.toKey(name);
+        } else {
+            identifier = await Custom.toKey(name);
+        }
+        return identifier;
+    }
+
+    private wrap(syncable: Syncable): Promise<Envelope> {
+        return new Promise<Envelope>(async (resolve, reject) => {
+            syncable.setSigner(this.signer);
+            syncable.onwrap = async (env) => {
+                if (env === undefined) {
+                    reject();
+                    return;
+                }
+                resolve(env);
+            };
+            syncable.sign();
+        });
+    }
+}
+
+export {
+    Meta,
+}

apps/cic-meta/src/phone.ts

@@ -1,4 +1,4 @@
-import { Syncable, Addressable, mergeKey } from 'crdt-meta';
+import { Syncable, Addressable, mergeKey } from '@cicnet/crdt-meta';
 
 class Phone extends Syncable implements Addressable {
 

apps/cic-meta/src/user.ts

@@ -1,4 +1,4 @@
-import { Syncable, Addressable, toAddressKey } from 'crdt-meta';
+import { Syncable, Addressable, toAddressKey } from '@cicnet/crdt-meta';
 
 const keySalt = new TextEncoder().encode(':cic.person');
 class User extends Syncable implements Addressable {

apps/cic-meta/tests/server.ts

@@ -4,7 +4,8 @@ import pgp = require('openpgp');
 import sqlite = require('sqlite3');
 
 import * as handlers from '../scripts/server/handlers';
-import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer, SqliteAdapter } from 'crdt-meta';
+import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer } from '@cicnet/crdt-meta';
+import { SqliteAdapter } from '../src/db';
 
 function createKeystore() {
 	const pksa = fs.readFileSync(__dirname + '/privatekeys.asc', 'utf-8');

apps/cic-meta/tsconfig.json

@@ -19,6 +19,7 @@
   "include": [
     "src/**/*",
     "scripts/server/*",
-    "index.ts"
+    "index.ts",
+    "bin"
   ]
 }

apps/cic-notify/.config/africastalking.ini

@@ -1,4 +1,4 @@
 [AFRICASTALKING]
-api_username = foo
-api_key = bar
-api_sender_id = baz
+api_username =
+api_key =
+api_sender_id =

apps/cic-notify/cic_notify/error.py

@@ -9,3 +9,7 @@ class AlreadyInitializedError(Exception):
 class PleaseCommitFirstError(Exception):
     """Raised when there exists uncommitted changes in the code while trying to build out the package."""
     pass
+
+
+class NotificationSendError(Exception):
+    """Raised when a notification failed to due to some error as per the service responsible for dispatching the notification."""

apps/cic-notify/cic_notify/ext/enums.py

@@ -0,0 +1,19 @@
+# standard imports
+from enum import IntEnum
+
+
+class AfricasTalkingStatusCodes(IntEnum):
+    PROCESSED = 100
+    SENT = 101
+    QUEUED = 102
+    RISK_HOLD = 401
+    INVALID_SENDER_ID = 402
+    INVALID_PHONE_NUMBER = 403
+    UNSUPPORTED_NUMBER_TYPE = 404
+    INSUFFICIENT_BALANCE = 405
+    USER_IN_BLACKLIST = 406
+    COULD_NOT_ROUTE = 407
+    INTERNAL_SERVER_ERROR = 500
+    GATEWAY_ERROR = 501
+    REJECTED_BY_GATEWAY = 502
+

apps/cic-notify/cic_notify/tasks/sms/africastalking.py

@@ -6,7 +6,8 @@ import celery
 import africastalking
 
 # local imports
-from cic_notify.error import NotInitializedError, AlreadyInitializedError
+from cic_notify.error import NotInitializedError, AlreadyInitializedError, NotificationSendError
+from cic_notify.ext.enums import AfricasTalkingStatusCodes
 
 logg = logging.getLogger()
 celery_app = celery.current_app
@@ -50,10 +51,27 @@ class AfricasTalkingNotifier:
         if self.sender_id:
             response = self.api_client.send(message=message, recipients=[recipient], sender_id=self.sender_id)
             logg.debug(f'Africastalking response sender-id {response}')
+
         else:
             response = self.api_client.send(message=message, recipients=[recipient])
             logg.debug(f'africastalking response no-sender-id {response}')
 
+        recipients = response.get('Recipients')
+
+        if len(recipients) != 1:
+            status = response.get('SMSMessageData').get('Message')
+            raise NotificationSendError(f'Unexpected number of recipients: {len(recipients)}. Status: {status}')
+
+        status_code = recipients[0].get('statusCode')
+        status = recipients[0].get('status')
+
+        if status_code not in [
+            AfricasTalkingStatusCodes.PROCESSED.value,
+            AfricasTalkingStatusCodes.SENT.value,
+            AfricasTalkingStatusCodes.QUEUED.value
+        ]:
+            raise NotificationSendError(f'Sending notification failed due to: {status}')
+
 
 @celery_app.task
 def send(message, recipient):

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.4')
+version = (0, 4, 0, 'alpha.5')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/.config/app.ini

@@ -4,7 +4,7 @@ LOCALE_FALLBACK=en
 LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
-SERVICE_CODE=*483*46#
+SERVICE_CODE=*483*46#,*483*061#,*384*96#
 
 [phone_number]
 REGION=KE

apps/cic-ussd/cic_ussd/conversions.py

@@ -24,7 +24,7 @@ def from_wei(value: int) -> float:
     """This function converts values in Wei to a token in the cic network.
     :param value: Value in Wei
     :type value: int
-    :return: SRF equivalent of value in Wei
+    :return: platform's default token equivalent of value in Wei
     :rtype: float
     """
     value = float(value) / 1e+6
@@ -33,9 +33,9 @@ def from_wei(value: int) -> float:
 
 def to_wei(value: int) -> int:
     """This functions converts values from a token in the cic network to Wei.
-    :param value: Value in SRF
+    :param value: Value in platform's default token
     :type value: int
-    :return: Wei equivalent of value in SRF
+    :return: Wei equivalent of value in platform's default token
     :rtype: int
     """
-    return int(value * 1e+6)
+    return int(value * 1e+6)

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -102,7 +102,7 @@ class MetadataRequestsHandler(Metadata):
                 'digest': json.loads(data).get('digest'),
             }
         }
-        formatted_data = json.dumps(formatted_data).encode('utf-8')
+        formatted_data = json.dumps(formatted_data)
         result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
         logg.info(f'signed metadata submission status: {result.status_code}.')
         metadata_http_error_handler(result=result)
@@ -116,11 +116,13 @@ class MetadataRequestsHandler(Metadata):
         """This function is responsible for querying the metadata server for data corresponding to a unique pointer."""
         result = make_request(method='GET', url=self.url)
         metadata_http_error_handler(result=result)
-        response_data = result.content
-        data = json.loads(response_data.decode('utf-8'))
+        response_data = result.json()
+        data = json.loads(response_data)
+        if not isinstance(data, dict):
+            raise ValueError(f'Invalid data object: {data}.')
         if result.status_code == 200 and self.cic_type == ':cic.person':
             person = Person()
-            deserialized_person = person.deserialize(person_data=json.loads(data))
+            deserialized_person = person.deserialize(person_data=data)
             data = json.dumps(deserialized_person.serialize())
             cache_data(self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/operations.py

@@ -325,6 +325,14 @@ def process_menu_interaction_requests(chain_str: str,
         # get user
         user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
+        # retrieve and cache user's metadata
+        blockchain_address = user.blockchain_address
+        s_query_person_metadata = celery.signature(
+            'cic_ussd.tasks.metadata.query_person_metadata',
+            [blockchain_address]
+        )
+        s_query_person_metadata.apply_async(queue='cic-ussd')
+
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
             external_session_id=external_session_id).first()

apps/cic-ussd/cic_ussd/processor.py

@@ -371,13 +371,6 @@ def process_start_menu(display_key: str, user: Account):
     # get operational balance
     operational_balance = compute_operational_balance(balances=balances_data)
 
-    # retrieve and cache account's metadata
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
-        [blockchain_address]
-    )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
-
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -13,7 +13,7 @@ import argparse
 import logging
 import urllib
 from xdg.BaseDirectory import xdg_config_home
-from urllib import request
+from urllib import parse, request
 
 # third-party imports
 from confini import Config
@@ -92,9 +92,9 @@ def main():
             data['text'] = user_input
 
         req = urllib.request.Request(url)
-        data_str = json.dumps(data)
-        data_bytes = data_str.encode('utf-8')
-        req.add_header('Content-Type', 'application/json')
+        urlencoded_data = parse.urlencode(data)
+        data_bytes = urlencoded_data.encode('utf-8')
+        req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         req.data = data_bytes
         response = urllib.request.urlopen(req)
         response_data = response.read().decode('utf-8')

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -4,6 +4,7 @@
 # standard imports
 import json
 import logging
+from urllib.parse import parse_qs
 
 # third-party imports
 import celery
@@ -33,8 +34,7 @@ from cic_ussd.requests import (get_request_endpoint,
 from cic_ussd.runnable.server_base import exportable_parser, logg
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.state_machine import UssdStateMachine
-from cic_ussd.validator import check_ip, check_request_content_length, check_service_code, validate_phone_number, \
-    validate_presence
+from cic_ussd.validator import check_ip, check_request_content_length, validate_phone_number, validate_presence
 
 args = exportable_parser.parse_args()
 
@@ -124,6 +124,9 @@ else:
     raise InitializationError(f'Default token data for: {chain_str} not found.')
 
 
+valid_service_codes = config.get('APP_SERVICE_CODE').split(",")
+
+
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
     :param env: Object containing server and request information
@@ -139,13 +142,27 @@ def application(env, start_response):
 
     if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/':
 
-        # get post data
-        post_data = json.load(env.get('wsgi.input'))
+        if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded':
+            start_response('405 Play by the rules', errors_headers)
+            return []
 
-        service_code = post_data.get('serviceCode')
-        phone_number = post_data.get('phoneNumber')
-        external_session_id = post_data.get('sessionId')
-        user_input = post_data.get('text')
+        post_data = env.get('wsgi.input').read()
+        post_data = post_data.decode('utf-8')
+
+        try:
+            post_data = parse_qs(post_data)
+        except TypeError:
+            start_response('400 Size matters', errors_headers)
+            return []
+
+        service_code = post_data.get('serviceCode')[0]
+        phone_number = post_data.get('phoneNumber')[0]
+        external_session_id = post_data.get('sessionId')[0]
+
+        try:
+            user_input = post_data.get('text')[0]
+        except TypeError:
+            user_input = ""
 
         # add validation for phone number
         if phone_number:
@@ -162,14 +179,14 @@ def application(env, start_response):
             return []
 
         # validate service code
-        if not check_service_code(code=service_code, config=config):
+        if service_code not in valid_service_codes:
             response = define_multilingual_responses(
                 key='ussd.kenya.invalid_service_code',
                 locales=['en', 'sw'],
                 prefix='END',
-                valid_service_code=config.get('APP_SERVICE_CODE'))
-            response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
-            start_response('400 Invalid service code', headers)
+                valid_service_code=valid_service_codes[0])
+            response_bytes, headers = define_response_with_content(headers=headers, response=response)
+            start_response('200 OK', headers)
             return [response_bytes]
 
         # validate phone number
@@ -192,3 +209,8 @@ def application(env, start_response):
         start_response('200 OK,', headers)
         SessionBase.session.close()
         return [response_bytes]
+
+    else:
+        start_response('405 Play by the rules', errors_headers)
+        return []
+

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -12,6 +12,7 @@ from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
 from cic_ussd.phone_number import get_user_by_phone_number
+from cic_ussd.processor import retrieve_token_symbol
 from cic_ussd.redis import create_cached_data_key, get_cached_data
 from cic_ussd.transactions import OutgoingTransactionProcessor
 
@@ -124,14 +125,18 @@ def process_transaction_request(state_machine_data: Tuple[str, dict, Account]):
     """
     user_input, ussd_session, user = state_machine_data
 
+    # retrieve token symbol
+    chain_str = Chain.spec.__str__()
+
     # get user from phone number
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
     to_address = recipient.blockchain_address
     from_address = user.blockchain_address
     amount = int(ussd_session.get('session_data').get('transaction_amount'))
-    chain_str = Chain.spec.__str__()
+    token_symbol = retrieve_token_symbol(chain_str=chain_str)
+
     outgoing_tx_processor = OutgoingTransactionProcessor(chain_str=chain_str,
                                                          from_address=from_address,
                                                          to_address=to_address)
-    outgoing_tx_processor.process_outgoing_transfer_transaction(amount=amount)
+    outgoing_tx_processor.process_outgoing_transfer_transaction(amount=amount, token_symbol=token_symbol)

apps/cic-ussd/cic_ussd/transactions.py

@@ -120,7 +120,7 @@ class OutgoingTransactionProcessor:
         self.from_address = from_address
         self.to_address = to_address
 
-    def process_outgoing_transfer_transaction(self, amount: int, token_symbol='SRF'):
+    def process_outgoing_transfer_transaction(self, amount: int, token_symbol: str):
         """This function initiates standard transfers between one account to another
         :param amount: The amount of tokens to be sent
         :type amount: int

apps/cic-ussd/cic_ussd/validator.py

@@ -45,19 +45,6 @@ def check_request_content_length(config: Config, env: dict):
         config.get('APP_MAX_BODY_LENGTH'))
 
 
-def check_service_code(code: str, config: Config):
-    """Checks whether provided code matches expected service code
-    :param config: A dictionary object containing configuration values
-    :type config: Config
-    :param code: Service code passed over request
-    :type code: str
-
-    :return: Service code validity
-    :rtype: boolean
-    """
-    return code == config.get('APP_SERVICE_CODE')
-
-
 def check_known_user(phone: str):
     """
     This method attempts to ascertain whether the user already exists and is known to the system.

apps/cic-ussd/var/lib/locale/sms.en.yml

@@ -1,7 +1,7 @@
 en:
   account_successfully_created: |-
-    Hello, you have been registered on Sarafu Network! Your balance is %{balance} %{token_symbol}. To use dial *483*46#. For help 0757628885.
+    You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
   received_tokens: |-
     Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
   terms: |-
-    By using the service, you agree to the terms and conditions at https://www.grassrootseconomics.org/terms-and-conditions.
+    By using the service, you agree to the terms and conditions at http://grassecon.org/tos

apps/cic-ussd/var/lib/locale/sms.sw.yml

@@ -1,7 +1,7 @@
 sw:
   account_successfully_created: |-
-    Habari, umesajiliwa kwa huduma ya sarafu! Salio lako ni %{token_symbol} %{balance}. Kutumia bonyeza *483*46#. Kwa Usaidizi 0757628885.
+    Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
   received_tokens: |-
     Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}.
   terms: |-
-    Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo https://www.grassrootseconomics.org/terms-and-conditions.
+    Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -1,29 +1,30 @@
 en:
   kenya:
     initial_language_selection: |-
-      CON Welcome to Sarafu
+      CON Welcome to Sarafu Network
       1. English
       2. Kiswahili
       3. Help
     initial_pin_entry: |-
-      CON Please enter a PIN to manage your account.
+      CON Please enter a new four number PIN for your account. 
       0. Back
     initial_pin_confirmation: |-
-      CON Enter your PIN again
+      CON Enter your four number PIN again
       0. Back
     enter_given_name: |-
       CON Enter first name
       0. Back
     enter_family_name: |-
-      CON Enter last name
+      CON Enter family name
       0. Back
     enter_gender: |-
       CON Enter gender
       1. Male
       2. Female
+      3. Other
       0. Back
     enter_location: |-
-      CON Enter location
+      CON Enter your location
       0. Back
     enter_products: |-
       CON Please enter a product or service you offer
@@ -83,34 +84,34 @@ en:
         Please enter your PIN to confirm.
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     display_metadata_pin_authorization:
       first: |-
-        CON Please enter your PIN.
+        CON Please enter your PIN
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_balances_pin_authorization:
       first: |-
-        CON Please enter your PIN to view balances.
+        CON Please enter your PIN to view balances
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_statement_pin_authorization:
       first: |-
-        CON Please enter your PIN to view statement.
+        CON Please enter your PIN to view statement
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_balances: |-
       CON Your balances are as follows:
         balance: %{operational_balance} %{token_symbol}
-        taxes: %{tax} %{token_symbol}
-        bonsuses: %{bonus} %{token_symbol}
+        fees: %{tax} %{token_symbol}
+        rewards: %{bonus} %{token_symbol}
       0. Back
     first_transaction_set: |-
       CON %{first_transaction_set}
@@ -140,9 +141,9 @@ en:
     exit_pin_blocked: |-
       END Your PIN has been blocked. For help, please call %{support_phone}.
     exit_invalid_pin: |-
-      END The PIN you have entered is Invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
+      END The PIN you have entered is invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
     exit_invalid_new_pin: |-
-      END The PIN you have entered is Invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
+      END The PIN you have entered is invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
     exit_pin_mismatch: |-
       END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}.
     exit_invalid_recipient: |-
@@ -158,6 +159,8 @@ en:
       Your Sarafu-Network balances is: %{token_balance}
       00. Back
       99. Exit
+    invalid_service_code: |-
+      Please dial %{valid_service_code} to access Sarafu Network
     help: |-
       CON For assistance call %{support_phone}
       00. Back
@@ -167,4 +170,4 @@ en:
       00. Back
       99. Exit
     account_creation_prompt: |-
-      Your account is being created. You will receive an SMS when your account is ready.
+      Your account is being created. You will receive an SMS when your account is ready.

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -1,12 +1,12 @@
 sw:
   kenya:
     initial_language_selection: |-
-      CON Welcome to Sarafu
+      CON Karibu Sarafu Network
       1. English
       2. Kiswahili
       3. Help
     initial_pin_entry: |-
-      CON Tafadhali weka PIN ili kudhibiti akaunti yako.
+      CON Tafadhali weka pin mpya yenye nambari nne kwa akaunti yako
       0. Nyuma
     initial_pin_confirmation: |-
       CON Weka PIN yako tena
@@ -21,12 +21,13 @@ sw:
       CON Weka jinsia yako
       1. Mwanaume
       2. Mwanamke
+      3. Nyngine 
       0. Nyuma
     enter_location: |-
       CON Weka eneo lako
       0. Nyuma
     enter_products: |-
-      CON Tafadhali weka bidhaa ama huduma unauza
+      CON Weka bidhaa ama huduma unauza
       0. Nyuma
     start: |-
       CON Salio %{account_balance} %{account_token_name}
@@ -155,9 +156,11 @@ sw:
       99. Ondoka
     exit_insufficient_balance: |-
       CON Malipo ya %{amount} %{token_symbol} kwa %{recipient_information} halijakamilika kwa sababu salio lako haitoshi.
-      Akaunti yako ya Sarafu-Network ina salio ifuatayo: %{token_balance}
+      Akaunti yako ya Sarafu ina salio ifuatayo: %{token_balance}
       00. Nyuma
       99. Ondoka
+    invalid_service_code: |-
+      Bonyeza %{valid_service_code} kutumia mtandao wa Sarafu
     help: |-
       CON Kwa usaidizi piga simu %{support_phone}
       0. Nyuma
@@ -167,4 +170,4 @@ sw:
       00. Nyuma
       99. Ondoka
     account_creation_prompt: |-
-      Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.
+      Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.

apps/contract-migration/docker/Dockerfile

@@ -47,54 +47,60 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh |
 ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
 ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
 
-RUN useradd --create-home grassroots
-WORKDIR /home/grassroots
-USER grassroots
+#RUN useradd --create-home grassroots
+# WORKDIR /home/grassroots
+# USER grassroots
 
+ARG pip_extra_args=""
+ARG pip_index_url=https://pypi.org/simple
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a79
-ARG cic_eth_version=0.11.0b8+build.c2286e5c
-ARG sarafu_faucet_version=0.0.2a28
-ARG sarafu_token_version=0.0.1a6
-ARG cic_contracts_version=0.0.2a2
-RUN pip install --user --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
+ARG cic_base_version=0.1.2b11
+ARG cic_eth_version=0.11.0b14
+ARG sarafu_token_version=0.0.1a8
+ARG sarafu_faucet_version=0.0.3a3
+RUN pip install --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url \
+	cic-base[full_graph]==$cic_base_version \
 	cic-eth==$cic_eth_version \
-	cic-contracts==$cic_contracts_version \
 	sarafu-faucet==$sarafu_faucet_version \
-	sarafu-token==$sarafu_token_version
+	sarafu-token==$sarafu_token_version \
+	cic-eth==$cic_eth_version 
 
+# -------------- begin runtime container ---------------- 
 FROM python:3.8.6-slim-buster as runtime-image
 
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
-RUN apt-get install -y --no-install-recommends jq
+RUN apt-get install -y jq bash iputils-ping socat
 
 COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
 COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
+COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
 
-RUN useradd --create-home grassroots
-WORKDIR /home/grassroots
-# COPY python dependencies to user dir
-COPY --from=compile-image /home/grassroots/.local .local
-ENV PATH=/home/grassroots/.local/bin:$PATH
+ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
+# RUN useradd -u 1001 --create-home grassroots
+# RUN adduser grassroots sudo && \
+#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+# WORKDIR /home/grassroots
 
 COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
 COPY contract-migration/envlist .
 
-# RUN chown grassroots:grassroots .local/
-
-RUN mkdir -p /tmp/cic/config
-RUN chown grassroots:grassroots /tmp/cic/config
 # A shared output dir for environment configs
+RUN mkdir -p /tmp/cic/config
+# RUN chown grassroots:grassroots /tmp/cic/config
 RUN chmod a+rwx /tmp/cic/config 
 
 COPY contract-migration/*.sh ./
-RUN chown grassroots:grassroots -R .
+# RUN chown grassroots:grassroots -R .
 RUN chmod gu+x *.sh
 
+# we copied these from the root build container.
+# this is dumb though...I guess the compile image should have the same user
+# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
 
-USER grassroots
+# USER grassroots
 
 ENTRYPOINT [ ]