Remove leading '0x' from saved address.

# Conflicts:
#	apps/contract-migration/scripts/package-lock.json
#	apps/contract-migration/scripts/package.json

.gitignore

@@ -4,7 +4,3 @@ service-configs/*
 __pycache__
 *.pyc
 *.o
-gmon.out
-*.egg-info
-dist/
-build/

apps/cic-cache/.config/database.ini

@@ -6,4 +6,3 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=

apps/cic-cache/.config/test/database.ini

@@ -6,4 +6,3 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=

apps/cic-cache/cic_cache/db/__init__.py

@@ -2,14 +2,9 @@
 import logging
 
 # local imports
-from .list import (
-        list_transactions_mined,
-        list_transactions_account_mined,
-        add_transaction,
-        tag_transaction,
-        add_tag,
-    )
-
+from .list import list_transactions_mined
+from .list import list_transactions_account_mined
+from .list import add_transaction
 
 logg = logging.getLogger()
 

apps/cic-cache/cic_cache/db/list.py

@@ -2,9 +2,8 @@
 import logging
 import datetime
 
-# external imports
+# third-party imports
 from cic_cache.db.models.base import SessionBase
-from sqlalchemy import text
 
 logg = logging.getLogger()
 
@@ -51,8 +50,7 @@ def list_transactions_account_mined(
 
 
 def add_transaction(
-        session,
-        tx_hash,
+        session, tx_hash,
         block_number,
         tx_index,
         sender,
@@ -64,33 +62,6 @@ def add_transaction(
         success,
         timestamp,
         ):
-    """Adds a single transaction to the cache persistent storage. Sensible interpretation of all fields is the responsibility of the caller.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param block_number: Block number
-    :type block_number: int
-    :param tx_index: Transaction index in block
-    :type tx_index: int
-    :param sender: Ethereum address of effective sender
-    :type sender: str, 0x-hex
-    :param receiver: Ethereum address of effective recipient
-    :type receiver: str, 0x-hex
-    :param source_token: Ethereum address of token used by sender
-    :type source_token: str, 0x-hex
-    :param destination_token: Ethereum address of token received by recipient
-    :type destination_token: str, 0x-hex
-    :param from_value: Source token value spent in transaction
-    :type from_value: int
-    :param to_value: Destination token value received in transaction
-    :type to_value: int
-    :param success: True if code execution on network was successful
-    :type success: bool
-    :param date_block: Block timestamp
-    :type date_block: datetime
-    """
     date_block = datetime.datetime.fromtimestamp(timestamp)
     s = "INSERT INTO tx (tx_hash, block_number, tx_index, sender, recipient, source_token, destination_token, from_value, to_value, success, date_block) VALUES ('{}', {}, {}, '{}', '{}', '{}', '{}', {}, {}, {}, '{}')".format(
             tx_hash,
@@ -106,74 +77,3 @@ def add_transaction(
             date_block,
             )
     session.execute(s)
-
-
-
-def tag_transaction(
-        session,
-        tx_hash,
-        name,
-        domain=None,
-        ):
-    """Tag a single transaction with a single tag.
-
-    Tag must already exist in storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises ValueError: Unknown tag or transaction hash
-
-    """
-
-    s = text("SELECT id from tx where tx_hash = :a")
-    r = session.execute(s, {'a': tx_hash}).fetchall()
-    tx_id = r[0].values()[0]
-
-    if tx_id == None:
-        raise ValueError('unknown tx hash {}'.format(tx_hash))
-
-    #s = text("SELECT id from tag where value = :a and domain = :b")
-    if domain == None:
-        s = text("SELECT id from tag where value = :a")
-    else:
-        s = text("SELECT id from tag where value = :a and domain = :b")
-    r = session.execute(s, {'a': name, 'b': domain}).fetchall()
-    tag_id = r[0].values()[0]
-
-    logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
-
-    if tag_id == None:
-        raise ValueError('unknown tag name {} domain {}'.format(name, domain))
-
-    s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
-    r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
-
-
-def add_tag(
-        session,
-        name,
-        domain=None,
-        ):
-    """Add a single tag to storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises sqlalchemy.exc.IntegrityError: Tag already exists
-    """
-
-    s = None
-    if domain == None: 
-        s = text("INSERT INTO tag (value) VALUES (:b)")
-    else:
-        s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
-    session.execute(s, {'a': domain, 'b': name})

apps/cic-cache/cic_cache/db/migrations/default/versions/aaf2bdce7d6e_transaction_tags.py

@@ -1,38 +0,0 @@
-"""Transaction tags
-
-Revision ID: aaf2bdce7d6e
-Revises: 6604de4203e2
-Create Date: 2021-05-01 09:20:20.775082
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'aaf2bdce7d6e'
-down_revision = '6604de4203e2'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'tag',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('domain', sa.String(), nullable=True),
-            sa.Column('value', sa.String(), nullable=False),
-            )
-    op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
-
-    op.create_table(
-            'tag_tx_link',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
-            sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
-                )
-
-def downgrade():
-    op.drop_table('tag_tx_link')
-    op.drop_index('idx_tag_domain_value')
-    op.drop_table('tag')

apps/cic-cache/cic_cache/runnable/daemons/filters/base.py

@@ -1,27 +1,2 @@
-class TagSyncFilter:
-    """Holds tag name and domain for an implementing filter.
-
-    :param name: Tag value 
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    """
-
-    def __init__(self, name, domain=None):
-        self.tag_name = name
-        self.tag_domain = domain
-
-
-    def tag(self):
-        """Return tag value/domain.
-
-        :rtype: Tuple
-        :returns: tag value/domain.
-        """
-        return (self.tag_name, self.tag_domain)
-
-
-    def __str__(self):
-        if self.tag_domain == None:
-            return self.tag_name
-        return '{}.{}'.format(self.tag_domain, self.tag_name)
+class SyncFilter:
+    pass

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -2,6 +2,7 @@
 import logging
 
 # external imports
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.address import (
         to_checksum_address,
         )
@@ -12,19 +13,17 @@ from cic_eth_registry.error import (
         NotAContractError,
         ContractMismatchError,
         )
-from eth_erc20 import ERC20
 
 # local imports
-from .base import TagSyncFilter
+from .base import SyncFilter
 from cic_cache import db as cic_cache_db
 
 logg = logging.getLogger().getChild(__name__)
 
 
-class ERC20TransferFilter(TagSyncFilter):
+class ERC20TransferFilter(SyncFilter):
 
     def __init__(self, chain_spec):
-        super(ERC20TransferFilter, self).__init__('transfer', domain='erc20')
         self.chain_spec = chain_spec
 
 
@@ -47,9 +46,6 @@ class ERC20TransferFilter(TagSyncFilter):
         except RequestMismatchException:
             logg.debug('erc20 match but not a transfer, skipping')
             return False
-        except ValueError:
-            logg.debug('erc20 match but bogus data, skipping')
-            return False
 
         token_sender = tx.outputs[0]
         token_recipient = transfer_data[0]
@@ -71,13 +67,7 @@ class ERC20TransferFilter(TagSyncFilter):
                 tx.status == Status.SUCCESS,
                 block.timestamp,
                 )
-        db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
+        #db_session.flush()
         db_session.commit()
 
         return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -7,10 +7,9 @@ import argparse
 import sys
 import re
 
-# external imports
+# third-party imports
 import confini
 import celery
-import sqlalchemy
 import rlp
 import cic_base.config
 import cic_base.log
@@ -35,10 +34,7 @@ from chainsyncer.driver import (
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
-from cic_cache.db import (
-        dsn_from_config,
-        add_tag,
-        )
+from cic_cache.db import dsn_from_config
 from cic_cache.runnable.daemons.filters import (
         ERC20TransferFilter,
         )
@@ -63,17 +59,6 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
-def register_filter_tags(filters, session):
-    for f in filters:
-        tag = f.tag()
-        try:
-            add_tag(session, tag[0], domain=tag[1])
-            session.commit()
-            logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
-        except sqlalchemy.exc.IntegrityError:
-            logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
-
-
 def main():
     # Connect to blockchain with chainlib
     rpc = RPCConnection.connect(chain_spec, 'default')
@@ -113,19 +98,10 @@ def main():
 
     erc20_transfer_filter = ERC20TransferFilter(chain_spec)
 
-    filters = [
-        erc20_transfer_filter,
-            ]
-
-    session = SessionBase.create_session()
-    register_filter_tags(filters, session)
-    session.close()
-
     i = 0
     for syncer in syncers:
         logg.debug('running syncer index {}'.format(i))
-        for f in filters:
-            syncer.add_filter(f)
+        syncer.add_filter(erc20_transfer_filter)
         r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 

apps/cic-cache/config/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=0
+DEBUG=

apps/cic-cache/config/test/cic.ini

@@ -1,4 +1,2 @@
 [cic]
 registry_address = 
-chain_spec = 
-trust_address =

apps/cic-cache/config/test/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=1
+DEBUG=

apps/cic-cache/docker/Dockerfile

@@ -43,6 +43,10 @@ COPY cic-cache/config/ /usr/local/etc/cic-cache/
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
 COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
+RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
+	mkdir -p /usr/local/share/cic/solidity && \
+	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
+
 COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
 COPY cic-cache/docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh

apps/cic-cache/docker/start_tracker.sh

@@ -2,9 +2,4 @@
 
 . ./db.sh
 
-if [ $? -ne "0" ]; then
-	>&2 echo db migrate fail
-	exit 1
-fi
-
 /usr/local/bin/cic-cache-trackerd $@

apps/cic-cache/requirements.txt

@@ -1,12 +1,12 @@
-cic-base~=0.1.2b8
+cic-base~=0.1.2a77
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.4a16
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a4
+chainsyncer[sql]~=0.0.2a2

apps/cic-cache/test_requirements.txt

@@ -4,8 +4,3 @@ pytest-mock==3.3.1
 pysqlite3==0.4.3
 sqlparse==0.4.1
 pytest-celery==0.0.0a1
-eth_tester==0.5.0b3
-py-evm==0.3.0a20
-web3==5.12.2
-cic-eth-registry~=0.5.5a3
-cic-base[full]==0.1.2b8

apps/cic-cache/tests/conftest.py

@@ -3,7 +3,7 @@ import os
 import sys
 import datetime
 
-# external imports
+# third-party imports
 import pytest
 
 # local imports
@@ -84,7 +84,3 @@ def txs(
     
     session.commit()
 
-    return [
-            tx_hash_first,
-            tx_hash_second,
-            ]

apps/cic-cache/tests/filters/conftest.py

@@ -1,3 +0,0 @@
-from chainlib.eth.pytest import *
-from cic_eth_registry.pytest.fixtures_tokens import *
-

apps/cic-cache/tests/filters/test_erc20.py

@@ -1,69 +0,0 @@
-# standard imports
-import os
-import datetime
-import logging
-import json
-
-# external imports
-import pytest
-from sqlalchemy import text
-from chainlib.eth.tx import Tx
-from chainlib.eth.block import Block
-from chainlib.chain import ChainSpec
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-
-# local imports
-from cic_cache.db import add_tag
-from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
-
-logg = logging.getLogger()
-
-
-def test_cache(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    r = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    assert r
-
-    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
-    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
-    assert r[0] == tx.hash

apps/cic-cache/tests/fixtures_config.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-# external imports
+# third-party imports
 import pytest
 import confini
 
@@ -13,7 +13,7 @@ logg = logging.getLogger(__file__)
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.path.join(root_dir, 'config/test')
+    config_dir = os.path.join(root_dir, '.config/test')
     conf = confini.Config(config_dir, 'CICTEST')
     conf.process()
     logg.debug('config {}'.format(conf))

apps/cic-cache/tests/fixtures_database.py

@@ -3,16 +3,13 @@ import os
 import logging
 import re
 
-# external imports
+# third-party imports
 import pytest
 import sqlparse
-import alembic
-from alembic.config import Config as AlembicConfig
 
 # local imports
 from cic_cache.db.models.base import SessionBase
 from cic_cache.db import dsn_from_config
-from cic_cache.db import add_tag
 
 logg = logging.getLogger(__file__)
 
@@ -29,10 +26,11 @@ def database_engine(
         except FileNotFoundError:
             pass
     dsn = dsn_from_config(load_config)
-    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
+    SessionBase.connect(dsn)
     return dsn
 
 
+# TODO: use alembic instead to migrate db, here we have to keep separate schema than migration script in script/migrate.py
 @pytest.fixture(scope='function')
 def init_database(
         load_config,
@@ -40,23 +38,52 @@ def init_database(
         ):
 
     rootdir = os.path.dirname(os.path.dirname(__file__))
-    dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-    migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
-    if not os.path.isdir(migrationsdir):
-        migrationsdir = os.path.join(dbdir, 'migrations', 'default')
-    logg.info('using migrations directory {}'.format(migrationsdir))
+    schemadir = os.path.join(rootdir, 'db', load_config.get('DATABASE_DRIVER'))
+
+    if load_config.get('DATABASE_ENGINE') == 'sqlite':
+        rconn = SessionBase.engine.raw_connection()
+        f = open(os.path.join(schemadir, 'db.sql'))
+        s = f.read()
+        f.close()
+        rconn.executescript(s)
+
+    else:
+        rconn = SessionBase.engine.raw_connection()
+        rcursor = rconn.cursor()
+
+        #rcursor.execute('DROP FUNCTION IF EXISTS public.transaction_list')
+        #rcursor.execute('DROP FUNCTION IF EXISTS public.balances')
+
+        f = open(os.path.join(schemadir, 'db.sql'))
+        s = f.read()
+        f.close()
+        r = re.compile(r'^[A-Z]', re.MULTILINE)
+        for l in sqlparse.parse(s):
+            strl = str(l)
+            # we need to check for empty query lines, as sqlparse doesn't do that on its own (and psycopg complains when it gets them)
+            if not re.search(r, strl):
+                logg.warning('skipping parsed query line {}'.format(strl))
+                continue
+            rcursor.execute(strl)
+        rconn.commit()
+
+        rcursor.execute('SET search_path TO public')
+
+# this doesn't work when run separately, no idea why
+# functions have been manually added to original schema from cic-eth 
+#        f = open(os.path.join(schemadir, 'proc_transaction_list.sql'))
+#        s = f.read()
+#        f.close()
+#        rcursor.execute(s)
+# 
+#        f = open(os.path.join(schemadir, 'proc_balances.sql'))
+#        s = f.read()
+#        f.close()
+#        rcursor.execute(s)
+
+        rcursor.close()
 
     session = SessionBase.create_session()
-
-    ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini'))
-    ac.set_main_option('sqlalchemy.url', database_engine)
-    ac.set_main_option('script_location', migrationsdir)
-
-    alembic.command.downgrade(ac, 'base')
-    alembic.command.upgrade(ac, 'head')
-
-    session.commit()
-
     yield session
     session.commit()
     session.close()
@@ -89,14 +116,3 @@ def list_defaults(
     return {
         'block': 420000,
     }
-
-
-@pytest.fixture(scope='function')
-def tags(
-        init_database,
-        ):
-
-    add_tag(init_database, 'foo') 
-    add_tag(init_database, 'baz', domain='bar') 
-    add_tag(init_database, 'xyzzy', domain='bar') 
-    init_database.commit()

apps/cic-cache/tests/test_cache.py

@@ -4,7 +4,7 @@ import datetime
 import logging
 import json
 
-# external imports
+# third-party imports
 import pytest
 
 # local imports

apps/cic-cache/tests/test_tag.py

@@ -1,37 +0,0 @@
-import os
-import datetime
-import logging
-import json
-
-# external imports
-import pytest
-
-# local imports
-from cic_cache.db import tag_transaction
-
-logg = logging.getLogger()
-
-
-def test_cache(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        tags,
-        ):
-
-    tag_transaction(init_database, txs[0], 'foo')
-    tag_transaction(init_database, txs[0], 'baz', domain='bar')
-    tag_transaction(init_database, txs[1], 'xyzzy', domain='bar')
-
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.value = 'foo'").fetchall()
-    assert r[0][0] == txs[0]
-
-
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'baz'").fetchall()
-    assert r[0][0] == txs[0]
-
-    
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'xyzzy'").fetchall()
-    assert r[0][0] == txs[1]

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -2,7 +2,7 @@
 import datetime
 import logging
 
-# external imports
+# third-party imports
 import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
@@ -32,9 +32,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
     logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
@@ -53,9 +51,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, flags, address=address)
     logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
@@ -131,9 +127,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
     r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
     if address != None:
@@ -145,9 +139,3 @@ def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
     session.flush()
     session.close()
     return chained_input
-
-
-@celery_app.task()
-def shutdown(message):
-    logg.critical('shutdown called: {}'.format(message))
-    celery_app.control.shutdown() #broadcast('shutdown')

apps/cic-eth/cic_eth/admin/token.py

@@ -1,19 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-# local imports
-from cic_eth.task import BaseTask
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-            'symbol': self.default_token_symbol,
-            'address': self.default_token_address,
-            }

apps/cic-eth/cic_eth/api/api_admin.py

@@ -60,29 +60,6 @@ class AdminApi:
         self.call_address = call_address
 
 
-    def proxy_do(self, chain_spec, o):
-        s_proxy = celery.signature(
-                'cic_eth.task.rpc_proxy',
-                [
-                    chain_spec.asdict(),
-                    o,
-                    'default',
-                    ],
-                queue=self.queue
-                )
-        return s_proxy.apply_async()
-
-
-    
-    def registry(self):
-        s_registry = celery.signature(
-                'cic_eth.task.registry',
-                [],
-                queue=self.queue
-                )
-        return s_registry.apply_async()
-
-
     def unlock(self, chain_spec, address, flags=None):
         s_unlock = celery.signature(
             'cic_eth.admin.ctrl.unlock',
@@ -169,6 +146,7 @@ class AdminApi:
 
         # TODO: This check should most likely be in resend task itself
         tx_dict = s_get_tx_cache.apply_async().get()
+        #if tx_dict['status'] in [StatusEnum.REVERTED, StatusEnum.SUCCESS, StatusEnum.CANCELLED, StatusEnum.OBSOLETED]: 
         if not is_alive(getattr(StatusEnum, tx_dict['status']).value):
             raise TxStateChangeError('Cannot resend mined or obsoleted transaction'.format(txold_hash_hex))
         
@@ -248,6 +226,9 @@ class AdminApi:
                 break
             last_nonce = nonce_otx
 
+        #nonce_cache = Nonce.get(address)
+        #nonce_w3 = self.w3.eth.getTransactionCount(address, 'pending') 
+        
         return {
             'nonce': {
                 #'network': nonce_cache,
@@ -291,6 +272,20 @@ class AdminApi:
         return s_nonce.apply_async()
 
 
+#    # TODO: this is a stub, complete all checks
+#    def ready(self):
+#        """Checks whether all required initializations have been performed.
+#
+#        :raises cic_eth.error.InitializationError: At least one setting pre-requisite has not been met.
+#        :raises KeyError: An address provided for initialization is not known by the keystore.
+#        """
+#        addr = AccountRole.get_address('ETH_GAS_PROVIDER_ADDRESS')
+#        if addr == ZERO_ADDRESS:
+#            raise InitializationError('missing account ETH_GAS_PROVIDER_ADDRESS')
+#
+#        self.w3.eth.sign(addr, text='666f6f')
+
+
     def account(self, chain_spec, address, include_sender=True, include_recipient=True, renderer=None, w=sys.stdout):
         """Lists locally originated transactions for the given Ethereum address.
 
@@ -353,7 +348,6 @@ class AdminApi:
 
 
     # TODO: Add exception upon non-existent tx aswell as invalid tx data to docstring 
-    # TODO: This method is WAY too long
     def tx(self, chain_spec, tx_hash=None, tx_raw=None, registry=None, renderer=None, w=sys.stdout):
         """Output local and network details about a given transaction with local origin.
 
@@ -376,6 +370,7 @@ class AdminApi:
 
         if tx_raw != None:
             tx_hash = add_0x(keccak256_hex_to_hex(tx_raw))
+            #tx_hash = self.w3.keccak(hexstr=tx_raw).hex()
 
         s = celery.signature(
             'cic_eth.queue.query.get_tx_cache',
@@ -391,78 +386,38 @@ class AdminApi:
   
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
-            if registry != None:
-                try:
-                    source_token = registry.by_address(tx['source_token'])
-                except UnknownContractError:
-                    logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['source_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                source_token = t.get()
-                if source_token == None:
-                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
-
+            try:
+                source_token = registry.by_address(tx['source_token'])
+                #source_token = CICRegistry.get_address(chain_spec, tx['source_token']).contract
+            except UnknownContractError:
+                #source_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
+                #source_token = CICRegistry.add_token(chain_spec, source_token_contract)
+                logg.warning('unknown source token contract {}'.format(tx['source_token']))
 
         destination_token = None
-        if tx['destination_token'] != ZERO_ADDRESS:
-            if registry != None:
-                try:
-                    destination_token = registry.by_address(tx['destination_token'])
-                except UnknownContractError:
-                    logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['destination_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                destination_token = t.get()
-                if destination_token == None:
-                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
-
+        if tx['source_token'] != ZERO_ADDRESS:
+            try:
+                #destination_token = CICRegistry.get_address(chain_spec, tx['destination_token'])
+                destination_token = registry.by_address(tx['destination_token'])
+            except UnknownContractError:
+                #destination_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
+                #destination_token = CICRegistry.add_token(chain_spec, destination_token_contract)
+                logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
 
         o = code(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
-                try:
-                    sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
-                    tx['sender_description'] = 'Contract at {}'.format(tx['sender'])
-                except UnknownContractError:
-                    tx['sender_description'] = 'Unknown contract'
-                except KeyError as e:
-                    tx['sender_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['sender'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['sender_description'] = t.get()
-                if tx['sender_description'] == None:
-                    tx['sender_description'] = 'Unknown contract'
-
-
+            try:
+                #sender_contract = CICRegistry.get_address(chain_spec, tx['sender'])
+                sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
+                tx['sender_description'] = 'Contract at {}'.format(tx['sender']) #sender_contract)
+            except UnknownContractError:
+                tx['sender_description'] = 'Unknown contract'
+            except KeyError as e:
+                tx['sender_description'] = 'Unknown contract'
         else:
             s = celery.signature(
                     'cic_eth.eth.account.have',
@@ -491,31 +446,16 @@ class AdminApi:
                     tx['sender_description'] = role
 
         o = code(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
-                try:
-                    recipient_contract = registry.by_address(tx['recipient'])
-                    tx['recipient_description'] = 'Contract at {}'.format(tx['recipient'])
-                except UnknownContractError as e:
-                    tx['recipient_description'] = 'Unknown contract'
-                except KeyError as e:
-                    tx['recipient_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['recipient'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['recipient_description'] = t.get()
-                if tx['recipient_description'] == None:
-                    tx['recipient_description'] = 'Unknown contract'
-
+            try:
+                #recipient_contract = CICRegistry.by_address(tx['recipient'])
+                recipient_contract = registry.by_address(tx['recipient'])
+                tx['recipient_description'] = 'Contract at {}'.format(tx['recipient']) #recipient_contract)
+            except UnknownContractError as e:
+                tx['recipient_description'] = 'Unknown contract'
+            except KeyError as e:
+                tx['recipient_description'] = 'Unknown contract'
         else:
             s = celery.signature(
                     'cic_eth.eth.account.have',
@@ -557,8 +497,7 @@ class AdminApi:
         r = None
         try:
             o = transaction(tx_hash)
-            t = self.proxy_do(chain_spec, o)
-            r = t.get()
+            r = self.rpc.do(o)
             if r != None:
                 tx['network_status'] = 'Mempool'
         except Exception as e:
@@ -567,8 +506,7 @@ class AdminApi:
         if r != None:
             try:
                 o = receipt(tx_hash)
-                t = self.proxy_do(chain_spec, o)
-                r = t.get()
+                r = self.rpc.do(o)
                 logg.debug('h {} o {}'.format(tx_hash, o))
                 if int(strip_0x(r['status'])) == 1:
                     tx['network_status'] = 'Confirmed'
@@ -583,13 +521,11 @@ class AdminApi:
                 pass
 
         o = balance(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         tx['sender_gas_balance'] = r
 
         o = balance(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         tx['recipient_gas_balance'] = r
 
         tx_unpacked = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)

apps/cic-eth/cic_eth/api/api_task.py

@@ -62,18 +62,6 @@ class Api:
                     )       
 
 
-    def default_token(self):
-        s_token = celery.signature(
-                'cic_eth.admin.token.default_token',
-                [],
-                queue=self.queue,
-                )
-        if self.callback_param != None:
-            s_token.link(self.callback_success)
-
-        return s_token.apply_async()
-
-
     def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
         """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 

apps/cic-eth/cic_eth/check/db.py

@@ -1,8 +0,0 @@
-from cic_eth.db.models.base import SessionBase
-
-
-def health(*args, **kwargs):
-    session = SessionBase.create_session()
-    session.execute('SELECT count(*) from alembic_version')
-    session.close()
-    return True

apps/cic-eth/cic_eth/check/gas.py

@@ -1,48 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.connection import RPCConnection
-from chainlib.chain import ChainSpec
-from chainlib.eth.gas import balance
-
-# local imports
-from cic_eth.db.models.role import AccountRole
-from cic_eth.db.models.base import SessionBase
-from cic_eth.db.enum import LockEnum
-from cic_eth.error import LockedError
-from cic_eth.admin.ctrl import check_lock
-
-logg = logging.getLogger().getChild(__name__)
-
-
-def health(*args, **kwargs):
-
-    session = SessionBase.create_session()
-
-    config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-    logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
-
-    try:
-        check_lock(None, None, LockEnum.INIT)
-    except LockedError:
-        logg.warning('INIT lock is set, skipping GAS GIFTER balance check.')
-        return True
-
-    gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
-    session.close()
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    o = balance(gas_provider)
-    r = rpc.do(o)
-    try: 
-        r = int(r, 16)
-    except TypeError:
-        r = int(r)
-    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
-    if r < gas_min:
-        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
-        return False
-
-    return True

apps/cic-eth/cic_eth/check/redis.py

@@ -1,18 +0,0 @@
-# external imports
-import redis
-import os
-
-
-def health(*args, **kwargs):
-    r = redis.Redis(
-            host=kwargs['config'].get('REDIS_HOST'),
-            port=kwargs['config'].get('REDIS_PORT'),
-            db=kwargs['config'].get('REDIS_DB'),
-            )
-    try:
-        r.set(kwargs['unit'], os.getpid())
-    except redis.connection.ConnectionError:
-        return False
-    except redis.connection.ResponseError:
-        return False
-    return True

apps/cic-eth/cic_eth/check/signer.py

@@ -1,37 +0,0 @@
-# standard imports
-import time
-import logging
-from urllib.error import URLError
-
-# external imports
-from chainlib.connection import RPCConnection
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.sign import sign_message
-from chainlib.error import JSONRPCException
-
-logg = logging.getLogger().getChild(__name__)
-
-
-def health(*args, **kwargs):
-    blocked = True
-    max_attempts = 5
-    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
-    for i in range(max_attempts):
-        idx = i + 1
-        logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))
-        try:
-            conn.do(sign_message(ZERO_ADDRESS, '0x2a'))
-        except FileNotFoundError:
-            pass
-        except ConnectionError:
-            pass
-        except URLError:
-            pass
-        except JSONRPCException:
-            logg.debug('signer connection succeeded')
-            return True
-
-        if idx < max_attempts:
-            time.sleep(0.5)
-
-    return False

apps/cic-eth/cic_eth/db/enum.py

@@ -74,11 +74,10 @@ class LockEnum(enum.IntEnum):
     QUEUE: Disable queueing new or modified transactions
     """
     STICKY=1
-    INIT=2
-    CREATE=4
-    SEND=8
-    QUEUE=16
-    QUERY=32
+    CREATE=2
+    SEND=4
+    QUEUE=8
+    QUERY=16
     ALL=int(0xfffffffffffffffe)
 
 

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -5,11 +5,8 @@ Revises: 1f1b3b641d08
 Create Date: 2021-04-02 18:41:20.864265
 
 """
-import datetime
 from alembic import op
 import sqlalchemy as sa
-from chainlib.eth.constant import ZERO_ADDRESS
-from cic_eth.db.enum import LockEnum
 
 
 # revision identifiers, used by Alembic.
@@ -26,11 +23,10 @@ def upgrade():
             sa.Column("address", sa.String(42), nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
-            sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
+            sa.Column("date_created", sa.DateTime, nullable=False),
             sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
             )
     op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
-    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
 
 
 def downgrade():

apps/cic-eth/cic_eth/error.py

@@ -48,8 +48,6 @@ class RoleMissingError(Exception):
     pass
 
 
-
-
 class IntegrityError(Exception):
     """Exception raised to signal irregularities with deduplication and ordering of tasks
 
@@ -64,19 +62,15 @@ class LockedError(Exception):
     pass
 
 
-class SeppukuError(Exception):
-    """Exception base class for all errors that should cause system shutdown
-
-    """
-
-
-class SignerError(SeppukuError):
+class SignerError(Exception):
     """Exception raised when signer is unavailable or generates an error
 
     """
     pass
 
 
-class RoleAgencyError(SeppukuError):
-    """Exception raise when a role cannot perform its function. This is a critical exception
+class EthError(Exception):
+    """Exception raised when unspecified error from evm node is encountered
+
     """
+    pass

apps/cic-eth/cic_eth/eth/account.py

@@ -3,11 +3,11 @@ import logging
 
 # external imports
 import celery
-from erc20_faucet import Faucet
+from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import (
         strip_0x,
         )
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.connection import RPCConnection
 from chainlib.eth.sign import (
         new_account,
@@ -19,9 +19,8 @@ from chainlib.eth.tx import (
         unpack,
         )
 from chainlib.chain import ChainSpec
-from chainlib.error import JSONRPCException
-from eth_accounts_index.registry import AccountRegistry # TODO, use interface module instead (needs gas limit method)
-from sarafu_faucet import MinterFaucet
+from eth_accounts_index import AccountRegistry
+from sarafu_faucet import MinterFaucet as Faucet
 from chainqueue.db.models.tx import TxCache
 
 # local import
@@ -71,18 +70,11 @@ def create(self, password, chain_spec_dict):
     a = None
     conn = RPCConnection.connect(chain_spec, 'signer')
     o = new_account()
-    try:
-        a = conn.do(o)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
+    a = conn.do(o)
     conn.disconnect()
 
-    # TODO: It seems infeasible that a can be None in any case, verify
     if a == None:
         raise SignerError('create account')
-        
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -127,13 +119,13 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     if writer_address == ZERO_ADDRESS:
         session.close()
         raise RoleMissingError('call address for resgistering {}'.format(account_address))
-    account_registry_address = registry.by_name('AccountsIndex', sender_address=call_address)
+    account_registry_address = registry.by_name('AccountRegistry', sender_address=call_address)
    
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, AccountsIndex.gas)
-    account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
+    account_registry = AccountRegistry(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
 
@@ -185,7 +177,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -227,22 +219,21 @@ def have(self, account, chain_spec_dict):
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     o = sign_message(account, '0x2a')
-    conn = RPCConnection.connect(chain_spec, 'signer')
+    try:
+        conn = RPCConnection.connect(chain_spec, 'signer')
+    except Exception as e:
+        logg.debug('cannot sign with {}: {}'.format(account, e))
+        return None
 
     try:
         conn.do(o)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except JSONRPCException as e:
+        conn.disconnect()
+        return account
+    except Exception as e:
         logg.debug('cannot sign with {}: {}'.format(account, e))
         conn.disconnect()
         return None
    
-    conn.disconnect()
-    return account
-
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):
@@ -338,7 +329,7 @@ def cache_account_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
     tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = AccountsIndex.parse_add_request(tx['data'])
+    tx_data = AccountRegistry.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_cache = TxCache(

apps/cic-eth/cic_eth/eth/erc20.py

@@ -6,6 +6,7 @@ import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -15,7 +16,6 @@ from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
 from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
-from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -108,13 +108,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
-    
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
 
     rpc_signer.disconnect()
     rpc.disconnect()
@@ -177,12 +171,7 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
+    (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
 
     rpc_signer.disconnect()
     rpc.disconnect()

apps/cic-eth/cic_eth/eth/gas.py

@@ -328,12 +328,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
 
     # build and add transaction
     logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address))
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
     logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex))
     cache_task = 'cic_eth.eth.gas.cache_gas_data'
     register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
@@ -409,12 +404,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
     logg.debug('change gas price from old {} to new {} for tx {}'.format(tx['gasPrice'], new_gas_price, tx))
     tx['gasPrice'] = new_gas_price
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
+    (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
     queue_create(
         chain_spec,
         tx['nonce'],

apps/cic-eth/cic_eth/ext/address.py

@@ -7,7 +7,7 @@ from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth_registry import CICRegistry
-from eth_address_declarator import Declarator
+from eth_address_declarator import AddressDeclarator
 
 # local imports
 from cic_eth.task import BaseTask
@@ -23,12 +23,12 @@ def translate_address(address, trusted_addresses, chain_spec, sender_address=ZER
     registry = CICRegistry(chain_spec, rpc)
     
     declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    c = Declarator(chain_spec)
+    c = AddressDeclarator(chain_spec)
 
     for trusted_address in trusted_addresses:
         o = c.declaration(declarator_address, trusted_address, address, sender_address=sender_address)
         r = rpc.do(o)
-        declaration_hex = Declarator.parse_declaration(r)
+        declaration_hex = AddressDeclarator.parse_declaration(r)
         declaration_hex = declaration_hex[0].rstrip('0')
         declaration_bytes = bytes.fromhex(declaration_hex)
         declaration = None

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,13 +14,13 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
+from chainlib.eth.erc20 import ERC20
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.query import get_tx_cache
-from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.queue.time import tx_times
@@ -114,7 +114,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
-                        token = ERC20Token(chain_spec, rpc, tx['to'])
+                        token = ERC20Token(rpc, tx['to'])
                         token_symbol = token.symbol
                         token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)

apps/cic-eth/cic_eth/queue/time.py

@@ -12,7 +12,6 @@ from chainqueue.error import NotLocalTxError
 
 # local imports
 from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
-from cic_eth.db.models.base import SessionBase
 
 celery_app = celery.current_app
 

apps/cic-eth/cic_eth/registry.py

@@ -29,5 +29,5 @@ def connect(rpc, chain_spec, registry_address):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
     registry_address = registry.by_name('ContractRegistry')
+    
     return registry
-

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -23,6 +23,7 @@ default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-r', '--registry-address', type=str, help='CIC registry address')
 argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
@@ -58,7 +59,6 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         }
 # override args
-config.dict_override(args_override, 'cli')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@@ -67,9 +67,7 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
 
 queue = args.q
 
-chain_spec = None
-if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 admin_api = AdminApi(None)
 
 
@@ -84,9 +82,6 @@ def lock_names_to_flag(s):
 
 # TODO: move each command to submodule
 def main():
-    chain_spec_dict = None
-    if chain_spec != None:
-        chain_spec_dict = chain_spec.asdict()
     if args.command == 'unlock':
         flags = lock_names_to_flag(args.flags)
         if not is_checksum_address(args.address):
@@ -96,7 +91,7 @@ def main():
             'cic_eth.admin.ctrl.unlock',
             [
                 None,
-                chain_spec_dict,
+                chain_spec.asdict(),
                 args.address,
                 flags,
                 ],
@@ -115,7 +110,7 @@ def main():
             'cic_eth.admin.ctrl.lock',
             [
                 None,
-                chain_spec_dict,
+                chain_spec.asdict(),
                 args.address,
                 flags,
                 ],

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -13,7 +13,9 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-from erc20_faucet import Faucet
+# TODO: use sarafu_Faucet for both when inheritance has been implemented
+from erc20_single_shot_faucet import SingleShotFaucet
+from sarafu_faucet import MinterFaucet as Faucet
 
 # local imports
 from .base import SyncFilter
@@ -69,13 +71,14 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        o = Faucet.token(faucet_contract, sender_address=self.caller_address)
+        c = SingleShotFaucet(self.chain_spec)
+        o = c.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 
-        o = c.token_amount(faucet_contract, sender_address=self.caller_address)
+        o = c.amount(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
-        transfer_data['value'] = c.parse_token_amount(r)
+        transfer_data['value'] = c.parse_amount(r)
 
         return ('tokengift', transfer_data)
 

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -11,19 +11,10 @@ import websocket
 # external imports
 import celery
 import confini
-from chainlib.connection import (
-        RPCConnection,
-        ConnType,
-        )
-from chainlib.eth.connection import (
-        EthUnixSignerConnection,
-        EthHTTPSignerConnection,
-        )
+from chainlib.connection import RPCConnection
+from chainlib.eth.connection import EthUnixSignerConnection
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
-from cic_eth_registry.error import UnknownContractError
-import liveness.linux
-
 
 # local imports
 from cic_eth.eth import (
@@ -36,7 +27,6 @@ from cic_eth.eth import (
 from cic_eth.admin import (
         debug,
         ctrl,
-        token
         )
 from cic_eth.queue import (
         query,
@@ -61,8 +51,6 @@ from cic_eth.registry import (
         connect_declarator,
         connect_token_registry,
         )
-from cic_eth.task import BaseTask
-
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -74,7 +62,6 @@ argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provide
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
 argparser.add_argument('-r', type=str, help='CIC registry address')
-argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
@@ -94,7 +81,6 @@ config.process()
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
@@ -104,15 +90,14 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-health_modules = config.get('CIC_HEALTH_MODULES', [])
-if len(health_modules) != 0:
-    health_modules = health_modules.split(',')
-logg.debug('health mods {}'.format(health_modules))
-
 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
 
+# verify database connection with minimal sanity query
+session = SessionBase.create_session()
+session.execute('select version_num from alembic_version')
+session.close()
 
 # set up celery
 current_app = celery.Celery(__name__)
@@ -149,18 +134,11 @@ else:
         })
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
 RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
-RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
+RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
 
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 
-#import cic_eth.checks.gas
-#if not cic_eth.checks.gas.health(config=config):
-#    raise RuntimeError()
-liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker')
 
 def main():
     argv = ['worker']
@@ -184,11 +162,7 @@ def main():
 
     rpc = RPCConnection.connect(chain_spec, 'default')
 
-    try:
-        registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-    except UnknownContractError as e:
-        logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-        sys.exit(1)
+    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
@@ -197,18 +171,10 @@ def main():
     trusted_addresses = trusted_addresses_src.split(',')
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
-
     connect_declarator(rpc, chain_spec, trusted_addresses)
     connect_token_registry(rpc, chain_spec)
-
-    BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-    BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    BaseTask.run_dir = config.get('CIC_RUN_DIR')
-    logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
-   
-    liveness.linux.set(rundir=config.get('CIC_RUN_DIR'))
+    
     current_app.worker_main(argv)
-    liveness.linux.reset(rundir=config.get('CIC_RUN_DIR'))
 
 
 @celery.signals.eventlet_pool_postshutdown.connect

apps/cic-eth/cic_eth/runnable/info.py

@@ -1,65 +0,0 @@
-#!python3
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import logging
-import argparse
-import os
-
-# external imports
-import confini
-import celery
-
-# local imports
-from cic_eth.api import Api
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-default_format = 'terminal'
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-args = argparser.parse_args()
-
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.dict_override(args_override, 'cli args')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-queue = args.q
-
-api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
-
-def main():
-    t = api.default_token()
-    token_info = t.get()
-    print('Default token symbol: {}'.format(token_info['symbol']))
-    print('Default token address: {}'.format(token_info['address']))
-
-
-if __name__ == '__main__':
-    main()

apps/cic-eth/cic_eth/runnable/transfer.py

@@ -85,6 +85,9 @@ def main():
             callback_queue=args.q,
             )
 
+    #register = not args.no_register
+    #logg.debug('register {}'.format(register))
+    #t = api.create_account(register=register)
     t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL'))
 
     ps.get_message()

apps/cic-eth/cic_eth/runnable/view.py

@@ -81,14 +81,10 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 rpc = EthHTTPConnection(args.p)
 
-#registry_address = config.get('CIC_REGISTRY_ADDRESS')
+registry_address = config.get('CIC_REGISTRY_ADDRESS')
 
 admin_api = AdminApi(rpc)
 
-t = admin_api.registry()
-registry_address = t.get()
-logg.info('got registry address from task pool: {}'.format(registry_address))
-
 trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
 if trusted_addresses_src == None:
     logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
@@ -155,16 +151,14 @@ def main():
     txs  = []
     renderer = render_tx
     if len(config.get('_QUERY')) > 66:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
+        registry = connect_registry(rpc, chain_spec, registry_address)
+        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
     elif len(config.get('_QUERY')) > 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
+        registry = connect_registry(rpc, chain_spec, registry_address)
+        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
 
     elif len(config.get('_QUERY')) == 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
+        registry = connect_registry(rpc, chain_spec, registry_address)
         txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
         renderer = render_account
     elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':

apps/cic-eth/cic_eth/task.py

@@ -7,20 +7,18 @@ import uuid
 # external imports
 import celery
 import sqlalchemy
-from chainlib.chain import ChainSpec
-from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.gas import RPCGasOracle
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.error import UnknownContractError
-import liveness.linux
 
 # local imports
-from cic_eth.error import SeppukuError
+from cic_eth.error import (
+        SignerError,
+        EthError,
+        )
 from cic_eth.db.models.base import SessionBase
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 celery_app = celery.current_app
 
@@ -31,9 +29,6 @@ class BaseTask(celery.Task):
     call_address = ZERO_ADDRESS
     create_nonce_oracle = RPCNonceOracle
     create_gas_oracle = RPCGasOracle
-    default_token_address = None
-    default_token_symbol = None
-    run_dir = '/run'
 
     def create_session(self):
         return BaseTask.session_func()
@@ -43,19 +38,6 @@ class BaseTask(celery.Task):
         logg.debug('task {} root uuid {}'.format(self.__class__.__name__, self.request.root_id))
         return
 
-
-    def on_failure(self, exc, task_id, args, kwargs, einfo):
-        if isinstance(exc, SeppukuError):
-            liveness.linux.reset(rundir=self.run_dir)
-            logg.critical(einfo)
-            msg = 'received critical exception {}, calling shutdown'.format(str(exc))
-            s = celery.signature(
-                'cic_eth.admin.ctrl.shutdown',
-                [msg],
-                queue=self.request.delivery_info.get('routing_key'),
-                    )
-            s.apply_async()
-
     
 class CriticalTask(BaseTask):
     retry_jitter = True
@@ -85,6 +67,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
         sqlalchemy.exc.TimeoutError,
         requests.exceptions.ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
+        EthError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
@@ -95,45 +78,19 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         sqlalchemy.exc.ResourceClosedError,
+        SignerError,
         ) 
 
 class CriticalWeb3AndSignerTask(CriticalTask):
     autoretry_for = (
         requests.exceptions.ConnectionError,
+        SignerError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
 
 
-@celery_app.task()
-def check_health(self):
-    pass
-
-
-# TODO: registry / rpc methods should perhaps be moved to better named module
-@celery_app.task()
-def registry():
-    return CICRegistry.address
-
-
-@celery_app.task()
-def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    registry = CICRegistry(chain_spec, conn)
-    return registry.by_address(address)
-
-
-@celery_app.task(throws=(UnknownContractError,))
-def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name)
-
-
-@celery_app.task()
-def rpc_proxy(chain_spec_dict, o, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    return conn.do(o)
+@celery_app.task(bind=True, base=BaseTask)
+def hello(self):
+    time.sleep(0.1)
+    return id(SessionBase.create_session)

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.12',
+        'beta.6',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/cic.ini

@@ -3,6 +3,3 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address =
-default_token_symbol = GFT
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
-run_dir = /run

apps/cic-eth/config/docker/cic.ini

@@ -3,6 +3,3 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
 tx_retry_delay = 20
-default_token_symbol = GFT
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
-run_dir = /run

apps/cic-eth/config/docker/eth.ini

@@ -1,3 +1,8 @@
 [eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_provider_address = 
+#chain_id = 
+abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis
+account_accounts_index_writer = 

apps/cic-eth/config/docker/signer.ini

@@ -1,5 +1,5 @@
 [signer]
-socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
+socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc
 secret = deedbeef
 database_name = signer_test
 dev_keys_path = 

apps/cic-eth/config/eth.ini

@@ -1,3 +1,8 @@
 [eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
 provider = http://localhost:8545
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_provider_address = 
+#chain_id = 
+abi_dir = /usr/local/share/cic/solidity/abi
+account_accounts_index_writer = 

apps/cic-eth/docker/Dockerfile

@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b8
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./
@@ -50,4 +50,6 @@ COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-COPY util/liveness/health.sh /usr/local/bin/health.sh
+RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
+	mkdir -p /usr/local/share/cic/solidity && \
+	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi

apps/cic-eth/requirements.txt

@@ -1,24 +1,25 @@
-cic-base==0.1.2b8
+cic-base~=0.1.2a76
 celery==4.4.7
-crypto-dev-signer~=0.4.14b3
+crypto-dev-signer~=0.4.14b2
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.4a16
+#cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a11
-erc20-transfer-authorization~=0.3.1a6
+eth_accounts_index~=0.0.11a9
+erc20-transfer-authorization~=0.3.1a5
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a11
-chainlib~=0.0.3a1
+eth-address-index~=0.1.1a9
+chainlib~=0.0.2a13
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a4
-chainqueue~=0.0.2a2
-sarafu-faucet==0.0.3a1
+chainsyncer[sql]~=0.0.2a2
+chainqueue~=0.0.1a7
+pysha3==1.0.2
 coincurve==15.0.0
-sarafu-faucet==0.0.3a2
+sarafu-faucet==0.0.2a28
 potaahto~=0.0.1a1

apps/cic-eth/setup.cfg

@@ -38,7 +38,6 @@ packages =
 	cic_eth.runnable.daemons.filters
 	cic_eth.callbacks
 	cic_eth.sync
-	cic_eth.check
 scripts =
 	./scripts/migrate.py
 
@@ -53,7 +52,6 @@ console_scripts =
 	cic-eth-create = cic_eth.runnable.create:main 
 	cic-eth-inspect = cic_eth.runnable.view:main
 	cic-eth-ctl = cic_eth.runnable.ctrl:main
-	cic-eth-info = cic_eth.runnable.info:main
 	# TODO: Merge this with ctl when subcmds sorted to submodules
 	cic-eth-tag = cic_eth.runnable.tag:main
 	cic-eth-resend = cic_eth.runnable.resend:main

apps/cic-eth/tests/conftest.py

@@ -3,12 +3,8 @@ import os
 import sys
 import logging
 
-# external imports
-from chainlib.eth.erc20 import ERC20
-
 # local imports
 from cic_eth.api import Api
-from cic_eth.task import BaseTask
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -32,26 +28,3 @@ def api(
     ):
     chain_str = str(default_chain_spec)
     return Api(chain_str, queue=None, callback_param='foo')
-
-
-@pytest.fixture(scope='function')
-def foo_token_symbol(
-    default_chain_spec,
-    foo_token,
-    eth_rpc,
-    contract_roles,
-    ):
-    
-    c = ERC20(default_chain_spec)
-    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)
-
-
-@pytest.fixture(scope='function')
-def default_token(
-        foo_token,
-        foo_token_symbol,
-        ):
-    BaseTask.default_token_symbol = foo_token_symbol
-    BaseTask.default_token_address = foo_token   

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -210,11 +210,9 @@ def test_callback_filter(
 
         def __init__(self):
             self.results = {}
-            self.queue = 'test'
 
         def call_back(self, transfer_type, result):
             self.results[transfer_type] = result
-            return self
 
     mock = CallbackMock()
     fltr.call_back = mock.call_back

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -65,7 +65,6 @@ def test_tx(
     tx_hash_hex_orig = tx_hash_hex
 
     gas_oracle = OverrideGasOracle(price=1100000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
     queue_create(
             default_chain_spec,

apps/cic-eth/tests/fixtures_celery.py

@@ -34,7 +34,6 @@ def celery_includes():
         'cic_eth.admin.ctrl',
         'cic_eth.admin.nonce',
         'cic_eth.admin.debug',
-        'cic_eth.admin.token',
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',

apps/cic-eth/tests/fixtures_database.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-# external imports
+# third-party imports
 import pytest
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -53,9 +53,6 @@ def init_database(
     alembic.command.downgrade(ac, 'base')
     alembic.command.upgrade(ac, 'head')
 
-    session.execute('DELETE FROM lock')
-    session.commit()
-
     yield session
     session.commit()
     session.close()

apps/cic-eth/tests/task/api/test_admin.py

@@ -273,7 +273,7 @@ def test_tx(
     eth_signer,
     agent_roles,
     contract_roles,
-    celery_session_worker,
+    celery_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)

apps/cic-eth/tests/task/api/test_list.py

@@ -35,7 +35,7 @@ def test_list_tx(
         foo_token,
         register_tokens,
         init_eth_tester,
-        celery_session_worker,
+        celery_worker,
         ):
 
     tx_hashes = []

apps/cic-eth/tests/unit/admin/test_default_token.py

@@ -1,21 +0,0 @@
-# external imports
-import celery
-
-
-def test_default_token(
-        default_token,
-        celery_session_worker,
-        foo_token,
-        foo_token_symbol,
-        ):
-      
-    s = celery.signature(
-            'cic_eth.admin.token.default_token',
-            [],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get()
-
-    assert r['address'] == foo_token
-    assert r['symbol'] == foo_token_symbol

apps/cic-meta/.config/pgp.ini

@@ -1,5 +1,5 @@
 [pgp]
-exports_dir = /root/pgp
+exports_dir = pgp
 privatekey_file = privatekeys.asc
 passphrase = merman
 publickey_trusted_file = publickeys.asc

apps/cic-meta/docker/Dockerfile

@@ -2,21 +2,17 @@ FROM node:15.3.0-alpine3.10
 
 WORKDIR /tmp/src/cic-meta
 
-RUN apk add --no-cache postgresql bash
-
 COPY cic-meta/package.json \
      ./
 
 COPY cic-meta/src/ src/
 COPY cic-meta/tests/ tests/
 COPY cic-meta/scripts/ scripts/
+#COPY docker/*.sh /root/
 
-RUN npm install
-
-# see exports_dir gpg.ini
-COPY cic-meta/tests/*.asc /root/pgp/
 RUN alias tsc=node_modules/typescript/bin/tsc 
 
+RUN npm install
 
 COPY cic-meta/.config/ /usr/local/etc/cic-meta/
 # COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql
@@ -24,9 +20,8 @@ COPY cic-meta/.config/ /usr/local/etc/cic-meta/
 COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
 
-#RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
-#ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
+RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
+ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
 
-COPY cic-meta/docker/start_server.sh ./start_server.sh
-RUN chmod 755 ./start_server.sh
-ENTRYPOINT ["sh", "./start_server.sh"]
+# COPY cic-meta/docker/start_server.sh ./start_server.sh
+# RUN chmod 755 ./start_server.sh

apps/cic-meta/docker/db.sh

@@ -1,6 +1,3 @@
 #!/bin/bash
-set -e 
-
-PGPASSWORD=$DATABASE_PASSWORD psql -v ON_ERROR_STOP=1 -U $DATABASE_USER -h $DATABASE_HOST -p $DATABASE_PORT -d $DATABASE_NAME -f $SCHEMA_SQL_PATH 
-
 
+PGPASSWORD=$DATABASE_PASSWORD psql -U $DATABASE_USER -h $DATABASE_HOST -p $DATABASE_PORT -d $DATABASE_NAME /usr/local/share/cic-meta/sql/server.sql

apps/cic-meta/docker/start_server.sh

@@ -1,9 +1,3 @@
-#!/bin/bash
-set -euo pipefail
-
-# db migration
 sh ./db.sh
 
-# /usr/local/bin/node /usr/local/bin/cic-meta-server $@
-# ./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts $@
-npm run start "$@"
+/usr/local/bin/node /usr/local/bin/cic-meta-server $@

apps/cic-meta/package.json

@@ -10,8 +10,7 @@
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
 		"clean": "rm -rf dist",
-		"prepare": "npm run build && npm run build-server",
-                "start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts" 
+		"prepare": "npm run build && npm run build-server"
 	},
 	"dependencies": {
 		"@ethereumjs/tx": "^3.0.0-beta.1",

apps/cic-meta/scripts/initdb/postgresql.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username grassroots --dbname cic_meta <<-EOSQL
+    create table if not exists store (
+        id serial primary key not null,
+        owner_fingerprint text not null,
+        hash char(64) not null unique,
+        content text not null
+    );
+
+    create index if not exists idx_fp on store ((lower(owner_fingerprint)));
+EOSQL
+
+

apps/cic-meta/scripts/initdb/server.postgres.sql

@@ -1,4 +1,4 @@
-create table if not exists store (
+create table if not exists cic_meta.store (
 	id serial primary key not null,
 	owner_fingerprint text not null,
 	hash char(64) not null unique,

apps/cic-ussd/cic_ussd/account.py

@@ -20,7 +20,7 @@ def define_account_tx_metadata(user: Account):
     )
     key = generate_metadata_pointer(
         identifier=identifier,
-        cic_type=':cic.person'
+        cic_type='cic.person'
     )
     account_metadata = get_cached_data(key=key)
 

apps/cic-ussd/cic_ussd/balance.py

@@ -80,7 +80,7 @@ def get_cached_operational_balance(blockchain_address: str):
     """
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),
-        salt=':cic.balances_data'
+        salt='cic.balances_data'
     )
     cached_balance = get_cached_data(key=key)
     if cached_balance:

apps/cic-ussd/cic_ussd/error.py

@@ -38,13 +38,3 @@ class MetadataStoreError(Exception):
     pass
 
 
-class SeppukuError(Exception):
-    """Exception base class for all errors that should cause system shutdown"""
-    pass
-
-
-class InitializationError(Exception):
-    """Exception raised when initialization state is insufficient to run component"""
-    pass
-
-

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -118,7 +118,7 @@ class MetadataRequestsHandler(Metadata):
         metadata_http_error_handler(result=result)
         response_data = result.content
         data = json.loads(response_data.decode('utf-8'))
-        if result.status_code == 200 and self.cic_type == ':cic.person':
+        if result.status_code == 200 and self.cic_type == 'cic.person':
             person = Person()
             deserialized_person = person.deserialize(person_data=json.loads(data))
             data = json.dumps(deserialized_person.serialize())

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -9,4 +9,4 @@ from .base import MetadataRequestsHandler
 class PersonMetadata(MetadataRequestsHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.person', identifier=identifier)
+        super().__init__(cic_type='cic.person', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -10,4 +10,4 @@ from .base import MetadataRequestsHandler
 class PhonePointerMetadata(MetadataRequestsHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.phone', identifier=identifier)
+        super().__init__(cic_type='cic.msisdn', identifier=identifier)

apps/cic-ussd/cic_ussd/operations.py

@@ -48,9 +48,10 @@ def define_response_with_content(headers: list, response: str) -> tuple:
     content_length_header = ('Content-Length', str(content_length))
     # check for content length defaulted to zero in error headers
     for position, header in enumerate(headers):
-        if 'Content-Length' in header:
-            headers.pop(position)
-    headers.append(content_length_header)
+        if header[0] == 'Content-Length':
+            headers[position] = content_length_header
+        else:
+            headers.append(content_length_header)
     return response_bytes, headers
 
 

apps/cic-ussd/cic_ussd/processor.py

@@ -7,7 +7,6 @@ from typing import Optional
 # third party imports
 import celery
 from sqlalchemy import desc
-from cic_eth.api import Api
 from tinydb.table import Document
 
 # local imports
@@ -16,7 +15,7 @@ from cic_ussd.balance import BalanceManager, compute_operational_balance, get_ca
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.db.models.ussd_session import UssdSession
-from cic_ussd.error import MetadataNotFoundError, SeppukuError
+from cic_ussd.error import MetadataNotFoundError
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.phone_number import get_user_by_phone_number
@@ -29,38 +28,6 @@ from cic_types.models.person import generate_metadata_pointer, get_contact_data_
 logg = logging.getLogger(__name__)
 
 
-def get_default_token_data():
-    chain_str = Chain.spec.__str__()
-    cic_eth_api = Api(chain_str=chain_str)
-    default_token_request_task = cic_eth_api.default_token()
-    default_token_data = default_token_request_task.get()
-    return default_token_data
-
-
-def retrieve_token_symbol(chain_str: str = Chain.spec.__str__()):
-    """
-    :param chain_str:
-    :type chain_str:
-    :return:
-    :rtype:
-    """
-    cache_key = create_cached_data_key(
-        identifier=chain_str.encode('utf-8'),
-        salt=':cic.default_token_data'
-    )
-    cached_data = get_cached_data(key=cache_key)
-    if cached_data:
-        default_token_data = json.loads(cached_data)
-        return default_token_data.get('symbol')
-    else:
-        logg.warning('Cached default token data not found. Attempting retrieval from default token API')
-        default_token_data = get_default_token_data()
-        if default_token_data:
-            return default_token_data.get('symbol')
-        else:
-            raise SeppukuError(f'Could not retrieve default token for: {chain_str}')
-
-
 def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
     """
     This method provides translation for all ussd menu entries that follow the pin authorization pattern.
@@ -106,9 +73,7 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     # compile response data
     user_input = ussd_session.get('user_input').split('*')[-1]
     transaction_amount = to_wei(value=int(user_input))
-
-    # get default data
-    token_symbol = retrieve_token_symbol()
+    token_symbol = 'SRF'
 
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
@@ -137,7 +102,7 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     :rtype: str
     """
     transaction_amount = to_wei(int(ussd_session.get('session_data').get('transaction_amount')))
-    token_symbol = retrieve_token_symbol()
+    token_symbol = 'SRF'
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
     tx_recipient_information = define_account_tx_metadata(user=recipient)
@@ -172,7 +137,7 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     tx_recipient_information = define_account_tx_metadata(user=recipient)
     tx_sender_information = define_account_tx_metadata(user=user)
 
-    token_symbol = retrieve_token_symbol()
+    token_symbol = 'SRF'
     user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
     logg.debug('Requires integration to determine user tokens.')
@@ -203,18 +168,18 @@ def process_account_balances(user: Account, display_key: str, ussd_session: dict
     logg.debug('Requires call to retrieve tax and bonus amounts')
     tax = ''
     bonus = ''
-    token_symbol = retrieve_token_symbol()
+
     return translation_for(
         key=display_key,
         preferred_language=user.preferred_language,
         operational_balance=operational_balance,
         tax=tax,
         bonus=bonus,
-        token_symbol=token_symbol
+        token_symbol='SRF'
     )
 
 
-def format_transactions(transactions: list, preferred_language: str, token_symbol: str):
+def format_transactions(transactions: list, preferred_language: str):
     
     formatted_transactions = ''
     if len(transactions) > 0:
@@ -225,7 +190,7 @@ def format_transactions(transactions: list, preferred_language: str, token_symbo
             timestamp = transaction.get('timestamp')
             action_tag = transaction.get('action_tag')
             direction = transaction.get('direction')
-            token_symbol = token_symbol
+            token_symbol = 'SRF'
 
             if action_tag == 'SENT' or action_tag == 'ULITUMA':
                 formatted_transactions += f'{action_tag} {value} {token_symbol} {direction} {recipient_phone_number} {timestamp}.\n'
@@ -249,7 +214,7 @@ def process_display_user_metadata(user: Account, display_key: str):
     """
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
+        cic_type='cic.person'
     )
     user_metadata = get_cached_data(key)
     if user_metadata:
@@ -286,11 +251,9 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
     """
     # retrieve cached statement
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address)
-    key = create_cached_data_key(identifier=identifier, salt=':cic.statement')
+    key = create_cached_data_key(identifier=identifier, salt='cic.statement')
     transactions = get_cached_data(key=key)
 
-    token_symbol = retrieve_token_symbol()
-
     first_transaction_set = []
     middle_transaction_set = []
     last_transaction_set = []
@@ -314,8 +277,7 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             first_transaction_set=format_transactions(
                 transactions=first_transaction_set,
-                preferred_language=user.preferred_language,
-                token_symbol=token_symbol
+                preferred_language=user.preferred_language
             )
         )
     elif display_key == 'ussd.kenya.middle_transaction_set':
@@ -324,8 +286,7 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             middle_transaction_set=format_transactions(
                 transactions=middle_transaction_set,
-                preferred_language=user.preferred_language,
-                token_symbol=token_symbol
+                preferred_language=user.preferred_language
             )
         )
 
@@ -335,8 +296,7 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             last_transaction_set=format_transactions(
                 transactions=last_transaction_set,
-                preferred_language=user.preferred_language,
-                token_symbol=token_symbol
+                preferred_language=user.preferred_language
             )
         )
 
@@ -352,19 +312,18 @@ def process_start_menu(display_key: str, user: Account):
     :return: Corresponding translation text response
     :rtype: str
     """
-    token_symbol = retrieve_token_symbol()
     chain_str = Chain.spec.__str__()
     blockchain_address = user.blockchain_address
     balance_manager = BalanceManager(address=blockchain_address,
                                      chain_str=chain_str,
-                                     token_symbol=token_symbol)
+                                     token_symbol='SRF')
 
     # get balances synchronously for display on start menu
     balances_data = balance_manager.get_balances()
 
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),
-        salt=':cic.balances_data'
+        salt='cic.balances_data'
     )
     cache_data(key=key, data=json.dumps(balances_data))
 
@@ -381,6 +340,9 @@ def process_start_menu(display_key: str, user: Account):
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 
+    # TODO [Philip]: figure out how to get token symbol from a metadata layer of sorts.
+    token_symbol = 'SRF'
+
     return translation_for(
         key=display_key,
         preferred_language=user.preferred_language,
@@ -413,13 +375,6 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
     :return: A ussd menu's corresponding text value.
     :rtype: Document
     """
-    # retrieve metadata before any transition
-    key = generate_metadata_pointer(
-        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
-    )
-    person_metadata = get_cached_data(key=key)
-
     if ussd_session:
         if user_input == "0":
             return UssdMenu.parent_menu(menu_name=ussd_session.get('state'))
@@ -430,6 +385,12 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
         if user.has_valid_pin():
             last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
 
+            key = generate_metadata_pointer(
+                identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
+                cic_type='cic.person'
+            )
+            person_metadata = get_cached_data(key=key)
+
             if last_ussd_session:
                 # get last state
                 last_state = last_ussd_session.state

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -1,73 +0,0 @@
-"""
-This module handles requests originating from CICADA or any other management client for custodial wallets, processing
-requests offering control of user account states to a staff behind the client.
-"""
-
-# standard imports
-import logging
-from urllib.parse import quote_plus
-
-# third-party imports
-from confini import Config
-
-# local imports
-from cic_ussd.db import dsn_from_config
-from cic_ussd.db.models.base import SessionBase
-from cic_ussd.operations import define_response_with_content
-from cic_ussd.requests import (get_request_endpoint,
-                               get_query_parameters,
-                               process_pin_reset_requests,
-                               process_locked_accounts_requests)
-from cic_ussd.runnable.server_base import exportable_parser, logg
-args = exportable_parser.parse_args()
-
-# define log levels
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-# parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
-
-# set up db
-data_source_name = dsn_from_config(config)
-SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
-# create session for the life time of http request
-SessionBase.session = SessionBase.create_session()
-
-
-# handle requests from CICADA
-def application(env, start_response):
-    """Loads python code for application to be accessible over web server
-    :param env: Object containing server and request information
-    :type env: dict
-    :param start_response: Callable to define responses.
-    :type start_response: any
-    :return: a list containing a bytes representation of the response object
-    :rtype: list
-    """
-
-    # define headers
-    errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
-    headers = [('Content-Type', 'text/plain')]
-
-    if get_request_endpoint(env) == '/pin':
-        phone_number = get_query_parameters(env=env, query_name='phoneNumber')
-        phone_number = quote_plus(phone_number)
-        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
-        response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
-        SessionBase.session.close()
-        start_response(message, headers)
-        return [response_bytes]
-
-    # handle requests for locked accounts
-    response, message = process_locked_accounts_requests(env=env)
-    response_bytes, headers = define_response_with_content(headers=headers, response=response)
-    start_response(message, headers)
-    SessionBase.session.close()
-    return [response_bytes]
-

apps/cic-ussd/cic_ussd/runnable/server.py

@@ -1,23 +1,25 @@
-"""This module handles requests originating from the ussd service provider.
+"""Functions defining WSGI interaction with external http requests
+Defines an application function essential for the uWSGI python loader to run th python application code.
 """
-
 # standard imports
-import json
-import logging
-
-# third-party imports
+import argparse
 import celery
 import i18n
+import json
+import logging
+import os
 import redis
-from chainlib.chain import ChainSpec
+
+# third-party imports
 from confini import Config
+from chainlib.chain import ChainSpec
+from urllib.parse import quote_plus
 
 # local imports
 from cic_ussd.chain import Chain
 from cic_ussd.db import dsn_from_config
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.encoder import PasswordEncoder
-from cic_ussd.error import InitializationError
 from cic_ussd.files.local_files import create_local_file_data_stores, json_file_parser
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata.signer import Signer
@@ -26,17 +28,34 @@ from cic_ussd.operations import (define_response_with_content,
                                  process_menu_interaction_requests,
                                  define_multilingual_responses)
 from cic_ussd.phone_number import process_phone_number
-from cic_ussd.processor import get_default_token_data
-from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore
+from cic_ussd.redis import InMemoryStore
 from cic_ussd.requests import (get_request_endpoint,
-                               get_request_method)
-from cic_ussd.runnable.server_base import exportable_parser, logg
+                               get_request_method,
+                               get_query_parameters,
+                               process_locked_accounts_requests,
+                               process_pin_reset_requests)
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.state_machine import UssdStateMachine
 from cic_ussd.validator import check_ip, check_request_content_length, check_service_code, validate_phone_number, \
     validate_presence
 
-args = exportable_parser.parse_args()
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+config_directory = '/usr/local/etc/cic-ussd/'
+
+# define arguments
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=config_directory, help='config directory.')
+arg_parser.add_argument('-q', type=str, default='cic-ussd', help='queue name for worker tasks')
+arg_parser.add_argument('-v', action='store_true', help='be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
+arg_parser.add_argument('--env-prefix',
+                        default=os.environ.get('CONFINI_ENV_PREFIX'),
+                        dest='env_prefix',
+                        type=str,
+                        help='environment prefix for variables to overwrite configuration')
+args = arg_parser.parse_args()
 
 # define log levels
 if args.vv:
@@ -50,14 +69,7 @@ config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-# set up db
-data_source_name = dsn_from_config(config)
-SessionBase.connect(data_source_name,
-                    pool_size=int(config.get('DATABASE_POOL_SIZE')),
-                    debug=config.true('DATABASE_DEBUG'))
-# create session for the life time of http request
-SessionBase.session = SessionBase.create_session()
-
+# initialize elements
 # set up translations
 i18n.load_path.append(config.get('APP_LOCALE_PATH'))
 i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
@@ -70,6 +82,12 @@ ussd_menu_db = create_local_file_data_stores(file_location=config.get('USSD_MENU
                                              table_name='ussd_menu')
 UssdMenu.ussd_menu_db = ussd_menu_db
 
+# set up db
+data_source_name = dsn_from_config(config)
+SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
+# create session for the life time of http request
+SessionBase.session = SessionBase.create_session()
+
 # define universal redis cache access
 InMemoryStore.cache = redis.StrictRedis(host=config.get('REDIS_HOSTNAME'),
                                         port=config.get('REDIS_PORT'),
@@ -109,20 +127,6 @@ Chain.spec = chain_spec
 UssdStateMachine.states = states
 UssdStateMachine.transitions = transitions
 
-# retrieve default token data
-default_token_data = get_default_token_data()
-chain_str = Chain.spec.__str__()
-
-# cache default token for re-usability
-if default_token_data:
-    cache_key = create_cached_data_key(
-        identifier=chain_str.encode('utf-8'),
-        salt=':cic.default_token_data'
-    )
-    cache_data(key=cache_key, data=json.dumps(default_token_data))
-else:
-    raise InitializationError(f'Default token data for: {chain_str} not found.')
-
 
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
@@ -130,8 +134,6 @@ def application(env, start_response):
     :type env: dict
     :param start_response: Callable to define responses.
     :type start_response: any
-    :return: a list containing a bytes representation of the response object
-    :rtype: list
     """
     # define headers
     errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
@@ -192,3 +194,20 @@ def application(env, start_response):
         start_response('200 OK,', headers)
         SessionBase.session.close()
         return [response_bytes]
+
+    # handle pin requests
+    if get_request_endpoint(env) == '/pin':
+        phone_number = get_query_parameters(env=env, query_name='phoneNumber')
+        phone_number = quote_plus(phone_number)
+        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
+        response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
+        SessionBase.session.close()
+        start_response(message, headers)
+        return [response_bytes]
+
+    # handle requests for locked accounts
+    response, message = process_locked_accounts_requests(env=env)
+    response_bytes, headers = define_response_with_content(headers=headers, response=response)
+    start_response(message, headers)
+    SessionBase.session.close()
+    return [response_bytes]

apps/cic-ussd/cic_ussd/runnable/server_base.py

@@ -1,38 +0,0 @@
-"""This module handles generic wsgi server configurations that can then be subsumed by different server flavors for the
-cic-ussd component.
-"""
-
-# standard imports
-import logging
-import os
-from argparse import ArgumentParser
-
-# third-party imports
-
-# local imports
-
-# define a logging system
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-# define default config directory as would be defined in docker
-default_config_dir = '/usr/local/etc/cic-ussd/'
-
-# define args parser
-arg_parser = ArgumentParser(description='CLI for handling cic-ussd server applications.')
-arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-arg_parser.add_argument('-v', help='be verbose', action='store_true')
-arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
-arg_parser.add_argument('-q', type=str, default='cic-ussd', help='queue name for worker tasks')
-arg_parser.add_argument('--env-prefix',
-                        default=os.environ.get('CONFINI_ENV_PREFIX'),
-                        dest='env_prefix',
-                        type=str,
-                        help='environment prefix for variables to overwrite configuration')
-exportable_parser = arg_parser
-
-
-
-
-
-

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -64,7 +64,7 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> boo
     # get cached balance
     key = create_cached_data_key(
         identifier=bytes.fromhex(user.blockchain_address[2:]),
-        salt=':cic.balances_data'
+        salt='cic.balances_data'
     )
     cached_balance = get_cached_data(key=key)
     operational_balance = compute_operational_balance(balances=json.loads(cached_balance))

apps/cic-ussd/cic_ussd/state_machine/logic/user.py

@@ -176,7 +176,7 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
     blockchain_address = user.blockchain_address
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
+        cic_type='cic.person'
     )
     user_metadata = get_cached_data(key=key)
 

apps/cic-ussd/cic_ussd/state_machine/logic/validator.py

@@ -23,7 +23,7 @@ def has_cached_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     # check for user metadata in cache
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
+        cic_type='cic.person'
     )
     user_metadata = get_cached_data(key=key)
     return user_metadata is not None

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -136,7 +136,7 @@ def process_balances_callback(result: list, param: str, status_code: int):
         blockchain_address = balances_data.get('address')
         key = create_cached_data_key(
             identifier=bytes.fromhex(blockchain_address[2:]),
-            salt=':cic.balances_data'
+            salt='cic.balances_data'
         )
         cache_data(key=key, data=json.dumps(balances_data))
     else:
@@ -226,7 +226,7 @@ def process_statement_callback(result, param: str, status_code: int):
 
         # cache account statement
         identifier = bytes.fromhex(param[2:])
-        key = create_cached_data_key(identifier=identifier, salt=':cic.statement')
+        key = create_cached_data_key(identifier=identifier, salt='cic.statement')
         data = json.dumps(processed_transactions)
 
         # cache statement data

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.10')
+version = (0, 3, 0, 'alpha.9')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/docker/Dockerfile

@@ -38,9 +38,8 @@ COPY cic-ussd/transitions/ cic-ussd/transitions/
 COPY cic-ussd/var/ cic-ussd/var/
 
 COPY cic-ussd/docker/db.sh \
-     cic-ussd/docker/start_cic_user_tasker.sh \
-     cic-ussd/docker/start_cic_user_ussd_server.sh\
-     cic-ussd/docker/start_cic_user_server.sh\
+     cic-ussd/docker/start_tasker.sh \
+     cic-ussd/docker/start_uwsgi.sh \
      /root/
 
 RUN chmod +x /root/*.sh

apps/cic-ussd/docker/start_cic_user_server.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-. /root/db.sh
-
-user_server_port=${SERVER_PORT:-9500}
-
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_cic_user_tasker.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-. /root/db.sh
-
-/usr/local/bin/cic-user-tasker "$@"

apps/cic-ussd/docker/start_cic_user_ussd_server.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-. /root/db.sh
-
-user_ussd_server_port=${SERVER_PORT:-9000}
-
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_tasker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. /root/db.sh
+
+/usr/local/bin/cic-ussd-tasker $@

apps/cic-ussd/docker/start_uwsgi.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. /root/db.sh
+
+server_port=${SERVER_PORT:-9000}
+
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/server.py --http :$server_port --pyargv "$@"

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b2
-cic-eth~=0.11.0b9
+cic_base[full_graph]~=0.1.2a68
+cic-eth~=0.11.0b3
 cic-notify~=0.4.0a4
 cic-types~=0.1.0a10

apps/cic-ussd/setup.cfg

@@ -35,7 +35,6 @@ packages =
 	cic_ussd.menu
 	cic_ussd.metadata
 	cic_ussd.runnable
-	cic_ussd.runnable.daemons
 	cic_ussd.session
 	cic_ussd.state_machine
 	cic_ussd.state_machine.logic
@@ -45,5 +44,5 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-user-tasker = cic_ussd.runnable.daemons.cic_user_tasker:main
+	cic-ussd-tasker = cic_ussd.runnable.tasker:main
 	cic-ussd-client = cic_ussd.runnable.client:main

apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py

@@ -105,7 +105,7 @@ def test_get_user_metadata(caplog,
         assert 'Get latest data status: 200' in caplog.text
     key = generate_metadata_pointer(
         identifier=identifier,
-        cic_type=':cic.person'
+        cic_type='cic.person'
     )
     cached_user_metadata = get_cached_data(key=key)
     assert cached_user_metadata

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_validator_logic.py

@@ -36,7 +36,7 @@ def test_has_cached_user_metadata(create_in_db_ussd_session,
     user = create_activated_user
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
+        cic_type='cic.person'
     )
     cache_data(key=key, data=json.dumps(person_metadata))
     result = has_cached_user_metadata(state_machine_data=state_machine_data)