Omit version code in cov

apps/cic-cache/.gitlab-ci.yml

@@ -36,7 +36,7 @@ test-mr-cic-cache:
     rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-          - apps/$APP_NAME/**/*
+          - apps/cic-eth/**/*
       when: always
 
 build-push-cic-cache:

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,4 +100,3 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-cache/requirements.txt

@@ -1,13 +1,13 @@
-cic-base~=0.2.0a4
+cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
-confini>=0.3.6rc3,<0.5.0
+confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.6a2
+cic-eth-registry~=0.5.6a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a5
+chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a2
+erc20-faucet~=0.2.2a1

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry~=0.5.6a2
+cic-eth-registry~=0.5.6a1
 hexathon~=0.0.1a7
-chainqueue~=0.0.2b6
+chainqueue~=0.0.2b5
-eth-erc20~=0.0.10a3
+eth-erc20==0.0.10a2

apps/cic-eth/cic_eth/db/models/base.py

@@ -126,4 +126,3 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,11 +14,9 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
-from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
-from cic_eth_registry.error import UnknownContractError
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
@@ -116,6 +114,9 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
+                        token = ERC20Token(chain_spec, rpc, tx['to'])
+                        token_symbol = token.symbol
+                        token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
@@ -125,6 +126,12 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
                             'destination_token': tx['to'],
+                            'source_token_symbol': token_symbol,
+                            'destination_token_symbol': token_symbol,
+                            'source_token_decimals': token_decimals,
+                            'destination_token_decimals': token_decimals,
+                            'source_token_chain': chain_str,
+                            'destination_token_chain': chain_str,
                             'nonce': tx['nonce'],
                                 }
                         if times['queue'] != None:
@@ -139,8 +146,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
-@celery_app.task(bind=True, base=BaseTask)
+@celery_app.task()
-def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=True, verify_contracts=True):
+def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
     """Merges transaction data from multiple sources and sorts them in chronological order.
 
     :param tx_batches: Transaction data inputs
@@ -189,32 +196,6 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr
     if newest_first:
         ks.reverse()
     for k in ks:
-        tx = txs_by_block[k]
+        txs.append(txs_by_block[k])
-        if verify_contracts:
-            try:
-                tx = verify_and_expand(tx, chain_spec, sender_address=BaseTask.call_address)
-            except UnknownContractError:
-                logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
-                continue
-        txs.append(tx)
 
     return txs
-
-
-def verify_and_expand(tx, chain_spec, sender_address=ZERO_ADDRESS):
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    registry = CICRegistry(chain_spec, rpc)
-
-    if tx.get('source_token_symbol') == None and tx['source_token'] != ZERO_ADDRESS:
-        r = registry.by_address(tx['source_token'], sender_address=sender_address)
-        token = ERC20Token(chain_spec, rpc, tx['source_token'])
-        tx['source_token_symbol'] = token.symbol
-        tx['source_token_decimals'] = token.decimals
-
-    if tx.get('destination_token_symbol') == None and tx['destination_token'] != ZERO_ADDRESS:
-        r = registry.by_address(tx['destination_token'], sender_address=sender_address)
-        token = ERC20Token(chain_spec, rpc, tx['destination_token'])
-        tx['destination_token_symbol'] = token.symbol
-        tx['destination_token_decimals'] = token.decimals
-
-    return tx

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -27,7 +27,7 @@ def database_engine(
         SessionBase.poolable = False
     dsn = dsn_from_config(load_config)
     #SessionBase.connect(dsn, True)
-    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
+    SessionBase.connect(dsn, debug=load_config.get('DATABASE_DEBUG') != None)
     return dsn
 
 

apps/cic-eth/cic_eth/queue/query.py

@@ -100,7 +100,6 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     q_outer = q_outer.join(Lock, isouter=True)
     q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
 
-
     if not is_alive(status):
         SessionBase.release_session(session)
         raise ValueError('not a valid non-final tx value: {}'.format(status))

apps/cic-eth/cic_eth/runnable/create.py

@@ -80,12 +80,7 @@ def main():
     t = api.create_account(register=register)
 
     ps.get_message()
-    try:
+    o = ps.get_message(timeout=args.timeout)
-        o = ps.get_message(timeout=args.timeout)
-    except TimeoutError as e:
-        sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e))
-        sys.exit(1) 
-    ps.unsubscribe()
     m = json.loads(o['data'])
     print(m['result'])
 

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -90,7 +90,6 @@ class DispatchSyncer:
 
     def __init__(self, chain_spec):
         self.chain_spec = chain_spec
-        self.session = None
 
 
     def chain(self):
@@ -101,18 +100,16 @@ class DispatchSyncer:
         c = len(txs.keys())
         logg.debug('processing {} txs {}'.format(c, list(txs.keys())))
         chain_str = str(self.chain_spec)
-        self.session = SessionBase.create_session()
+        session = SessionBase.create_session()
         for k in txs.keys():
             tx_raw = txs[k]
             tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
             tx = unpack(tx_raw_bytes, self.chain_spec)
             
             try:
-                set_reserved(self.chain_spec, tx['hash'], session=self.session)
+                set_reserved(self.chain_spec, tx['hash'], session=session)
-                self.session.commit()
             except NotLocalTxError as e:
                 logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash']))
-                self.session.rollback()
                 continue
 
             s_check = celery.signature(
@@ -135,25 +132,16 @@ class DispatchSyncer:
             s_check.link(s_send)
             t = s_check.apply_async()
             logg.info('processed {}'.format(k))
-        self.session.close()
-        self.session = None
 
 
-    def loop(self, interval):
+    def loop(self, w3, interval):
         while run:
             txs = {}
             typ = StatusBits.QUEUED
             utxs = get_upcoming_tx(self.chain_spec, typ)
             for k in utxs.keys():
                 txs[k] = utxs[k]
-            try:
+            self.process(w3, txs)
-                conn = RPCConnection.connect(self.chain_spec, 'default')
-                self.process(conn, txs)
-            except ConnectionError as e:
-                if self.session != None:
-                    self.session.close()
-                    self.session = None
-                logg.error('connection to node failed: {}'.format(e))
 
             if len(utxs) > 0:
                 time.sleep(self.yield_delay)
@@ -163,7 +151,8 @@ class DispatchSyncer:
 
 def main(): 
     syncer = DispatchSyncer(chain_spec)
-    syncer.loop(float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    conn = RPCConnection.connect(chain_spec, 'default')
+    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -12,24 +12,20 @@ from hexathon import (
 # local imports
 from .base import SyncFilter
 
-logg = logging.getLogger(__name__)
+logg = logging.getLogger().getChild(__name__)
 
 account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 
 
 class RegistrationFilter(SyncFilter):
 
-    def __init__(self, chain_spec, contract_address, queue=None):
+    def __init__(self, chain_spec, queue):
         self.chain_spec = chain_spec
         self.queue = queue
-        self.contract_address = contract_address
 
 
     def filter(self, conn, block, tx, db_session=None): 
-        if self.contract_address != tx.inputs[0]:
+        registered_address = None
-            logg.debug('not an account registry tx; {} != {}'.format(self.contract_address, tx.inputs[0]))
-            return None
-
         for l in tx.logs:
             event_topic_hex = l['topics'][0]
             if event_topic_hex == account_registry_add_log_hash:

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -78,14 +78,6 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
-rpc = RPCConnection.connect(chain_spec, 'default')
-registry = None
-try:
-    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-except UnknownContractError as e:
-    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-    sys.exit(1)
-logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 
 def main():
@@ -93,6 +85,7 @@ def main():
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # Connect to blockchain with chainlib
+    rpc = RPCConnection.connect(chain_spec, 'default')
 
     o = block_latest()
     r = rpc.do(o)
@@ -158,8 +151,7 @@ def main():
 
     tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
 
-    account_registry_address = registry.by_name('AccountRegistry')
+    registration_filter = RegistrationFilter(chain_spec, config.get('_CELERY_QUEUE'))
-    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE'))
 
     gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
 

apps/cic-eth/cic_eth/version.py

@@ -9,7 +9,7 @@ import semver
 version = (
         0,
         12,
-        1,
+        0,
         'alpha.2',
         )
 

apps/cic-eth/config/test/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=0
+DEBUG=

apps/cic-eth/docker/Dockerfile

@@ -13,17 +13,11 @@ ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages
 #    --force-reinstall \
 #	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
 #    -r requirements.txt 
-COPY *requirements.txt . 
+
+COPY . .
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL .
-    --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt \
-    -r services_requirements.txt \
-    -r admin_requirements.txt
-    
-COPY . .
-RUN python setup.py install
 
 ENV PYTHONPATH .
 
@@ -41,31 +35,30 @@ COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 #COPY util/liveness/health.sh /usr/local/bin/health.sh
 ENTRYPOINT []
 
-## ------------------ PRODUCTION CONTAINER ----------------------
+# --- RUNTIME ---
-#FROM python:3.8.6-slim-buster as prod 
+FROM python:3.8.6-slim-buster as runtime 
-#
+
-#RUN apt-get update && \
+RUN apt-get update && \
-#	apt install -y gnupg libpq-dev procps
+	apt install -y gnupg libpq-dev procps
-#
+
-#WORKDIR /root
+WORKDIR /usr/src/cic-eth
-#
+
-#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+COPY --from=dev /usr/local/bin/ /usr/local/bin/
-#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
-#                          /usr/local/lib/python3.8/site-packages/
+                          /usr/local/lib/python3.8/site-packages/
-#
+
-#COPY docker/entrypoints/* ./ 
+COPY docker/entrypoints/* ./ 
-#RUN chmod 755 *.sh
+RUN chmod 755 *.sh
-#
+
-## # ini files in config directory defines the configurable parameters for the application
+# # ini files in config directory defines the configurable parameters for the application
-## # they can all be overridden by environment variables
+# # they can all be overridden by environment variables
-## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-eth/
+COPY config/ /usr/local/etc/cic-eth/
-#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-#COPY scripts/ scripts/ 
+
-#
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
-##COPY util/liveness/health.sh /usr/local/bin/health.sh
+
-#
+ENTRYPOINT []
-#ENTRYPOINT []
 

apps/cic-eth/docker/Dockerfile_ci

@@ -14,16 +14,10 @@ ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages
 #    --force-reinstall \
 #	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
 #    -r requirements.txt 
-COPY *requirements.txt . 
+
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt \
-    -r services_requirements.txt \
-    -r admin_requirements.txt
-    
 COPY . .
-RUN python setup.py install
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL .
 
 COPY docker/entrypoints/* ./ 
 RUN chmod 755 *.sh
@@ -39,31 +33,30 @@ COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 #COPY util/liveness/health.sh /usr/local/bin/health.sh
 ENTRYPOINT []
 
-# ------------------ PRODUCTION CONTAINER ----------------------
+# --- RUNTIME ---
-#FROM python:3.8.6-slim-buster as prod 
+FROM python:3.8.6-slim-buster as runtime 
-#
+
-#RUN apt-get update && \
+RUN apt-get update && \
-#	apt install -y gnupg libpq-dev procps
+	apt install -y gnupg libpq-dev procps
-#
+
-#WORKDIR /root
+WORKDIR /usr/src/cic-eth
-#
+
-#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+COPY --from=dev /usr/local/bin/ /usr/local/bin/
-#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
-#                          /usr/local/lib/python3.8/site-packages/
+                          /usr/local/lib/python3.8/site-packages/
-#
+
-#COPY docker/entrypoints/* ./ 
+COPY docker/entrypoints/* ./ 
-#RUN chmod 755 *.sh
+RUN chmod 755 *.sh
-#
+
-## # ini files in config directory defines the configurable parameters for the application
+# # ini files in config directory defines the configurable parameters for the application
-## # they can all be overridden by environment variables
+# # they can all be overridden by environment variables
-## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-eth/
+COPY config/ /usr/local/etc/cic-eth/
-#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
-#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-#COPY scripts/ scripts/ 
+
-#
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
-## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
-##COPY util/liveness/health.sh /usr/local/bin/health.sh
+
-#
+ENTRYPOINT []
-#ENTRYPOINT []
+
-#

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib~=0.0.5a2
+chainlib~=0.0.5a1
 semver==2.13.0

apps/cic-eth/services_requirements.txt

@@ -1,15 +1,15 @@
-chainsyncer[sql]~=0.0.3a5
+chainsyncer[sql]~=0.0.3a3
-chainqueue~=0.0.2b7
+chainqueue~=0.0.2b5
 alembic==1.4.2
-confini>=0.3.6rc4,<0.5.0
+confini~=0.3.6rc4
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index~=0.1.2a2
+eth-address-index~=0.1.2a1
-eth-accounts-index~=0.0.12a2
+eth-accounts-index~=0.0.12a1
-cic-eth-registry~=0.5.6a2
+cic-eth-registry~=0.5.6a1
-erc20-faucet~=0.2.2a2
+erc20-faucet~=0.2.2a1
-erc20-transfer-authorization~=0.3.2a2
+erc20-transfer-authorization~=0.3.2a1
-sarafu-faucet~=0.0.4a3
+sarafu-faucet~=0.0.4a1
 moolb~=0.1.1b2

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.0.10a3
+eth-erc20~=0.0.10a2

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -18,7 +18,6 @@ def test_filter_bogus(
         cic_registry,
         contract_roles,
         register_lookups,
-        account_registry,
         ):
 
     fltrs = [
@@ -27,7 +26,7 @@ def test_filter_bogus(
         TxFilter(default_chain_spec, None),
         CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
         StragglerFilter(default_chain_spec, None),
-        RegistrationFilter(default_chain_spec, account_registry, queue=None),
+        RegistrationFilter(default_chain_spec, queue=None),
         ]
       
     for fltr in fltrs:

apps/cic-eth/tests/filters/test_register_filter.py

@@ -1,7 +1,3 @@
-# standard imports
-import logging
-import os
-
 # external imports
 from eth_accounts_index.registry import AccountRegistry
 from chainlib.connection import RPCConnection
@@ -18,17 +14,12 @@ from chainlib.eth.block import (
         Block,
         )
 from erc20_faucet import Faucet
-from hexathon import (
+from hexathon import strip_0x
-        strip_0x,
-        add_0x,
-        )
 from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
 
-logg = logging.getLogger()
-
 
 def test_register_filter(
         default_chain_spec,
@@ -69,11 +60,7 @@ def test_register_filter(
     tx = Tx(tx_src, block=block, rcpt=rcpt)
     tx.apply_receipt(rcpt)
 
-    fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None)
+    fltr = RegistrationFilter(default_chain_spec, queue=None)
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    assert t == None
-
-    fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
 
     t.get_leaf()

apps/cic-eth/tests/task/test_task_list.py

@@ -1,92 +0,0 @@
-# external imports
-import celery
-import pytest
-from chainlib.connection import RPCConnection
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.gas import (
-        RPCGasOracle,
-        )
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from eth_erc20 import ERC20
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
-from chainqueue.db.models.tx import TxCache
-from chainqueue.db.models.otx import Otx
-
-
-def test_ext_tx_collate(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        custodial_roles,
-        agent_roles,
-        foo_token,
-        bar_token,
-        register_tokens,
-        cic_registry,
-        register_lookups,
-        init_celery_tasks,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], eth_rpc)
-    gas_oracle = RPCGasOracle(eth_rpc)
-
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    transfer_value_foo = 1000
-    transfer_value_bar = 1024
-    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], transfer_value_foo, tx_format=TxFormat.RLP_SIGNED)
-    tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
-
-    otx = Otx(
-        tx['nonce'],
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        )
-    init_database.add(otx)
-    init_database.commit()
-
-    txc = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        foo_token,
-        bar_token,
-        transfer_value_foo,
-        transfer_value_bar,
-        666,
-        13,
-        session=init_database,
-            )
-    init_database.add(txc)
-    init_database.commit()
-    
-    s = celery.signature(
-            'cic_eth.ext.tx.tx_collate',
-            [
-                {tx_hash_hex: tx_signed_raw_hex},
-                default_chain_spec.asdict(),
-                0,
-                100,
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    assert len(r) == 1
-
-    tx = r[0]
-    assert tx['source_token_symbol'] == 'FOO'
-    assert tx['source_token_decimals'] == 6
-    assert tx['destination_token_symbol'] == 'BAR'
-    assert tx['destination_token_decimals'] == 9

apps/cic-eth/tests/unit/db/test_nonce_db.py

@@ -1,7 +1,6 @@
-# extended imports
+# third-party imports
 import pytest
 import uuid
-import unittest
 
 # local imports
 from cic_eth.db.models.nonce import (
@@ -56,7 +55,7 @@ def test_nonce_reserve(
     o = q.first()
     assert o.nonce == 43
 
-    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu), session=init_database)
+    nonce = NonceReservation.release(eth_empty_accounts[0], str(uu))
     init_database.commit()
     assert nonce == (str(uu), 42)
 

apps/cic-eth/tests/unit/ext/test_address.py

@@ -1,5 +1,4 @@
 # external imports
-import celery
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
@@ -21,7 +20,6 @@ def test_translate(
         cic_registry,
         init_celery_tasks,
         register_lookups,
-        celery_session_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
@@ -48,20 +46,6 @@ def test_translate(
         'recipient': agent_roles['BOB'],
         'recipient_label': None,
             }
-    
+    tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
-    #tx = translate_tx_addresses(tx, [contract_roles['CONTRACT_DEPLOYER']], default_chain_spec.asdict())
+    assert tx['sender_label'] == 'alice'
-    s = celery.signature(
+    assert tx['recipient_label'] == 'bob'
-            'cic_eth.ext.address.translate_tx_addresses',
-            [
-                tx,
-                [contract_roles['CONTRACT_DEPLOYER']],
-                default_chain_spec.asdict(),
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    assert r['sender_label'] == 'alice'
-    assert r['recipient_label'] == 'bob'

apps/cic-eth/tools_requirements.txt

@@ -1,7 +1,7 @@
-crypto-dev-signer~=0.4.14b7
+crypto-dev-signer~=0.4.14b6
-chainqueue~=0.0.2b6
+chainqueue~=0.0.2b5
-confini>=0.3.6rc4,<0.5.0
+confini~=0.3.6rc4
-cic-eth-registry~=0.5.6a2
+cic-eth-registry~=0.5.6a1
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1

apps/cic-meta/.dockerignore

@@ -1,5 +1,4 @@
 .git
 .cache
 .dot
-**/doc
+**/doc
-**/node_modules

apps/cic-meta/.gitlab-ci.yml

@@ -2,8 +2,8 @@
 .cic_meta_variables:
     variables:
         APP_NAME: cic-meta
-        DOCKERFILE_PATH: docker/Dockerfile_ci
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        CONTEXT: apps/$APP_NAME
+        CONTEXT: apps
 
 build-mr-cic-meta:
     extends:
@@ -21,7 +21,7 @@ test-mr-cic-meta:
     stage: test
     image: $MR_IMAGE_TAG 
     script:
-        - cd /root
+        - cd /tmp/src/cic-meta
         - npm install --dev 
         - npm run test
         - npm run test:coverage

apps/cic-meta/docker/Dockerfile

@@ -1,33 +1,31 @@
-# syntax = docker/dockerfile:1.2
+FROM node:15.3.0-alpine3.10
-#FROM node:15.3.0-alpine3.10
-FROM node:lts-alpine3.14
 
-WORKDIR /root
+WORKDIR /tmp/src/cic-meta
 
 RUN apk add --no-cache postgresql bash
 
+# required to build the cic-client-meta module
+COPY cic-meta/src/ src/
+COPY cic-meta/scripts/ scripts/
+
 # copy the dependencies
-COPY package.json package-lock.json .
+COPY cic-meta/package.json .
-RUN --mount=type=cache,mode=0755,target=/root/.npm \
+COPY cic-meta/tsconfig.json . 
-    npm set cache /root/.npm && \
+COPY cic-meta/webpack.config.js .
-    npm ci
+     
+RUN npm install
 
-COPY webpack.config.js .
+COPY cic-meta/tests/ tests/
-COPY tsconfig.json . 
+COPY cic-meta/tests/*.asc /root/pgp/
-## required to build the cic-client-meta module
-COPY src/ src/
-COPY scripts/ scripts/
-COPY tests/ tests/
-COPY tests/*.asc /root/pgp/
 
-## copy runtime configs
+# copy runtime configs
-COPY .config/ /usr/local/etc/cic-meta/
+COPY cic-meta/.config/ /usr/local/etc/cic-meta/
-#
+
-## db migrations
+# db migrations
-COPY docker/db.sh ./db.sh
+COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
-#
+
 RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY docker/start_server.sh ./start_server.sh
+COPY cic-meta/docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/Dockerfile_ci

@@ -1,32 +0,0 @@
-# syntax = docker/dockerfile:1.2
-#FROM node:15.3.0-alpine3.10
-FROM node:lts-alpine3.14
-
-WORKDIR /root
-
-RUN apk add --no-cache postgresql bash
-
-# copy the dependencies
-COPY package.json package-lock.json .
-RUN npm set cache /root/.npm && \
-    npm ci
-
-COPY webpack.config.js .
-COPY tsconfig.json . 
-## required to build the cic-client-meta module
-COPY src/ src/
-COPY scripts/ scripts/
-COPY tests/ tests/
-COPY tests/*.asc /root/pgp/
-
-## copy runtime configs
-COPY .config/ /usr/local/etc/cic-meta/
-#
-## db migrations
-COPY docker/db.sh ./db.sh
-RUN chmod 755 ./db.sh
-#
-RUN alias tsc=node_modules/typescript/bin/tsc 
-COPY docker/start_server.sh ./start_server.sh
-RUN chmod 755 ./start_server.sh
-ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-notify/.gitlab-ci.yml

@@ -36,7 +36,7 @@ test-mr-cic-notify:
     rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-          - apps/$APP_NAME/**/*
+          - apps/cic-eth/**/*
       when: always
 
 build-push-cic-notify:

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.8')
+version = (0, 4, 0, 'alpha.7')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/requirements.txt

@@ -1 +1 @@
-cic_base[full_graph]~=0.2.0a3
+cic_base[full_graph]==0.1.3a3+build.984b5cff

apps/cic-ussd/.config/app.ini

@@ -1,7 +1,7 @@
 [app]
 ALLOWED_IP=0.0.0.0/0
 LOCALE_FALLBACK=en
-LOCALE_PATH=var/lib/locale/
+LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#,*483*061#,*384*96#
@@ -11,13 +11,13 @@ SUPPORT_PHONE_NUMBER=0757628885
 REGION=KE
 
 [ussd]
-MENU_FILE=data/ussd_menu.json
+MENU_FILE=/usr/src/data/ussd_menu.json
 user = 
 pass = 
 
 [statemachine]
-STATES=states/
+STATES=/usr/src/cic-ussd/states/
-TRANSITIONS=transitions/
+TRANSITIONS=/usr/src/cic-ussd/transitions/
 
 [client]
 host =

apps/cic-ussd/.config/pgp.ini

@@ -1,5 +1,5 @@
 [pgp]
-export_dir = pgp/keys/
+export_dir = /usr/src/pgp/keys/
 keys_path = /usr/src/secrets/
 private_keys = privatekeys_meta.asc
 passphrase =

apps/cic-ussd/.gitlab-ci.yml

@@ -36,7 +36,7 @@ test-mr-cic-ussd:
     rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-          - apps/$APP_NAME/**/*
+          - apps/cic-eth/**/*
       when: always
 
 build-push-cic-ussd:

apps/cic-ussd/cic_ussd/db/models/base.py

@@ -127,4 +127,3 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -28,7 +28,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(args.c, env_prefix=args.env_prefix)
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -45,7 +45,7 @@ elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
 # parse config
-config = Config(args.c, env_prefix=args.env_prefix)
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))

apps/cic-ussd/docker/start_cic_user_server.sh

@@ -4,4 +4,4 @@
 
 user_server_port=${SERVER_PORT:-9500}
 
-/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_cic_user_ussd_server.sh

@@ -4,4 +4,4 @@
 
 user_ussd_server_port=${SERVER_PORT:-9000}
 
-/usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]==0.2.0a3
+cic_base[full_graph]==0.1.3a3+build.984b5cff
 cic-eth~=0.12.0a1
-cic-notify~=0.4.0a8
+cic-notify~=0.4.0a7
 cic-types~=0.1.0a11

apps/cic-ussd/scripts/migrate.py

@@ -30,7 +30,7 @@ arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
 args = arg_parser.parse_args()
 
 
-config = Config(args.c, env_prefix=args.env_prefix)
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug(f'config:\n{config}')

apps/contract-migration/config_template/bancor.ini

@@ -0,0 +1,2 @@
+[bancor]
+dir = /usr/local/share/cic/bancor

apps/contract-migration/config_template/chain.ini

@@ -1,2 +0,0 @@
-[chain]
-spec = evm:ethereum:1

apps/contract-migration/config_template/redis.ini

@@ -1,4 +0,0 @@
-[redis]
-host = localhost
-port = 63379
-db = 0

apps/contract-migration/config_template/rpc.ini

@@ -1,5 +0,0 @@
-[rpc]
-http_provider = http://localhost:8545
-http_authentication = 
-http_username =
-http_password =

apps/contract-migration/config_template/token.ini

@@ -1,8 +0,0 @@
-[token]
-name =
-symbol =
-decimals =
-demurrage_level =
-redistribution_period =
-sink_address =
-supply_limit = 0

apps/contract-migration/config_template/wallet.ini

@@ -1,3 +0,0 @@
-[wallet]
-key_file = 
-passphrase = 

apps/contract-migration/docker/Dockerfile

@@ -14,18 +14,17 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C20
 RUN mkdir -vp /usr/local/etc/cic
 
 ENV CONFINI_DIR /usr/local/etc/cic/
-#RUN mkdir -vp $CONFINI_DIR
+RUN mkdir -vp $CONFINI_DIR
 
-#ARG cic_config_commit=24287fb253196820f23ff8a7177b122f2cd99a11
+ARG cic_config_commit=0abe0867f18077907c7023bf0ef5e466a3984dd8
-#ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
+ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
-#RUN echo Install confini schema files && \
+RUN echo Install confini schema files && \
-#	git clone --depth 1 $cic_config_url cic-config && \
+	git clone --depth 1 $cic_config_url cic-config && \
-#	cd cic-config && \
+	cd cic-config && \
-#	git fetch --depth 1 origin $cic_config_commit && \
+	git fetch --depth 1 origin $cic_config_commit && \
-#	git checkout $cic_config_commit && \	
+	git checkout $cic_config_commit && \	
-#	cp -v *.ini $CONFINI_DIR
+	cp -v *.ini $CONFINI_DIR
 
-COPY config_template/ /usr/local/etc/cic/
 COPY requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple

apps/contract-migration/requirements.txt

@@ -1,11 +1,5 @@
-sarafu-faucet~=0.0.4a4
+cic_base[full]==0.1.3a4+build.ce68c833
-cic-eth[tools]==0.12.1a2
+sarafu-faucet~=0.0.4a1
+cic-eth[tools]==0.12.0a2
 eth-erc20~=0.0.10a3
-erc20-demurrage-token==0.0.2a4
+erc20-demurrage-token==0.0.2a3
-eth-address-index~=0.1.2a2
-eth-accounts-index~=0.0.12a2
-cic-eth-registry~=0.5.6a2
-erc20-faucet~=0.2.2a2
-erc20-transfer-authorization~=0.3.2a2
-sarafu-faucet~=0.0.4a3
-chainlib-eth~=0.0.5a4

apps/contract-migration/reset.sh

@@ -3,9 +3,14 @@
 set -a
 
 default_token=giftable_erc20_token
+CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
 TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL}
-TOKEN_NAME=${TOKEN_NAME}
+TOKEN_NAME=${TOKEN_NAME:-$TOKEN_SYMBOL}
 TOKEN_TYPE=${TOKEN_TYPE:-$default_token}
+if [ $TOKEN_TYPE == 'default' ]; then
+	>&2 echo resolving "default" token to $default_token
+	TOKEN_TYPE=$default_token
+fi
 cat <<EOF
 external token settings:
 token_type: $TOKEN_TYPE
@@ -18,6 +23,7 @@ token_supply_limit: $TOKEN_SUPPLY_LIMIT
 EOF
 
 CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
+TOKEN_SYMBOL=${TOKEN_SYMBOL:-GFT}
 DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
@@ -69,34 +75,17 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 	fi
 
-	if [ "$TOKEN_TYPE" == "$default_token" ]; then
+	if [ $TOKEN_TYPE == $default_token ]; then
-		if [ -z "$TOKEN_SYMBOL" ]; then
-			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
-			TOKEN_SYMBOL="GFT"
-			TOKEN_NAME="Giftable Token"
-		elif [ -z "$TOKEN_NAME" ]; then
-			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
-			TOKEN_NAME=$TOKEN_SYMBOL
-		fi
 		>&2 echo deploying default token $TOKEN_TYPE
-		echo giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name $TOKEN_NAME --symbol $TOKEN_SYMBOL --decimals 6 -vv`
-		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
+	elif [ $TOKEN_TYPE == 'erc20_demurrage_token' ]; then
-	elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
-		if [ -z "$TOKEN_SYMBOL" ]; then
-			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
-			TOKEN_SYMBOL="SARAFU"
-			TOKEN_NAME="Sarafu Token"
-		elif [ -z "$TOKEN_NAME" ]; then
-			>&2 echo token name not set, setting same as symbol for type $TOKEN_TYPE
-			TOKEN_NAME=$TOKEN_SYMBOL
-		fi
 		>&2 echo deploying token $TOKEN_TYPE
 		if [ -z $TOKEN_SINK_ADDRESS ]; then
 			if [ ! -z $TOKEN_REDISTRIBUTION_PERIOD ]; then
 				>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
 			fi
 		fi
-		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww`
+		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww`
 	else
 		>&2 echo unknown token type $TOKEN_TYPE
 		exit 1
@@ -148,7 +137,6 @@ else
 fi
 
 mkdir -p $CIC_DATA_DIR
->&2 echo using data dir $CIC_DATA_DIR for environment variable dump
 
 # this is consumed in downstream services to set environment variables
 cat << EOF > $CIC_DATA_DIR/.env

apps/contract-migration/seed_cic_eth.sh

@@ -15,6 +15,10 @@ CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic}
 ETH_PASSPHRASE=''
 CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
 TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
+if [[ $CIC_DEFAULT_TOKEN_SYMBOL != 'GFT' && $CIC_DEFAULT_TOKEN_SYMBOL != 'SRF' ]]; then
+	>&2 echo CIC_DEFAULT_TOKEN_SYMBOL must be one of [GFT,SRF], but was $CIC_DEFAULT_TOKEN_SYMBOL
+	exit 1
+fi
 
 # Debug flag
 DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
@@ -44,18 +48,18 @@ EOF
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
-DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
 
 
 >&2 echo "create account for sarafu gifter"
-DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER
 
 >&2 echo "create account for approval escrow owner"
-DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER 
 
@@ -65,7 +69,7 @@ cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRA
 #cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER
 
 >&2 echo "create account for accounts index writer"
-DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"
@@ -73,13 +77,13 @@ eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER
 
 # Transfer gas to custodial gas provider adddress
 >&2 echo gift gas to gas gifter
->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
 
 >&2 echo gift gas to sarafu token owner
->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
 
 >&2 echo gift gas to account index owner
->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
 
 
 # Send token to token creator

apps/data-seeding/.dockerignore

@@ -1,8 +1,4 @@
 .git
 .cache
 .dot
-**/doc
+**/doc
-**/node_modules
-**/venv
-**/.venv
-

apps/data-seeding/README.md

@@ -167,7 +167,7 @@ If you have previously run the `cic_ussd` import incompletely, it could be a goo
 
 Then, in sequence, run in first terminal:
 
-`python cic_ussd/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
 
 In second terminal:
 

apps/data-seeding/cic_eth/import_balance.py

@@ -11,6 +11,7 @@ import csv
 import json
 
 # external imports
+import eth_abi
 import confini
 from hexathon import (
         strip_0x,
@@ -28,10 +29,7 @@ from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
 from chainlib.jsonrpc import JSONRPCRequest
-from chainlib.eth.error import (
+from chainlib.eth.error import EthException
-        EthException,
-        RequestMismatchException,
-        )
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
@@ -39,9 +37,6 @@ from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
 from cic_base.eth.syncer import chain_interface
-from eth_accounts_index import AccountsIndex
-from eth_contract_registry import Registry
-from eth_token_index import TokenUniqueSymbolIndex
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -136,52 +131,58 @@ class Handler:
             logg.debug('no payload, skipping {}'.format(tx))
             return
 
-        recipient = None
+        if tx.payload[:8] == self.account_index_add_signature:
-        try:
+            recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
-            r = AccountsIndex.parse_add_request(tx.payload)
+            #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
-        except RequestMismatchException:
+            user_file = 'new/{}/{}/{}.json'.format(
-            return
+                    recipient[2:4].upper(),
-        recipient = r[0]
+                    recipient[4:6].upper(),
-        
+                    recipient[2:].upper(),
-        user_file = 'new/{}/{}/{}.json'.format(
+                    )
-                recipient[2:4].upper(),
+            filepath = os.path.join(self.user_dir, user_file)
-                recipient[4:6].upper(),
+            o = None
-                recipient[2:].upper(),
+            try:
-                )
+                f = open(filepath, 'r')
-        filepath = os.path.join(self.user_dir, user_file)
+                o = json.load(f)
-        o = None
+                f.close()
-        try:
+            except FileNotFoundError:
-            f = open(filepath, 'r')
+                logg.error('no import record of address {}'.format(recipient))
-            o = json.load(f)
+                return
+            u = Person.deserialize(o)
+            original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
+            try:
+                balance = self.balances[original_address]
+            except KeyError as e:
+                logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
+                return
+
+            # TODO: store token object in handler ,get decimals from there
+            multiplier = 10**6
+            balance_full = balance * multiplier
+            logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
+
+            (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
+            logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
+            r = conn.do(o)
+
+            tx_path = os.path.join(
+                    user_dir,
+                    'txs',
+                    strip_0x(tx_hash_hex),
+                    )
+            f = open(tx_path, 'w')
+            f.write(strip_0x(o['params'][0]))
             f.close()
-        except FileNotFoundError:
+#        except TypeError as e:
-            logg.error('no import record of address {}'.format(recipient))
+#            logg.warning('typerror {}'.format(e))
-            return
+#            pass
-        u = Person.deserialize(o)
+#        except IndexError as e:
-        original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
+#            logg.warning('indexerror {}'.format(e))
-        try:
+#            pass
-            balance = self.balances[original_address]
+#        except EthException as e:
-        except KeyError as e:
+#            logg.error('send error {}'.format(e).ljust(200))
-            logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
+        #except KeyError as e:
-            return
+        #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
-
-        # TODO: store token object in handler ,get decimals from there
-        multiplier = 10**6
-        balance_full = balance * multiplier
-        logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
-
-        (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
-        logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
-        r = conn.do(o)
-
-        tx_path = os.path.join(
-                user_dir,
-                'txs',
-                strip_0x(tx_hash_hex),
-                )
-        f = open(tx_path, 'w')
-        f.write(strip_0x(o['params'][0]))
-        f.close()
 
 
 def progress_callback(block_number, tx_index):
@@ -197,26 +198,49 @@ def main():
     nonce_oracle = RPCNonceOracle(signer_address, conn)
 
     # Get Token registry address
-    registry = Registry(chain_spec)
+    txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
-    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
+    tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
+
+    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
+    data = add_0x(registry_addressof_method)
+    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
+    txf.set_code(tx, data)
+   
+    j = JSONRPCRequest()
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    token_index_address = registry.parse_address_of(r)
+    token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
-    token_index_address = to_checksum_address(token_index_address)
     logg.info('found token index address {}'.format(token_index_address))
-    
+
+
     # Get Sarafu token address
-    token_index = TokenUniqueSymbolIndex(chain_spec)
+    tx = txf.template(signer_address, token_index_address)
-    o = token_index.address_of(token_index_address, token_symbol)
+    data = add_0x(registry_addressof_method)
+    h = hashlib.new('sha256')
+    h.update(token_symbol.encode('utf-8'))
+    z = h.digest()
+    data += eth_abi.encode_single('bytes32', z).hex()
+    txf.set_code(tx, data)
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    token_address = token_index.parse_address_of(r)
     try:
-        token_address = to_checksum_address(token_address)
+        sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     except ValueError as e:
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
-    logg.info('found token address {}'.format(token_address))
+
-    
+    if sarafu_token_address == ZERO_ADDRESS:
-    sys.exit(0)
+        raise KeyError('token address for symbol {} is zero'.format(token_symbol))
+
+    logg.info('found token address {}'.format(sarafu_token_address))
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -224,6 +248,22 @@ def main():
         o = block_latest()
         r = conn.do(o)
         block_offset = int(strip_0x(r), 16) + 1
+#
+#    addresses = {}
+#    f = open('{}/addresses.csv'.format(user_dir, 'r'))
+#    while True:
+#        l = f.readline()
+#        if l == None:
+#            break
+#        r = l.split(',')
+#        try:
+#            k = r[0]
+#            v = r[1].rstrip()
+#            addresses[k] = v
+#            sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
+#        except IndexError as e:
+#            break
+#    f.close()
 
     # TODO get decimals from token
     balances = {}

apps/data-seeding/cic_meta/import_meta.js

@@ -11,6 +11,7 @@ const config = new crdt.Config('./config');
 config.process();
 console.log(config);
 
+
 function sendit(uid, envelope) {
 	const d = envelope.toJSON();
 

apps/data-seeding/cic_ussd/import_ussd_data.py

@@ -17,7 +17,7 @@ default_config_dir = '/usr/local/etc/cic'
 
 arg_parser = argparse.ArgumentParser()
 arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file')
-arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='Task queue')
+arg_parser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
 arg_parser.add_argument('-v', action='store_true', help='Be verbose')
 arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
 arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree')

apps/data-seeding/create_import_users.py

@@ -206,7 +206,7 @@ def gen():
         # fake.local_latitude()
         p.location['latitude'] = (random.random() * 180) - 90
         # fake.local_latitude()
-        p.location['longitude'] = (random.random() * 360) - 180
+        p.location['longitude'] = (random.random() * 360) - 179
 
     return (old_blockchain_checksum_address, phone, p)
 

apps/data-seeding/eth/import_balance.py

@@ -11,6 +11,7 @@ import csv
 import json
 
 # external imports
+import eth_abi
 import confini
 from hexathon import (
         strip_0x,
@@ -28,20 +29,13 @@ from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
 from chainlib.jsonrpc import JSONRPCRequest
-from chainlib.eth.error import (
+from chainlib.eth.error import EthException
-        EthException,
-        RequestMismatchException,
-        )
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
 from cic_base.eth.syncer import chain_interface
-from eth_accounts_index import AccountsIndex
-from eth_contract_registry import Registry
-from eth_token_index import TokenUniqueSymbolIndex
-from erc20_faucet import Faucet
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -56,7 +50,7 @@ argparser.add_argument('-c', type=str, default=config_dir, help='config root to
 argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
-argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
+argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('--head', action='store_true', help='start at current block height (overrides --offset)')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@@ -118,16 +112,12 @@ class Handler:
 
     account_index_add_signature = keccak256_string_to_hex('add(address)')[:8]
 
-    def __init__(self, conn, chain_spec, user_dir, balances, token_address, faucet_address, signer_address, signer, gas_oracle, nonce_oracle):
+    def __init__(self, conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle):
         self.token_address = token_address
-        self.faucet_address = faucet_address
         self.user_dir = user_dir
         self.balances = balances
         self.chain_spec = chain_spec
-        self.nonce_oracle = nonce_oracle
+        self.tx_factory = ERC20(chain_spec, signer, gas_oracle, nonce_oracle)
-        self.gas_oracle = gas_oracle
-        self.signer_address = signer_address
-        self.signer = signer
 
 
     def name(self):
@@ -139,59 +129,58 @@ class Handler:
             logg.debug('no payload, skipping {}'.format(tx))
             return
 
-        recipient = None
+        if tx.payload[:8] == self.account_index_add_signature:
-        try:
+            recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
-            r = AccountsIndex.parse_add_request(tx.payload)
+            #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
-        except RequestMismatchException:
+            user_file = 'new/{}/{}/{}.json'.format(
-            return
+                    recipient[2:4].upper(),
-        recipient = r[0]
+                    recipient[4:6].upper(),
-        
+                    recipient[2:].upper(),
-        user_file = 'new/{}/{}/{}.json'.format(
+                    )
-                recipient[2:4].upper(),
+            filepath = os.path.join(self.user_dir, user_file)
-                recipient[4:6].upper(),
+            o = None
-                recipient[2:].upper(),
+            try:
-                )
+                f = open(filepath, 'r')
-        filepath = os.path.join(self.user_dir, user_file)
+                o = json.load(f)
-        o = None
+                f.close()
-        try:
+            except FileNotFoundError:
-            f = open(filepath, 'r')
+                logg.error('no import record of address {}'.format(recipient))
-            o = json.load(f)
+                return
+            u = Person.deserialize(o)
+            original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
+            try:
+                balance = self.balances[original_address]
+            except KeyError as e:
+                logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
+                return
+
+            # TODO: store token object in handler ,get decimals from there
+            multiplier = 10**6
+            balance_full = balance * multiplier
+            logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
+
+            (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
+            logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
+            r = conn.do(o)
+
+            tx_path = os.path.join(
+                    user_dir,
+                    'txs',
+                    strip_0x(tx_hash_hex),
+                    )
+            f = open(tx_path, 'w')
+            f.write(strip_0x(o['params'][0]))
             f.close()
-        except FileNotFoundError:
+#        except TypeError as e:
-            logg.error('no import record of address {}'.format(recipient))
+#            logg.warning('typerror {}'.format(e))
-            return
+#            pass
-        u = Person.deserialize(o)
+#        except IndexError as e:
-        original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
+#            logg.warning('indexerror {}'.format(e))
-        try:
+#            pass
-            balance = self.balances[original_address]
+#        except EthException as e:
-        except KeyError as e:
+#            logg.error('send error {}'.format(e).ljust(200))
-            logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
+        #except KeyError as e:
-            return
+        #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
-
-        # TODO: store token object in handler ,get decimals from there
-        erc20 = ERC20(self.chain_spec, signer=self.signer, gas_oracle=self.gas_oracle, nonce_oracle=self.nonce_oracle)
-        o = erc20.decimals(self.token_address)
-        r = conn.do(o)
-        decimals = erc20.parse_decimals(r)
-        multiplier = 10 ** decimals
-        balance_full = balance * multiplier
-        logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
-        (tx_hash_hex, o) = erc20.transfer(self.token_address, self.signer_address, recipient, balance_full)
-        logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
-        r = conn.do(o)
-
-        tx_path = os.path.join(
-                user_dir,
-                'txs',
-                strip_0x(tx_hash_hex),
-                )
-        f = open(tx_path, 'w')
-        f.write(strip_0x(o['params'][0]))
-        f.close()
-
-        faucet = Faucet(self.chain_spec, signer=self.signer, gas_oracle=self.gas_oracle, nonce_oracle=self.nonce_oracle)
-        (tx_hash, o) = faucet.give_to(self.faucet_address, self.signer_address, recipient)
-        r = conn.do(o)
 
 
 def progress_callback(block_number, tx_index):
@@ -207,31 +196,45 @@ def main():
     nonce_oracle = RPCNonceOracle(signer_address, conn)
 
     # Get Token registry address
-    registry = Registry(chain_spec)
+    txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
-    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
+    tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
+
+    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
+    data = add_0x(registry_addressof_method)
+    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
+    txf.set_code(tx, data)
+   
+    j = JSONRPCRequest()
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    token_index_address = registry.parse_address_of(r)
+    token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
-    token_index_address = to_checksum_address(token_index_address)
     logg.info('found token index address {}'.format(token_index_address))
-    
+
-    # Get Faucet address
-    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'Faucet')
-    r = conn.do(o)
-    faucet_address = registry.parse_address_of(r)
-    faucet_address = to_checksum_address(faucet_address)
-    logg.info('found faucet {}'.format(faucet_address))
 
     # Get Sarafu token address
-    token_index = TokenUniqueSymbolIndex(chain_spec)
+    tx = txf.template(signer_address, token_index_address)
-    o = token_index.address_of(token_index_address, token_symbol)
+    data = add_0x(registry_addressof_method)
+    h = hashlib.new('sha256')
+    h.update(token_symbol.encode('utf-8'))
+    z = h.digest()
+    data += eth_abi.encode_single('bytes32', z).hex()
+    txf.set_code(tx, data)
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    token_address = token_index.parse_address_of(r)
     try:
-        token_address = to_checksum_address(token_address)
+        sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     except ValueError as e:
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
-    logg.info('found token address {}'.format(token_address))
+    logg.info('found token address {}'.format(sarafu_token_address))
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -239,6 +242,22 @@ def main():
         o = block_latest()
         r = conn.do(o)
         block_offset = int(strip_0x(r), 16) + 1
+#
+#    addresses = {}
+#    f = open('{}/addresses.csv'.format(user_dir, 'r'))
+#    while True:
+#        l = f.readline()
+#        if l == None:
+#            break
+#        r = l.split(',')
+#        try:
+#            k = r[0]
+#            v = r[1].rstrip()
+#            addresses[k] = v
+#            sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
+#        except IndexError as e:
+#            break
+#    f.close()
 
     # TODO get decimals from token
     balances = {}
@@ -263,7 +282,7 @@ def main():
 
     syncer_backend.set(block_offset, 0)
     syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
-    handler = Handler(conn, chain_spec, user_dir, balances, token_address, faucet_address, signer_address, signer, gas_oracle, nonce_oracle)
+    handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)
     

apps/data-seeding/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "dependencies": {
         "@cicnet/cic-client-meta": "^0.0.11",
-        "@cicnet/crdt-meta": "^0.0.12",
+        "@cicnet/crdt-meta": "^0.0.10",
         "vcard-parser": "^1.0.0"
       }
     },
@@ -34,7 +34,7 @@
         "node": ">=14.16.1"
       }
     },
-    "node_modules/@cicnet/cic-client-meta/node_modules/@cicnet/crdt-meta": {
+    "node_modules/@cicnet/crdt-meta": {
       "version": "0.0.10",
       "resolved": "https://registry.npmjs.org/@cicnet/crdt-meta/-/crdt-meta-0.0.10.tgz",
       "integrity": "sha512-f+H6BQA2tE718KuNYiNzrDJN4wY00zeuhXM6aPKJUX6nryzX9g2r0yf8iDhkz+Fts1R6M7Riz73MfFEa8fgvsw==",
@@ -48,36 +48,22 @@
         "node": ">=14.16.1"
       }
     },
-    "node_modules/@cicnet/crdt-meta": {
-      "version": "0.0.12",
-      "resolved": "https://registry.npmjs.org/@cicnet/crdt-meta/-/crdt-meta-0.0.12.tgz",
-      "integrity": "sha512-wPX86P1Lsq4RxkVUlhlouhLkMOtkqzHgpNuXicHvWuhH3ks2Nsg7yqvTw9yt+kqj+N8a5pPMrNhKvUEFW8rJjA==",
-      "dependencies": {
-        "automerge": "^0.14.2",
-        "ini": "^1.3.8",
-        "openpgp": "^4.10.8",
-        "readline-sync": "^1.4.10"
-      },
-      "engines": {
-        "node": ">=14.16.1"
-      }
-    },
     "node_modules/@ethereumjs/common": {
-      "version": "2.4.0",
+      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.3.0.tgz",
-      "integrity": "sha512-UdkhFWzWcJCZVsj1O/H8/oqj/0RVYjLc1OhPjBrQdALAkQHpCp8xXI4WLnuGTADqTdJZww0NtgwG+TRPkXt27w==",
+      "integrity": "sha512-Fmi15MdVptsC85n6NcUXIFiiXCXWEfZNgPWP+OGAQOC6ZtdzoNawtxH/cYpIgEgSuIzfOeX3VKQP/qVI1wISHg==",
       "dependencies": {
         "crc-32": "^1.2.0",
-        "ethereumjs-util": "^7.1.0"
+        "ethereumjs-util": "^7.0.10"
       }
     },
     "node_modules/@ethereumjs/tx": {
-      "version": "3.3.0",
+      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.2.0.tgz",
-      "integrity": "sha512-yTwEj2lVzSMgE6Hjw9Oa1DZks/nKTWM8Wn4ykDNapBPua2f4nXO3qKnni86O6lgDj5fVNRqbDsD0yy7/XNGDEA==",
+      "integrity": "sha512-D3X/XtZ3ldUg34hr99Jvj7NxW3NxVKdUKrwQnEWlAp4CmCQpvYoyn7NF4lk34rHEt7ScS+Agu01pcDHoOcd19A==",
       "dependencies": {
-        "@ethereumjs/common": "^2.4.0",
+        "@ethereumjs/common": "^2.3.0",
-        "ethereumjs-util": "^7.1.0"
+        "ethereumjs-util": "^7.0.10"
       }
     },
     "node_modules/@types/bn.js": {
@@ -89,9 +75,9 @@
       }
     },
     "node_modules/@types/node": {
-      "version": "16.3.2",
+      "version": "15.12.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.2.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz",
-      "integrity": "sha512-jJs9ErFLP403I+hMLGnqDRWT0RYKSvArxuBVh2veudHV7ifEC1WAmjJADacZ7mRbA2nWgHtn8xyECMAot0SkAw=="
+      "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw=="
     },
     "node_modules/@types/pbkdf2": {
       "version": "3.1.0",
@@ -102,9 +88,9 @@
       }
     },
     "node_modules/@types/secp256k1": {
-      "version": "4.0.3",
+      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/@types/secp256k1/-/secp256k1-4.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/secp256k1/-/secp256k1-4.0.2.tgz",
-      "integrity": "sha512-Da66lEIFeIz9ltsdMZcpQvmrmmoqrfju8pm1BH8WbYjZSwUgCwXLb9C+9XYogwBITnbsSaMdVPb2ekf7TV+03w==",
+      "integrity": "sha512-QMg+9v0bbNJ2peLuHRWxzmy0HRJIG6gFZNhaRSp7S3ggSbCCxiqQB2/ybvhXyhHOCequpNkrx7OavNhrWOsW0A==",
       "dependencies": {
         "@types/node": "*"
       }
@@ -287,9 +273,9 @@
       }
     },
     "node_modules/blakejs": {
-      "version": "1.1.1",
+      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/blakejs/-/blakejs-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/blakejs/-/blakejs-1.1.0.tgz",
-      "integrity": "sha512-bLG6PHOCZJKNshTjGRBvET0vTciwQE6zFKOKKXPDJfwFBd4Ac0yBfPZqcGvGJap50l7ktvlpFqc2jGVaUgbJgg=="
+      "integrity": "sha1-ad+S75U6qIylGjLfarHFShVfx6U="
     },
     "node_modules/block-stream": {
       "version": "0.0.9",
@@ -653,9 +639,9 @@
       }
     },
     "node_modules/ethereumjs-util": {
-      "version": "7.1.0",
+      "version": "7.0.10",
-      "resolved": "https://registry.npmjs.org/ethereumjs-util/-/ethereumjs-util-7.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/ethereumjs-util/-/ethereumjs-util-7.0.10.tgz",
-      "integrity": "sha512-kR+vhu++mUDARrsMMhsjjzPduRVAeundLGXucGRHF3B4oEltOUspfgCVco4kckucj3FMlLaZHUl9n7/kdmr6Tw==",
+      "integrity": "sha512-c/xThw6A+EAnej5Xk5kOzFzyoSnw0WX0tSlZ6pAsfGVvQj3TItaDg9b1+Fz1RJXA+y2YksKwQnuzgt1eY6LKzw==",
       "dependencies": {
         "@types/bn.js": "^5.1.0",
         "bn.js": "^5.1.2",
@@ -1162,9 +1148,9 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/needle": {
-      "version": "2.8.0",
+      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/needle/-/needle-2.8.0.tgz",
+      "resolved": "https://registry.npmjs.org/needle/-/needle-2.6.0.tgz",
-      "integrity": "sha512-ZTq6WYkN/3782H1393me3utVYdq2XyqNUFBsprEE3VMAT0+hP/cItpnITpqsY6ep2yeFE4Tqtqwc74VqUlUYtw==",
+      "integrity": "sha512-KKYdza4heMsEfSWD7VPUIz3zX2XDwOyX2d+geb4vrERZMT5RMU6ujjaD+I5Yr54uZxQ2w6XRTAhHBbSCyovZBg==",
       "dependencies": {
         "debug": "^3.2.6",
         "iconv-lite": "^0.4.4",
@@ -2139,9 +2125,9 @@
       }
     },
     "node_modules/yargs-parser": {
-      "version": "20.2.9",
+      "version": "20.2.7",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
-      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
+      "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==",
       "engines": {
         "node": ">=10"
       }
@@ -2203,25 +2189,12 @@
         "pg": "^8.4.2",
         "sqlite3": "^5.0.0",
         "yargs": "^16.1.0"
-      },
-      "dependencies": {
-        "@cicnet/crdt-meta": {
-          "version": "0.0.10",
-          "resolved": "https://registry.npmjs.org/@cicnet/crdt-meta/-/crdt-meta-0.0.10.tgz",
-          "integrity": "sha512-f+H6BQA2tE718KuNYiNzrDJN4wY00zeuhXM6aPKJUX6nryzX9g2r0yf8iDhkz+Fts1R6M7Riz73MfFEa8fgvsw==",
-          "requires": {
-            "automerge": "^0.14.2",
-            "ini": "^1.3.8",
-            "openpgp": "^4.10.8",
-            "readline-sync": "^1.4.10"
-          }
-        }
       }
     },
     "@cicnet/crdt-meta": {
-      "version": "0.0.12",
+      "version": "0.0.10",
-      "resolved": "https://registry.npmjs.org/@cicnet/crdt-meta/-/crdt-meta-0.0.12.tgz",
+      "resolved": "https://registry.npmjs.org/@cicnet/crdt-meta/-/crdt-meta-0.0.10.tgz",
-      "integrity": "sha512-wPX86P1Lsq4RxkVUlhlouhLkMOtkqzHgpNuXicHvWuhH3ks2Nsg7yqvTw9yt+kqj+N8a5pPMrNhKvUEFW8rJjA==",
+      "integrity": "sha512-f+H6BQA2tE718KuNYiNzrDJN4wY00zeuhXM6aPKJUX6nryzX9g2r0yf8iDhkz+Fts1R6M7Riz73MfFEa8fgvsw==",
       "requires": {
         "automerge": "^0.14.2",
         "ini": "^1.3.8",
@@ -2230,21 +2203,21 @@
       }
     },
     "@ethereumjs/common": {
-      "version": "2.4.0",
+      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.3.0.tgz",
-      "integrity": "sha512-UdkhFWzWcJCZVsj1O/H8/oqj/0RVYjLc1OhPjBrQdALAkQHpCp8xXI4WLnuGTADqTdJZww0NtgwG+TRPkXt27w==",
+      "integrity": "sha512-Fmi15MdVptsC85n6NcUXIFiiXCXWEfZNgPWP+OGAQOC6ZtdzoNawtxH/cYpIgEgSuIzfOeX3VKQP/qVI1wISHg==",
       "requires": {
         "crc-32": "^1.2.0",
-        "ethereumjs-util": "^7.1.0"
+        "ethereumjs-util": "^7.0.10"
       }
     },
     "@ethereumjs/tx": {
-      "version": "3.3.0",
+      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.2.0.tgz",
-      "integrity": "sha512-yTwEj2lVzSMgE6Hjw9Oa1DZks/nKTWM8Wn4ykDNapBPua2f4nXO3qKnni86O6lgDj5fVNRqbDsD0yy7/XNGDEA==",
+      "integrity": "sha512-D3X/XtZ3ldUg34hr99Jvj7NxW3NxVKdUKrwQnEWlAp4CmCQpvYoyn7NF4lk34rHEt7ScS+Agu01pcDHoOcd19A==",
       "requires": {
-        "@ethereumjs/common": "^2.4.0",
+        "@ethereumjs/common": "^2.3.0",
-        "ethereumjs-util": "^7.1.0"
+        "ethereumjs-util": "^7.0.10"
       }
     },
     "@types/bn.js": {
@@ -2256,9 +2229,9 @@
       }
     },
     "@types/node": {
-      "version": "16.3.2",
+      "version": "15.12.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.2.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz",
-      "integrity": "sha512-jJs9ErFLP403I+hMLGnqDRWT0RYKSvArxuBVh2veudHV7ifEC1WAmjJADacZ7mRbA2nWgHtn8xyECMAot0SkAw=="
+      "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw=="
     },
     "@types/pbkdf2": {
       "version": "3.1.0",
@@ -2269,9 +2242,9 @@
       }
     },
     "@types/secp256k1": {
-      "version": "4.0.3",
+      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/@types/secp256k1/-/secp256k1-4.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/secp256k1/-/secp256k1-4.0.2.tgz",
-      "integrity": "sha512-Da66lEIFeIz9ltsdMZcpQvmrmmoqrfju8pm1BH8WbYjZSwUgCwXLb9C+9XYogwBITnbsSaMdVPb2ekf7TV+03w==",
+      "integrity": "sha512-QMg+9v0bbNJ2peLuHRWxzmy0HRJIG6gFZNhaRSp7S3ggSbCCxiqQB2/ybvhXyhHOCequpNkrx7OavNhrWOsW0A==",
       "requires": {
         "@types/node": "*"
       }
@@ -2439,9 +2412,9 @@
       }
     },
     "blakejs": {
-      "version": "1.1.1",
+      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/blakejs/-/blakejs-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/blakejs/-/blakejs-1.1.0.tgz",
-      "integrity": "sha512-bLG6PHOCZJKNshTjGRBvET0vTciwQE6zFKOKKXPDJfwFBd4Ac0yBfPZqcGvGJap50l7ktvlpFqc2jGVaUgbJgg=="
+      "integrity": "sha1-ad+S75U6qIylGjLfarHFShVfx6U="
     },
     "block-stream": {
       "version": "0.0.9",
@@ -2755,9 +2728,9 @@
       }
     },
     "ethereumjs-util": {
-      "version": "7.1.0",
+      "version": "7.0.10",
-      "resolved": "https://registry.npmjs.org/ethereumjs-util/-/ethereumjs-util-7.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/ethereumjs-util/-/ethereumjs-util-7.0.10.tgz",
-      "integrity": "sha512-kR+vhu++mUDARrsMMhsjjzPduRVAeundLGXucGRHF3B4oEltOUspfgCVco4kckucj3FMlLaZHUl9n7/kdmr6Tw==",
+      "integrity": "sha512-c/xThw6A+EAnej5Xk5kOzFzyoSnw0WX0tSlZ6pAsfGVvQj3TItaDg9b1+Fz1RJXA+y2YksKwQnuzgt1eY6LKzw==",
       "requires": {
         "@types/bn.js": "^5.1.0",
         "bn.js": "^5.1.2",
@@ -3184,9 +3157,9 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "needle": {
-      "version": "2.8.0",
+      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/needle/-/needle-2.8.0.tgz",
+      "resolved": "https://registry.npmjs.org/needle/-/needle-2.6.0.tgz",
-      "integrity": "sha512-ZTq6WYkN/3782H1393me3utVYdq2XyqNUFBsprEE3VMAT0+hP/cItpnITpqsY6ep2yeFE4Tqtqwc74VqUlUYtw==",
+      "integrity": "sha512-KKYdza4heMsEfSWD7VPUIz3zX2XDwOyX2d+geb4vrERZMT5RMU6ujjaD+I5Yr54uZxQ2w6XRTAhHBbSCyovZBg==",
       "requires": {
         "debug": "^3.2.6",
         "iconv-lite": "^0.4.4",
@@ -3971,9 +3944,9 @@
       }
     },
     "yargs-parser": {
-      "version": "20.2.9",
+      "version": "20.2.7",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
-      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w=="
+      "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw=="
     }
   }
 }

apps/data-seeding/package.json

@@ -1,7 +1,7 @@
 {
   "dependencies": {
     "@cicnet/cic-client-meta": "^0.0.11",
-    "@cicnet/crdt-meta": "^0.0.12",
+    "@cicnet/crdt-meta": "^0.0.10",
     "vcard-parser": "^1.0.0"
   }
 }

apps/data-seeding/requirements.txt

@@ -1,13 +1,4 @@
-sarafu-faucet==0.0.4a3
+sarafu-faucet==0.0.4a1
-cic-eth[tools]==0.12.1a1
+cic-eth[tools]==0.12.0a1
 cic-types==0.1.0a13
-crypto-dev-signer==0.4.14b7
+crypto-dev-signer==0.4.14b6
-faker==4.17.1
-chainsyncer~=0.0.3a3
-chainlib-eth~=0.0.5a1
-eth-address-index~=0.1.2a1
-eth-contract-registry~=0.5.6a1
-eth-accounts-index~=0.0.12a1
-eth-erc20~=0.0.10a3
-erc20-faucet~=0.2.2a1
-psycopg2==2.8.6

apps/data-seeding/verify.py

@@ -14,6 +14,7 @@ import urllib.parse
 # external imports
 import celery
 import confini
+import eth_abi
 from chainlib.chain import ChainSpec
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.connection import EthHTTPConnection
@@ -32,9 +33,6 @@ from cic_types.models.person import (
 from erc20_faucet import Faucet
 from eth_erc20 import ERC20
 from hexathon.parse import strip_0x, add_0x
-from eth_contract_registry import Registry
-from eth_accounts_index import AccountsIndex
-from eth_token_index import TokenUniqueSymbolIndex
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -45,8 +43,7 @@ custodial_tests = [
         'local_key',
         'gas',
         'faucet',
-        'ussd',
+        'ussd'
-        'ussd_pins',
         ]
 
 metadata_tests = [
@@ -74,7 +71,6 @@ argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spe
 argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
 argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
-argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
 argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
@@ -134,11 +130,6 @@ if args.skip_custodial:
     for t in custodial_tests:
         if t not in exclude:
             exclude.append(t)
-if args.skip_metadata:
-    logg.info('will skip all metadata verifications ({})'.format(','.join(metadata_tests)))
-    for t in metadata_tests:
-        if t not in exclude:
-            exclude.append(t)
 for t in include:
     if t not in all_tests:
         raise ValueError('Cannot include unknown verification "{}"'.format(t))
@@ -149,7 +140,7 @@ for t in include:
 api = None
 for t in custodial_tests:
     if t in active_tests:
-        from cic_eth.api.admin import AdminApi
+        from cic_eth.api.api_admin import AdminApi
         api = AdminApi(None)
         logg.info('activating custodial module'.format(t))
         break
@@ -272,11 +263,19 @@ class Verifier:
 
 
     def verify_accounts_index(self, address, balance=None):
-        accounts_index = AccountsIndex(self.chain_spec)
+        tx = self.tx_factory.template(ZERO_ADDRESS, self.index_address)
-        o = accounts_index.have(self.index_address, address)
+        data = keccak256_string_to_hex('have(address)')[:8]
+        data += eth_abi.encode_single('address', address).hex()
+        tx = self.tx_factory.set_code(tx, data)
+        tx = self.tx_factory.normalize(tx)
+        j = JSONRPCRequest()
+        o = j.template()
+        o['method'] = 'eth_call'
+        o['params'].append(tx)
+        o = j.finalize(o)
         r = self.conn.do(o)
-        n = accounts_index.parse_have(r)
+        logg.debug('index check for {}: {}'.format(address, r))
-        logg.debug('index check for {}: {}'.format(address, n))
+        n = eth_abi.decode_single('uint256', bytes.fromhex(strip_0x(r)))
         if n != 1:
             raise VerifierError(n, 'accounts index')
 
@@ -290,7 +289,7 @@ class Verifier:
             actual_balance = int(r)
         balance = int(balance / 1000000) * 1000000
         balance += self.faucet_amount
-        logg.info('balance for {}: {}'.format(address, balance))
+        logg.debug('balance for {}: {}'.format(address, balance))
         if balance != actual_balance:
             raise VerifierError((actual_balance, balance), 'balance')
 
@@ -428,39 +427,70 @@ def main():
     gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
 
     # Get Token registry address
-    registry = Registry(chain_spec)
+    txf = TxFactory(chain_spec, signer=None, gas_oracle=gas_oracle, nonce_oracle=None)
-    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
+    tx = txf.template(ZERO_ADDRESS, config.get('CIC_REGISTRY_ADDRESS'))
+
+    # TODO: replace with cic-eth-registry
+    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
+    data = add_0x(registry_addressof_method)
+    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
+    txf.set_code(tx, data)
+    
+    j = JSONRPCRequest()
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    token_index_address = registry.parse_address_of(r)
+    token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
-    token_index_address = to_checksum_address(token_index_address)
     logg.info('found token index address {}'.format(token_index_address))
 
-    # Get Account registry address
+    data = add_0x(registry_addressof_method)
-    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'AccountRegistry')
+    data += eth_abi.encode_single('bytes32', b'AccountRegistry').hex()
+    txf.set_code(tx, data)
+    
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    account_index_address = registry.parse_address_of(r)
+    account_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
-    account_index_address = to_checksum_address(account_index_address)
     logg.info('found account index address {}'.format(account_index_address))
 
-    # Get Faucet address
+    data = add_0x(registry_addressof_method)
-    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'Faucet')
+    data += eth_abi.encode_single('bytes32', b'Faucet').hex()
+    txf.set_code(tx, data)
+    
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    faucet_address = registry.parse_address_of(r)
+    faucet_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
-    faucet_index_address = to_checksum_address(token_index_address)
     logg.info('found faucet {}'.format(faucet_address))
 
-# Get Sarafu token address
+
-    token_index = TokenUniqueSymbolIndex(chain_spec)
+
-    o = token_index.address_of(token_index_address, token_symbol)
+    # Get Sarafu token address
+    tx = txf.template(ZERO_ADDRESS, token_index_address)
+    data = add_0x(registry_addressof_method)
+    h = hashlib.new('sha256')
+    h.update(token_symbol.encode('utf-8'))
+    z = h.digest()
+    data += eth_abi.encode_single('bytes32', z).hex()
+    txf.set_code(tx, data)
+    o = j.template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
-    token_address = token_index.parse_address_of(r)
+    sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
-    try:
+    logg.info('found token address {}'.format(sarafu_token_address))
-        token_address = to_checksum_address(token_address)
+
-    except ValueError as e:
-        logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
-        sys.exit(1)
-    logg.info('found token address {}'.format(token_address))
- 
     balances = {}
     f = open('{}/balances.csv'.format(user_dir, 'r'))
     i = 0
@@ -481,7 +511,7 @@ def main():
 
     f.close()
 
-    verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, token_address, faucet_address, user_dir, exit_on_error)
+    verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, sarafu_token_address, faucet_address, user_dir, exit_on_error)
 
     user_new_dir = os.path.join(user_dir, 'new')
     i = 0

docker-compose.yml

@@ -103,14 +103,14 @@ services:
       DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
       #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
       DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
-      CIC_DEFAULT_TOKEN_SYMBOL: $CIC_DEFAULT_TOKEN_SYMBOL
+      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
       TOKEN_NAME: $TOKEN_NAME
       TOKEN_DECIMALS: $TOKEN_DECIMALS
       TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
       TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
       TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
       TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
-      TOKEN_TYPE: $TOKEN_TYPE
+      TOKEN_TYPE: ${TOKEN_TYPE:-default}
         #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
     command: ["./run_job.sh"]
         #command: ["./reset.sh"]
@@ -443,7 +443,7 @@ services:
       SERVER_HOST: localhost
       SERVER_PORT: 8000
       DATABASE_SCHEMA_SQL_PATH: ""
-      PGP_EXPORTS_DIR: /root/tests/
+      PGP_EXPORTS_DIR: /tmp/src/cic-meta/tests/
       PGP_PRIVATEKEY_FILE: privatekeys.asc
       PGP_PASSPHRASE: merman
       PGP_PUBLICKEY_TRUSTED_FILE: publickeys.asc