Add phone pointer and ussd import verifiers

.gitignore

@@ -1,10 +1,2 @@
 service-configs/*
 !service-configs/.gitkeep
-**/node_modules/
-__pycache__
-*.pyc
-*.o
-gmon.out
-*.egg-info
-dist/
-build/

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -67,7 +67,6 @@ class ERC20TransferFilter(SyncFilter):
                 tx.status == Status.SUCCESS,
                 block.timestamp,
                 )
-        #db_session.flush()
-        db_session.commit()
+        db_session.flush()
 
         return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -26,10 +26,9 @@ from chainlib.eth.block import (
 from hexathon import (
         strip_0x,
         )
-from chainsyncer.backend.sql import SQLBackend
+from chainsyncer.backend import SyncerBackend
 from chainsyncer.driver import (
         HeadSyncer,
-        HistorySyncer,
         )
 from chainsyncer.db.models.base import SessionBase
 
@@ -71,21 +70,19 @@ def main():
 
     syncers = []
 
-    #if SQLBackend.first(chain_spec):
-    #    backend = SQLBackend.initial(chain_spec, block_offset)
-    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
+    #if SyncerBackend.first(chain_spec):
+    #    backend = SyncerBackend.initial(chain_spec, block_offset)
+    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
         logg.info('found no backends to resume')
-        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
+        syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
 
-    for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend))
+    syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1))
 
-    syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
     syncers.append(HeadSyncer(syncer_backend))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')

apps/cic-cache/docker/Dockerfile

@@ -17,7 +17,7 @@ RUN apt-get update && \
 
 # Copy shared requirements from top of mono-repo
 RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a58
 
 COPY cic-cache/requirements.txt ./
 COPY cic-cache/setup.cfg \
@@ -47,9 +47,6 @@ RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
 	mkdir -p /usr/local/share/cic/solidity && \
 	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
 
-COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
-COPY cic-cache/docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server

apps/cic-cache/docker/db.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
->&2 echo executing database migration
-python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
-set +e

apps/cic-cache/docker/start_tracker.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-. ./db.sh
-
-if [ $? -ne "0" ]; then
-	>&2 echo db migrate fail
-	exit 1
-fi
-
-/usr/local/bin/cic-cache-trackerd $@

apps/cic-cache/requirements.txt

@@ -1,12 +1,13 @@
-cic-base~=0.1.2a77
+cic-base~=0.1.2a62
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.4a16
+cic-eth-registry~=0.5.4a12
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a2
+chainlib~=0.0.2a5
+chainsyncer~=0.0.1a21

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -2,7 +2,7 @@
 import datetime
 import logging
 
-# external imports
+# third-party imports
 import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
@@ -32,9 +32,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
     logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
@@ -53,9 +51,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, flags, address=address)
     logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
@@ -131,9 +127,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
     r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
     if address != None:
@@ -145,9 +139,3 @@ def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
     session.flush()
     session.close()
     return chained_input
-
-
-@celery_app.task()
-def shutdown(message):
-    logg.critical('shutdown called: {}'.format(message))
-    celery_app.control.shutdown() #broadcast('shutdown')

apps/cic-eth/cic_eth/admin/token.py

@@ -1,19 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-# local imports
-from cic_eth.task import BaseTask
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-            'symbol': self.default_token_symbol,
-            'address': self.default_token_address,
-            }

apps/cic-eth/cic_eth/api/api_admin.py

@@ -60,29 +60,6 @@ class AdminApi:
         self.call_address = call_address
 
 
-    def proxy_do(self, chain_spec, o):
-        s_proxy = celery.signature(
-                'cic_eth.task.rpc_proxy',
-                [
-                    chain_spec.asdict(),
-                    o,
-                    'default',
-                    ],
-                queue=self.queue
-                )
-        return s_proxy.apply_async()
-
-
-    
-    def registry(self):
-        s_registry = celery.signature(
-                'cic_eth.task.registry',
-                [],
-                queue=self.queue
-                )
-        return s_registry.apply_async()
-
-
     def unlock(self, chain_spec, address, flags=None):
         s_unlock = celery.signature(
             'cic_eth.admin.ctrl.unlock',
@@ -169,6 +146,7 @@ class AdminApi:
 
         # TODO: This check should most likely be in resend task itself
         tx_dict = s_get_tx_cache.apply_async().get()
+        #if tx_dict['status'] in [StatusEnum.REVERTED, StatusEnum.SUCCESS, StatusEnum.CANCELLED, StatusEnum.OBSOLETED]: 
         if not is_alive(getattr(StatusEnum, tx_dict['status']).value):
             raise TxStateChangeError('Cannot resend mined or obsoleted transaction'.format(txold_hash_hex))
         
@@ -248,6 +226,9 @@ class AdminApi:
                 break
             last_nonce = nonce_otx
 
+        #nonce_cache = Nonce.get(address)
+        #nonce_w3 = self.w3.eth.getTransactionCount(address, 'pending') 
+        
         return {
             'nonce': {
                 #'network': nonce_cache,
@@ -291,6 +272,20 @@ class AdminApi:
         return s_nonce.apply_async()
 
 
+#    # TODO: this is a stub, complete all checks
+#    def ready(self):
+#        """Checks whether all required initializations have been performed.
+#
+#        :raises cic_eth.error.InitializationError: At least one setting pre-requisite has not been met.
+#        :raises KeyError: An address provided for initialization is not known by the keystore.
+#        """
+#        addr = AccountRole.get_address('ETH_GAS_PROVIDER_ADDRESS')
+#        if addr == ZERO_ADDRESS:
+#            raise InitializationError('missing account ETH_GAS_PROVIDER_ADDRESS')
+#
+#        self.w3.eth.sign(addr, text='666f6f')
+
+
     def account(self, chain_spec, address, include_sender=True, include_recipient=True, renderer=None, w=sys.stdout):
         """Lists locally originated transactions for the given Ethereum address.
 
@@ -353,7 +348,6 @@ class AdminApi:
 
 
     # TODO: Add exception upon non-existent tx aswell as invalid tx data to docstring 
-    # TODO: This method is WAY too long
     def tx(self, chain_spec, tx_hash=None, tx_raw=None, registry=None, renderer=None, w=sys.stdout):
         """Output local and network details about a given transaction with local origin.
 
@@ -376,6 +370,7 @@ class AdminApi:
 
         if tx_raw != None:
             tx_hash = add_0x(keccak256_hex_to_hex(tx_raw))
+            #tx_hash = self.w3.keccak(hexstr=tx_raw).hex()
 
         s = celery.signature(
             'cic_eth.queue.query.get_tx_cache',
@@ -391,78 +386,38 @@ class AdminApi:
   
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
-            if registry != None:
-                try:
-                    source_token = registry.by_address(tx['source_token'])
-                except UnknownContractError:
-                    logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['source_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                source_token = t.get()
-                if source_token == None:
-                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
-
+            try:
+                source_token = registry.by_address(tx['source_token'])
+                #source_token = CICRegistry.get_address(chain_spec, tx['source_token']).contract
+            except UnknownContractError:
+                #source_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
+                #source_token = CICRegistry.add_token(chain_spec, source_token_contract)
+                logg.warning('unknown source token contract {}'.format(tx['source_token']))
 
         destination_token = None
-        if tx['destination_token'] != ZERO_ADDRESS:
-            if registry != None:
-                try:
-                    destination_token = registry.by_address(tx['destination_token'])
-                except UnknownContractError:
-                    logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['destination_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                destination_token = t.get()
-                if destination_token == None:
-                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
-
+        if tx['source_token'] != ZERO_ADDRESS:
+            try:
+                #destination_token = CICRegistry.get_address(chain_spec, tx['destination_token'])
+                destination_token = registry.by_address(tx['destination_token'])
+            except UnknownContractError:
+                #destination_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
+                #destination_token = CICRegistry.add_token(chain_spec, destination_token_contract)
+                logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
 
         o = code(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
-                try:
-                    sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
-                    tx['sender_description'] = 'Contract at {}'.format(tx['sender'])
-                except UnknownContractError:
-                    tx['sender_description'] = 'Unknown contract'
-                except KeyError as e:
-                    tx['sender_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['sender'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['sender_description'] = t.get()
-                if tx['sender_description'] == None:
-                    tx['sender_description'] = 'Unknown contract'
-
-
+            try:
+                #sender_contract = CICRegistry.get_address(chain_spec, tx['sender'])
+                sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
+                tx['sender_description'] = 'Contract at {}'.format(tx['sender']) #sender_contract)
+            except UnknownContractError:
+                tx['sender_description'] = 'Unknown contract'
+            except KeyError as e:
+                tx['sender_description'] = 'Unknown contract'
         else:
             s = celery.signature(
                     'cic_eth.eth.account.have',
@@ -491,31 +446,16 @@ class AdminApi:
                     tx['sender_description'] = role
 
         o = code(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
-                try:
-                    recipient_contract = registry.by_address(tx['recipient'])
-                    tx['recipient_description'] = 'Contract at {}'.format(tx['recipient'])
-                except UnknownContractError as e:
-                    tx['recipient_description'] = 'Unknown contract'
-                except KeyError as e:
-                    tx['recipient_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['recipient'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['recipient_description'] = t.get()
-                if tx['recipient_description'] == None:
-                    tx['recipient_description'] = 'Unknown contract'
-
+            try:
+                #recipient_contract = CICRegistry.by_address(tx['recipient'])
+                recipient_contract = registry.by_address(tx['recipient'])
+                tx['recipient_description'] = 'Contract at {}'.format(tx['recipient']) #recipient_contract)
+            except UnknownContractError as e:
+                tx['recipient_description'] = 'Unknown contract'
+            except KeyError as e:
+                tx['recipient_description'] = 'Unknown contract'
         else:
             s = celery.signature(
                     'cic_eth.eth.account.have',
@@ -557,8 +497,7 @@ class AdminApi:
         r = None
         try:
             o = transaction(tx_hash)
-            t = self.proxy_do(chain_spec, o)
-            r = t.get()
+            r = self.rpc.do(o)
             if r != None:
                 tx['network_status'] = 'Mempool'
         except Exception as e:
@@ -567,8 +506,7 @@ class AdminApi:
         if r != None:
             try:
                 o = receipt(tx_hash)
-                t = self.proxy_do(chain_spec, o)
-                r = t.get()
+                r = self.rpc.do(o)
                 logg.debug('h {} o {}'.format(tx_hash, o))
                 if int(strip_0x(r['status'])) == 1:
                     tx['network_status'] = 'Confirmed'
@@ -583,13 +521,11 @@ class AdminApi:
                 pass
 
         o = balance(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         tx['sender_gas_balance'] = r
 
         o = balance(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
+        r = self.rpc.do(o)
         tx['recipient_gas_balance'] = r
 
         tx_unpacked = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)

apps/cic-eth/cic_eth/api/api_task.py

@@ -62,18 +62,6 @@ class Api:
                     )       
 
 
-    def default_token(self):
-        s_token = celery.signature(
-                'cic_eth.admin.token.default_token',
-                [],
-                queue=self.queue,
-                )
-        if self.callback_param != None:
-            s_token.link(self.callback_success)
-
-        return s_token.apply_async()
-
-
     def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
         """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 

apps/cic-eth/cic_eth/check/db.py

@@ -1,8 +0,0 @@
-from cic_eth.db.models.base import SessionBase
-
-
-def health(*args, **kwargs):
-    session = SessionBase.create_session()
-    session.execute('SELECT count(*) from alembic_version')
-    session.close()
-    return True

apps/cic-eth/cic_eth/check/gas.py

@@ -1,48 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.connection import RPCConnection
-from chainlib.chain import ChainSpec
-from chainlib.eth.gas import balance
-
-# local imports
-from cic_eth.db.models.role import AccountRole
-from cic_eth.db.models.base import SessionBase
-from cic_eth.db.enum import LockEnum
-from cic_eth.error import LockedError
-from cic_eth.admin.ctrl import check_lock
-
-logg = logging.getLogger().getChild(__name__)
-
-
-def health(*args, **kwargs):
-
-    session = SessionBase.create_session()
-
-    config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-    logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
-
-    try:
-        check_lock(None, None, LockEnum.INIT)
-    except LockedError:
-        logg.warning('INIT lock is set, skipping GAS GIFTER balance check.')
-        return True
-
-    gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
-    session.close()
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    o = balance(gas_provider)
-    r = rpc.do(o)
-    try: 
-        r = int(r, 16)
-    except TypeError:
-        r = int(r)
-    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
-    if r < gas_min:
-        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
-        return False
-
-    return True

apps/cic-eth/cic_eth/check/redis.py

@@ -1,18 +0,0 @@
-# external imports
-import redis
-import os
-
-
-def health(*args, **kwargs):
-    r = redis.Redis(
-            host=kwargs['config'].get('REDIS_HOST'),
-            port=kwargs['config'].get('REDIS_PORT'),
-            db=kwargs['config'].get('REDIS_DB'),
-            )
-    try:
-        r.set(kwargs['unit'], os.getpid())
-    except redis.connection.ConnectionError:
-        return False
-    except redis.connection.ResponseError:
-        return False
-    return True

apps/cic-eth/cic_eth/check/signer.py

@@ -1,37 +0,0 @@
-# standard imports
-import time
-import logging
-from urllib.error import URLError
-
-# external imports
-from chainlib.connection import RPCConnection
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.sign import sign_message
-from chainlib.error import JSONRPCException
-
-logg = logging.getLogger().getChild(__name__)
-
-
-def health(*args, **kwargs):
-    blocked = True
-    max_attempts = 5
-    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
-    for i in range(max_attempts):
-        idx = i + 1
-        logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))
-        try:
-            conn.do(sign_message(ZERO_ADDRESS, '0x2a'))
-        except FileNotFoundError:
-            pass
-        except ConnectionError:
-            pass
-        except URLError:
-            pass
-        except JSONRPCException:
-            logg.debug('signer connection succeeded')
-            return True
-
-        if idx < max_attempts:
-            time.sleep(0.5)
-
-    return False

apps/cic-eth/cic_eth/db/enum.py

@@ -74,11 +74,10 @@ class LockEnum(enum.IntEnum):
     QUEUE: Disable queueing new or modified transactions
     """
     STICKY=1
-    INIT=2
-    CREATE=4
-    SEND=8
-    QUEUE=16
-    QUERY=32
+    CREATE=2
+    SEND=4
+    QUEUE=8
+    QUERY=16
     ALL=int(0xfffffffffffffffe)
 
 

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -5,11 +5,8 @@ Revises: 1f1b3b641d08
 Create Date: 2021-04-02 18:41:20.864265
 
 """
-import datetime
 from alembic import op
 import sqlalchemy as sa
-from chainlib.eth.constant import ZERO_ADDRESS
-from cic_eth.db.enum import LockEnum
 
 
 # revision identifiers, used by Alembic.
@@ -26,11 +23,10 @@ def upgrade():
             sa.Column("address", sa.String(42), nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
-            sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
+            sa.Column("date_created", sa.DateTime, nullable=False),
             sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
             )
     op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
-    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
 
 
 def downgrade():

apps/cic-eth/cic_eth/error.py

@@ -48,8 +48,6 @@ class RoleMissingError(Exception):
     pass
 
 
-
-
 class IntegrityError(Exception):
     """Exception raised to signal irregularities with deduplication and ordering of tasks
 
@@ -64,19 +62,15 @@ class LockedError(Exception):
     pass
 
 
-class SeppukuError(Exception):
-    """Exception base class for all errors that should cause system shutdown
-
-    """
-
-
-class SignerError(SeppukuError):
+class SignerError(Exception):
     """Exception raised when signer is unavailable or generates an error
 
     """
     pass
 
 
-class RoleAgencyError(SeppukuError):
-    """Exception raise when a role cannot perform its function. This is a critical exception
+class EthError(Exception):
+    """Exception raised when unspecified error from evm node is encountered
+
     """
+    pass

apps/cic-eth/cic_eth/eth/account.py

@@ -4,10 +4,10 @@ import logging
 # external imports
 import celery
 from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import (
         strip_0x,
         )
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.connection import RPCConnection
 from chainlib.eth.sign import (
         new_account,
@@ -19,7 +19,6 @@ from chainlib.eth.tx import (
         unpack,
         )
 from chainlib.chain import ChainSpec
-from chainlib.error import JSONRPCException
 from eth_accounts_index import AccountRegistry
 from sarafu_faucet import MinterFaucet as Faucet
 from chainqueue.db.models.tx import TxCache
@@ -71,18 +70,11 @@ def create(self, password, chain_spec_dict):
     a = None
     conn = RPCConnection.connect(chain_spec, 'signer')
     o = new_account()
-    try:
-        a = conn.do(o)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
+    a = conn.do(o)
     conn.disconnect()
 
-    # TODO: It seems infeasible that a can be None in any case, verify
     if a == None:
         raise SignerError('create account')
-        
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -227,22 +219,21 @@ def have(self, account, chain_spec_dict):
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     o = sign_message(account, '0x2a')
-    conn = RPCConnection.connect(chain_spec, 'signer')
+    try:
+        conn = RPCConnection.connect(chain_spec, 'signer')
+    except Exception as e:
+        logg.debug('cannot sign with {}: {}'.format(account, e))
+        return None
 
     try:
         conn.do(o)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except JSONRPCException as e:
+        conn.disconnect()
+        return account
+    except Exception as e:
         logg.debug('cannot sign with {}: {}'.format(account, e))
         conn.disconnect()
         return None
    
-    conn.disconnect()
-    return account
-
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):

apps/cic-eth/cic_eth/eth/erc20.py

@@ -108,13 +108,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
-    
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
 
     rpc_signer.disconnect()
     rpc.disconnect()
@@ -177,12 +171,7 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
+    (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
 
     rpc_signer.disconnect()
     rpc.disconnect()

apps/cic-eth/cic_eth/eth/gas.py

@@ -328,12 +328,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
 
     # build and add transaction
     logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address))
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
     logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex))
     cache_task = 'cic_eth.eth.gas.cache_gas_data'
     register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
@@ -409,12 +404,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
     logg.debug('change gas price from old {} to new {} for tx {}'.format(tx['gasPrice'], new_gas_price, tx))
     tx['gasPrice'] = new_gas_price
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
+    (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
     queue_create(
         chain_spec,
         tx['nonce'],

apps/cic-eth/cic_eth/eth/meta.py

@@ -1,10 +1,6 @@
-# external imports
+# extended imports
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.status import Status as TxStatus
-from cic_eth_registry.erc20 import ERC20Token
-
-# local imports
-from cic_eth.ext.address import translate_address
 
 
 class ExtendedTx:
@@ -31,12 +27,12 @@ class ExtendedTx:
         self.status_code = TxStatus.PENDING.value
 
 
-    def set_actors(self, sender, recipient, trusted_declarator_addresses=None, caller_address=ZERO_ADDRESS):
+    def set_actors(self, sender, recipient, trusted_declarator_addresses=None):
         self.sender = sender
         self.recipient = recipient
         if trusted_declarator_addresses != None:
-            self.sender_label = translate_address(sender, trusted_declarator_addresses, self.chain_spec, sender_address=caller_address)
-            self.recipient_label = translate_address(recipient, trusted_declarator_addresses, self.chain_spec, sender_address=caller_address)
+            self.sender_label = translate_address(sender, trusted_declarator_addresses, self.chain_spec)
+            self.recipient_label = translate_address(recipient, trusted_declarator_addresses, self.chain_spec)
 
 
     def set_tokens(self, source, source_value, destination=None, destination_value=None):
@@ -44,8 +40,8 @@ class ExtendedTx:
             destination = source
         if destination_value == None:
             destination_value = source_value
-        st = ERC20Token(self.chain_spec, self.rpc, source)
-        dt = ERC20Token(self.chain_spec, self.rpc, destination)
+        st = ERC20Token(self.rpc, source)
+        dt = ERC20Token(self.rpc, destination)
         self.source_token = source
         self.source_token_symbol = st.symbol
         self.source_token_name = st.name
@@ -66,10 +62,10 @@ class ExtendedTx:
         self.status_code = n
 
 
-    def asdict(self):
+    def to_dict(self):
         o = {}
         for attr in dir(self):
-            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'set_status', 'asdict', 'rpc']:
+            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'set_status', 'to_dict']:
                 continue
             o[attr] = getattr(self, attr)
         return o 

apps/cic-eth/cic_eth/ext/tx.py

@@ -114,7 +114,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
-                        token = ERC20Token(chain_spec, rpc, tx['to'])
+                        token = ERC20Token(rpc, tx['to'])
                         token_symbol = token.symbol
                         token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)

apps/cic-eth/cic_eth/queue/time.py

@@ -12,7 +12,6 @@ from chainqueue.error import NotLocalTxError
 
 # local imports
 from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
-from cic_eth.db.models.base import SessionBase
 
 celery_app = celery.current_app
 

apps/cic-eth/cic_eth/registry.py

@@ -29,5 +29,5 @@ def connect(rpc, chain_spec, registry_address):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
     registry_address = registry.by_name('ContractRegistry')
+    
     return registry
-

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -23,6 +23,7 @@ default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-r', '--registry-address', type=str, help='CIC registry address')
 argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
@@ -58,7 +59,6 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         }
 # override args
-config.dict_override(args_override, 'cli')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@@ -67,9 +67,7 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
 
 queue = args.q
 
-chain_spec = None
-if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 admin_api = AdminApi(None)
 
 
@@ -84,9 +82,6 @@ def lock_names_to_flag(s):
 
 # TODO: move each command to submodule
 def main():
-    chain_spec_dict = None
-    if chain_spec != None:
-        chain_spec_dict = chain_spec.asdict()
     if args.command == 'unlock':
         flags = lock_names_to_flag(args.flags)
         if not is_checksum_address(args.address):
@@ -96,7 +91,7 @@ def main():
             'cic_eth.admin.ctrl.unlock',
             [
                 None,
-                chain_spec_dict,
+                chain_spec.asdict(),
                 args.address,
                 flags,
                 ],
@@ -115,7 +110,7 @@ def main():
             'cic_eth.admin.ctrl.lock',
             [
                 None,
-                chain_spec_dict,
+                chain_spec.asdict(),
                 args.address,
                 flags,
                 ],

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -15,6 +15,7 @@ from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
 from chainlib.connection import RPCConnection
+from chainsyncer.error import SyncDone
 from hexathon import strip_0x
 from chainqueue.db.enum import (
     StatusEnum,
@@ -152,7 +153,10 @@ class DispatchSyncer:
 def main(): 
     syncer = DispatchSyncer(chain_spec)
     conn = RPCConnection.connect(chain_spec, 'default')
-    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    try:
+        syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    except SyncDone as e:
+        sys.stderr.write("dispatcher done at block {}\n".format(e))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -1,7 +1,7 @@
 # standard imports
 import logging
 
-# external imports
+# third-party imports
 import celery
 from cic_eth_registry.error import UnknownContractError
 from chainlib.status import Status as TxStatus
@@ -9,13 +9,7 @@ from chainlib.eth.address import to_checksum_address
 from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.erc20 import ERC20
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-# TODO: use sarafu_Faucet for both when inheritance has been implemented
-from erc20_single_shot_faucet import SingleShotFaucet
-from sarafu_faucet import MinterFaucet as Faucet
+from hexathon import strip_0x
 
 # local imports
 from .base import SyncFilter
@@ -24,73 +18,65 @@ from cic_eth.eth.meta import ExtendedTx
 logg = logging.getLogger().getChild(__name__)
 
 
+def parse_transfer(tx):
+    r = ERC20.parse_transfer_request(tx.payload)
+    transfer_data = {}
+    transfer_data['to'] = r[0]
+    transfer_data['value'] = r[1]
+    transfer_data['from'] = tx['from']
+    transfer_data['token_address'] = tx['to']
+    return ('transfer', transfer_data)
+
+
+def parse_transferfrom(tx):
+    r = ERC20.parse_transfer_request(tx.payload)
+    transfer_data = unpack_transferfrom(tx.payload)
+    transfer_data['from'] = r[0]
+    transfer_data['to'] = r[1]
+    transfer_data['value'] = r[2]
+    transfer_data['token_address'] = tx['to']
+    return ('transferfrom', transfer_data)
+
+
+def parse_giftto(tx):
+    # TODO: broken
+    logg.error('broken')
+    return
+    transfer_data = unpack_gift(tx.payload)
+    transfer_data['from'] = tx.inputs[0]
+    transfer_data['value'] = 0
+    transfer_data['token_address'] = ZERO_ADDRESS
+    # TODO: would be better to query the gift amount from the block state
+    for l in tx.logs:
+        topics = l['topics']
+        logg.debug('topixx {}'.format(topics))
+        if strip_0x(topics[0]) == '45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
+            #transfer_data['value'] = web3.Web3.toInt(hexstr=strip_0x(l['data']))
+            transfer_data['value'] = int.from_bytes(bytes.fromhex(strip_0x(l_data)))
+            #token_address_bytes = topics[2][32-20:]
+            token_address = strip_0x(topics[2])[64-40:]
+            transfer_data['token_address'] = to_checksum_address(token_address)
+    return ('tokengift', transfer_data)
+
 
 class CallbackFilter(SyncFilter):
 
     trusted_addresses = []
 
-    def __init__(self, chain_spec, method, queue, caller_address=ZERO_ADDRESS):
+    def __init__(self, chain_spec, method, queue):
         self.queue = queue
         self.method = method
         self.chain_spec = chain_spec
-        self.caller_address = caller_address
-
-
-    def parse_transfer(self, tx, conn):
-        if not tx.payload:
-            return (None, None)
-        r = ERC20.parse_transfer_request(tx.payload)
-        transfer_data = {}
-        transfer_data['to'] = r[0]
-        transfer_data['value'] = r[1]
-        transfer_data['from'] = tx.outputs[0]
-        transfer_data['token_address'] = tx.inputs[0]
-        return ('transfer', transfer_data)
-
-
-    def parse_transferfrom(self, tx, conn):
-        if not tx.payload:
-            return (None, None)
-        r = ERC20.parse_transfer_from_request(tx.payload)
-        transfer_data = {}
-        transfer_data['from'] = r[0]
-        transfer_data['to'] = r[1]
-        transfer_data['value'] = r[2]
-        transfer_data['token_address'] = tx.inputs[0]
-        return ('transferfrom', transfer_data)
-
-
-    def parse_giftto(self, tx, conn):
-        if not tx.payload:
-            return (None, None)
-        r = Faucet.parse_give_to_request(tx.payload)
-        transfer_data = {}
-        transfer_data['to'] = r[0]
-        transfer_data['value'] = tx.value
-        transfer_data['from'] = tx.outputs[0]
-        #transfer_data['token_address'] = tx.inputs[0]
-        faucet_contract = tx.inputs[0]
-
-        c = SingleShotFaucet(self.chain_spec)
-        o = c.token(faucet_contract, sender_address=self.caller_address)
-        r = conn.do(o)
-        transfer_data['token_address'] = add_0x(c.parse_token(r))
-
-        o = c.amount(faucet_contract, sender_address=self.caller_address)
-        r = conn.do(o)
-        transfer_data['value'] = c.parse_amount(r)
-
-        return ('tokengift', transfer_data)
 
 
     def call_back(self, transfer_type, result):
-        result['chain_spec'] = result['chain_spec'].asdict()
+        logg.debug('result {}'.format(result))
         s = celery.signature(
             self.method,
             [
                 result,
                 transfer_type,
-                int(result['status_code'] != 0),
+                int(result['status_code'] == 0),
             ],
             queue=self.queue,
             )
@@ -106,29 +92,26 @@ class CallbackFilter(SyncFilter):
 #        s_translate.link(s)
 #        s_translate.apply_async()
         t = s.apply_async()
-        return t
+        return s
 
 
-    def parse_data(self, tx, conn):
+    def parse_data(self, tx):
         transfer_type = None
         transfer_data = None
         # TODO: what's with the mix of attributes and dict keys
         logg.debug('have payload {}'.format(tx.payload))
+        method_signature = tx.payload[:8]
 
         logg.debug('tx status {}'.format(tx.status))
 
         for parser in [
-                self.parse_transfer,
-                self.parse_transferfrom,
-                self.parse_giftto,
+                parse_transfer,
+                parse_transferfrom,
+                parse_giftto,
                 ]:
             try:
-                if tx:
-                    (transfer_type, transfer_data) = parser(tx, conn)
-                    if transfer_type == None:
-                        continue
-                else:
-                    pass
+                (transfer_type, transfer_data) = parser(tx)
+                break
             except RequestMismatchException:
                 continue
 
@@ -145,7 +128,7 @@ class CallbackFilter(SyncFilter):
         transfer_data = None
         transfer_type = None
         try:
-            (transfer_type, transfer_data) = self.parse_data(tx, conn)
+            (transfer_type, transfer_data) = self.parse_data(tx)
         except TypeError:
             logg.debug('invalid method data length for tx {}'.format(tx.hash))
             return
@@ -161,17 +144,16 @@ class CallbackFilter(SyncFilter):
             result = None
             try:
                 tokentx = ExtendedTx(conn, tx.hash, self.chain_spec)
-                tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses, caller_address=self.caller_address)
+                tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses)
                 tokentx.set_tokens(transfer_data['token_address'], transfer_data['value'])
                 if transfer_data['status'] == 0:
                     tokentx.set_status(1)
                 else:
                     tokentx.set_status(0)
-                result = tokentx.asdict()
-                t = self.call_back(transfer_type, result)
-                logg.info('callback success task id {} tx {} queue {}'.format(t, tx.hash, t.queue))
+                t = self.call_back(transfer_type, tokentx.to_dict())
+                logg.info('callback success task id {} tx {}'.format(t, tx.hash))
             except UnknownContractError:
-                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tx.queue, tx.method, transfer_data['to'], tx.hash))
+                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tc.queue, tc.method, transfer_data['to'], tx.hash))
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -11,19 +11,10 @@ import websocket
 # external imports
 import celery
 import confini
-from chainlib.connection import (
-        RPCConnection,
-        ConnType,
-        )
-from chainlib.eth.connection import (
-        EthUnixSignerConnection,
-        EthHTTPSignerConnection,
-        )
+from chainlib.connection import RPCConnection
+from chainlib.eth.connection import EthUnixSignerConnection
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
-from cic_eth_registry.error import UnknownContractError
-import liveness.linux
-
 
 # local imports
 from cic_eth.eth import (
@@ -48,7 +39,6 @@ from cic_eth.queue import (
 from cic_eth.callbacks import (
         Callback,
         http,
-        noop,
         #tcp,
         redis,
         )
@@ -60,8 +50,6 @@ from cic_eth.registry import (
         connect_declarator,
         connect_token_registry,
         )
-from cic_eth.task import BaseTask
-
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -73,7 +61,6 @@ argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provide
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
 argparser.add_argument('-r', type=str, help='CIC registry address')
-argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
@@ -93,7 +80,6 @@ config.process()
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
@@ -103,15 +89,14 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-health_modules = config.get('CIC_HEALTH_MODULES', [])
-if len(health_modules) != 0:
-    health_modules = health_modules.split(',')
-logg.debug('health mods {}'.format(health_modules))
-
 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
 
+# verify database connection with minimal sanity query
+session = SessionBase.create_session()
+session.execute('select version_num from alembic_version')
+session.close()
 
 # set up celery
 current_app = celery.Celery(__name__)
@@ -148,18 +133,11 @@ else:
         })
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
 RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
-RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
+RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
 
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 
-#import cic_eth.checks.gas
-#if not cic_eth.checks.gas.health(config=config):
-#    raise RuntimeError()
-liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker')
 
 def main():
     argv = ['worker']
@@ -183,11 +161,7 @@ def main():
 
     rpc = RPCConnection.connect(chain_spec, 'default')
 
-    try:
-        registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-    except UnknownContractError as e:
-        logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-        sys.exit(1)
+    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
@@ -196,18 +170,10 @@ def main():
     trusted_addresses = trusted_addresses_src.split(',')
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
-
     connect_declarator(rpc, chain_spec, trusted_addresses)
     connect_token_registry(rpc, chain_spec)
-
-    BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-    BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    BaseTask.run_dir = config.get('CIC_RUN_DIR')
-    logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
-   
-    liveness.linux.set(rundir=config.get('CIC_RUN_DIR'))
+    
     current_app.worker_main(argv)
-    liveness.linux.reset(rundir=config.get('CIC_RUN_DIR'))
 
 
 @celery.signals.eventlet_pool_postshutdown.connect

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,6 +15,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -25,7 +26,7 @@ from chainlib.eth.block import (
 from hexathon import (
         strip_0x,
         )
-from chainsyncer.backend.sql import SQLBackend
+from chainsyncer.backend import SyncerBackend
 from chainsyncer.driver import (
         HeadSyncer,
         HistorySyncer,
@@ -42,12 +43,6 @@ from cic_eth.runnable.daemons.filters import (
         TransferAuthFilter,
         )
 from cic_eth.stat import init_chain_stat
-from cic_eth.registry import (
-        connect as connect_registry,
-        connect_declarator,
-        connect_token_registry,
-        )
-
 
 script_dir = os.path.realpath(os.path.dirname(__file__))
 
@@ -93,18 +88,18 @@ def main():
 
     syncers = []
 
-    #if SQLBackend.first(chain_spec):
-    #    backend = SQLBackend.initial(chain_spec, block_offset)
-    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
+    #if SyncerBackend.first(chain_spec):
+    #    backend = SyncerBackend.initial(chain_spec, block_offset)
+    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
         logg.info('found no backends to resume')
-        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
+        syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
 
-    syncer_backends.append(SQLBackend.live(chain_spec, block_offset+1))
+    syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1))
 
     for syncer_backend in syncer_backends:
         try:
@@ -114,8 +109,6 @@ def main():
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
             syncers.append(HeadSyncer(syncer_backend))
 
-    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
         logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
@@ -123,8 +116,6 @@ def main():
     trusted_addresses = trusted_addresses_src.split(',')
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
-    connect_declarator(rpc, chain_spec, trusted_addresses)
-    connect_token_registry(rpc, chain_spec)
     CallbackFilter.trusted_addresses = trusted_addresses
 
     callback_filters = []

apps/cic-eth/cic_eth/runnable/info.py

@@ -1,65 +0,0 @@
-#!python3
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import logging
-import argparse
-import os
-
-# external imports
-import confini
-import celery
-
-# local imports
-from cic_eth.api import Api
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-default_format = 'terminal'
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-args = argparser.parse_args()
-
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.dict_override(args_override, 'cli args')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-queue = args.q
-
-api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
-
-def main():
-    t = api.default_token()
-    token_info = t.get()
-    print('Default token symbol: {}'.format(token_info['symbol']))
-    print('Default token address: {}'.format(token_info['address']))
-
-
-if __name__ == '__main__':
-    main()

apps/cic-eth/cic_eth/runnable/transfer.py

@@ -85,6 +85,9 @@ def main():
             callback_queue=args.q,
             )
 
+    #register = not args.no_register
+    #logg.debug('register {}'.format(register))
+    #t = api.create_account(register=register)
     t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL'))
 
     ps.get_message()

apps/cic-eth/cic_eth/runnable/view.py

@@ -81,14 +81,10 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 rpc = EthHTTPConnection(args.p)
 
-#registry_address = config.get('CIC_REGISTRY_ADDRESS')
+registry_address = config.get('CIC_REGISTRY_ADDRESS')
 
 admin_api = AdminApi(rpc)
 
-t = admin_api.registry()
-registry_address = t.get()
-logg.info('got registry address from task pool: {}'.format(registry_address))
-
 trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
 if trusted_addresses_src == None:
     logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
@@ -155,16 +151,14 @@ def main():
     txs  = []
     renderer = render_tx
     if len(config.get('_QUERY')) > 66:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
+        registry = connect_registry(rpc, chain_spec, registry_address)
+        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
     elif len(config.get('_QUERY')) > 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
+        registry = connect_registry(rpc, chain_spec, registry_address)
+        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
 
     elif len(config.get('_QUERY')) == 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
+        registry = connect_registry(rpc, chain_spec, registry_address)
         txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
         renderer = render_account
     elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':

apps/cic-eth/cic_eth/sync/retry.py

@@ -4,7 +4,7 @@ import datetime
 
 # external imports
 from chainsyncer.driver import HeadSyncer
-from chainsyncer.backend.memory import MemBackend
+from chainsyncer.backend import MemBackend
 from chainsyncer.error import NoBlockForYou
 from chainlib.eth.block import (
         block_by_number,

apps/cic-eth/cic_eth/task.py

@@ -7,20 +7,18 @@ import uuid
 # external imports
 import celery
 import sqlalchemy
-from chainlib.chain import ChainSpec
-from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.gas import RPCGasOracle
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.error import UnknownContractError
-import liveness.linux
 
 # local imports
-from cic_eth.error import SeppukuError
+from cic_eth.error import (
+        SignerError,
+        EthError,
+        )
 from cic_eth.db.models.base import SessionBase
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 celery_app = celery.current_app
 
@@ -31,9 +29,6 @@ class BaseTask(celery.Task):
     call_address = ZERO_ADDRESS
     create_nonce_oracle = RPCNonceOracle
     create_gas_oracle = RPCGasOracle
-    default_token_address = None
-    default_token_symbol = None
-    run_dir = '/run'
 
     def create_session(self):
         return BaseTask.session_func()
@@ -43,19 +38,6 @@ class BaseTask(celery.Task):
         logg.debug('task {} root uuid {}'.format(self.__class__.__name__, self.request.root_id))
         return
 
-
-    def on_failure(self, exc, task_id, args, kwargs, einfo):
-        if isinstance(exc, SeppukuError):
-            liveness.linux.reset(rundir=self.run_dir)
-            logg.critical(einfo)
-            msg = 'received critical exception {}, calling shutdown'.format(str(exc))
-            s = celery.signature(
-                'cic_eth.admin.ctrl.shutdown',
-                [msg],
-                queue=self.request.delivery_info.get('routing_key'),
-                    )
-            s.apply_async()
-
     
 class CriticalTask(BaseTask):
     retry_jitter = True
@@ -85,6 +67,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
         sqlalchemy.exc.TimeoutError,
         requests.exceptions.ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
+        EthError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
@@ -95,45 +78,19 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         sqlalchemy.exc.ResourceClosedError,
+        SignerError,
         ) 
 
 class CriticalWeb3AndSignerTask(CriticalTask):
     autoretry_for = (
         requests.exceptions.ConnectionError,
+        SignerError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
 
 
-@celery_app.task()
-def check_health(self):
-    pass
-
-
-# TODO: registry / rpc methods should perhaps be moved to better named module
-@celery_app.task()
-def registry():
-    return CICRegistry.address
-
-
-@celery_app.task()
-def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    registry = CICRegistry(chain_spec, conn)
-    return registry.by_address(address)
-
-
-@celery_app.task(throws=(UnknownContractError,))
-def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name)
-
-
-@celery_app.task()
-def rpc_proxy(chain_spec_dict, o, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    return conn.do(o)
+@celery_app.task(bind=True, base=BaseTask)
+def hello(self):
+    time.sleep(0.1)
+    return id(SessionBase.create_session)

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.11',
+        'beta.1',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/cic.ini

@@ -3,6 +3,3 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address =
-default_token_symbol = GFT
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
-run_dir = /run

apps/cic-eth/config/docker/cic.ini

@@ -3,6 +3,3 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
 tx_retry_delay = 20
-default_token_symbol = GFT
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
-run_dir = /run

apps/cic-eth/config/docker/eth.ini

@@ -1,3 +1,8 @@
 [eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_provider_address = 
+#chain_id = 
+abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis
+account_accounts_index_writer = 

apps/cic-eth/config/docker/signer.ini

@@ -1,5 +1,5 @@
 [signer]
-socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
+socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc
 secret = deedbeef
 database_name = signer_test
 dev_keys_path = 

apps/cic-eth/config/eth.ini

@@ -1,3 +1,8 @@
 [eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
 provider = http://localhost:8545
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_provider_address = 
+#chain_id = 
+abi_dir = /usr/local/share/cic/solidity/abi
+account_accounts_index_writer = 

apps/cic-eth/docker/Dockerfile

@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./
@@ -50,4 +50,6 @@ COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-COPY util/liveness/health.sh /usr/local/bin/health.sh
+RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
+	mkdir -p /usr/local/share/cic/solidity && \
+	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi

apps/cic-eth/requirements.txt

@@ -1,25 +1,25 @@
-cic-base==0.1.2b5
+cic-base~=0.1.2a62
 celery==4.4.7
-crypto-dev-signer~=0.4.14b3
+crypto-dev-signer~=0.4.14a17
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.4a16
+cic-eth-registry~=0.5.4a12
 #cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a9
-erc20-transfer-authorization~=0.3.1a5
+eth_accounts_index~=0.0.11a7
+erc20-transfer-authorization~=0.3.1a3
+#simple-rlp==0.1.2
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a9
-chainlib~=0.0.2a20
+eth-address-index~=0.1.1a7
+chainlib~=0.0.2a5
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a2
-chainqueue~=0.0.2a2
+chainsyncer~=0.0.1a21
+chainqueue~=0.0.1a7
 pysha3==1.0.2
 coincurve==15.0.0
-sarafu-faucet==0.0.2a28
-potaahto~=0.0.1a1
+sarafu-faucet~=0.0.2a19

apps/cic-eth/setup.cfg

@@ -38,7 +38,6 @@ packages =
 	cic_eth.runnable.daemons.filters
 	cic_eth.callbacks
 	cic_eth.sync
-	cic_eth.check
 scripts =
 	./scripts/migrate.py
 
@@ -53,7 +52,6 @@ console_scripts =
 	cic-eth-create = cic_eth.runnable.create:main 
 	cic-eth-inspect = cic_eth.runnable.view:main
 	cic-eth-ctl = cic_eth.runnable.ctrl:main
-	cic-eth-info = cic_eth.runnable.info:main
 	# TODO: Merge this with ctl when subcmds sorted to submodules
 	cic-eth-tag = cic_eth.runnable.tag:main
 	cic-eth-resend = cic_eth.runnable.resend:main

apps/cic-eth/test_requirements.txt

@@ -4,4 +4,4 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-giftable-erc20-token==0.0.8a9
+giftable-erc20-token==0.0.8a4

apps/cic-eth/tests/conftest.py

@@ -3,12 +3,8 @@ import os
 import sys
 import logging
 
-# external imports
-from chainlib.eth.erc20 import ERC20
-
 # local imports
 from cic_eth.api import Api
-from cic_eth.task import BaseTask
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -32,26 +28,3 @@ def api(
     ):
     chain_str = str(default_chain_spec)
     return Api(chain_str, queue=None, callback_param='foo')
-
-
-@pytest.fixture(scope='function')
-def foo_token_symbol(
-    default_chain_spec,
-    foo_token,
-    eth_rpc,
-    contract_roles,
-    ):
-    
-    c = ERC20(default_chain_spec)
-    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)
-
-
-@pytest.fixture(scope='function')
-def default_token(
-        foo_token,
-        foo_token_symbol,
-        ):
-    BaseTask.default_token_symbol = foo_token_symbol
-    BaseTask.default_token_address = foo_token   

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -1,225 +0,0 @@
-# standard import
-import logging
-import datetime
-import os
-
-# external imports
-import pytest
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.tx import (
-        receipt,
-        transaction,
-        Tx,
-        )
-from chainlib.eth.block import Block
-from chainlib.eth.erc20 import ERC20
-from sarafu_faucet import MinterFaucet
-from eth_accounts_index import AccountRegistry
-from potaahto.symbols import snake_and_camel
-from hexathon import add_0x
-
-# local imports
-from cic_eth.runnable.daemons.filters.callback import CallbackFilter
-
-logg = logging.getLogger()
-
-
-@pytest.mark.skip()
-def test_transfer_tx(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        foo_token,
-        agent_roles,
-        token_roles,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
-    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
-   
-    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = txf.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
-    r = rpc.do(o)
-    
-    o = transaction(tx_hash_hex)
-    r = rpc.do(o)
-    logg.debug(r)
-    tx_src = snake_and_camel(r)
-    tx = Tx(tx_src)
-
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-
-    rcpt = snake_and_camel(r)
-    tx.apply_receipt(rcpt)
-
-    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
-    (transfer_type, transfer_data) = fltr.parse_transfer(tx, eth_rpc)
-
-    assert transfer_type == 'transfer'
-
-
-@pytest.mark.skip()
-def test_transfer_from_tx(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        foo_token,
-        agent_roles,
-        token_roles,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
-    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
-   
-    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-
-    (tx_hash_hex, o) = txf.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
-    r = rpc.do(o)
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], rpc)
-    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = txf.transfer_from(foo_token, agent_roles['ALICE'], token_roles['FOO_TOKEN_OWNER'], agent_roles['BOB'], 1024)
-    r = rpc.do(o)
-    
-    o = transaction(tx_hash_hex)
-    r = rpc.do(o)
-    tx_src = snake_and_camel(r)
-    tx = Tx(tx_src)
-
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-
-    rcpt = snake_and_camel(r)
-    tx.apply_receipt(rcpt)
-
-    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
-    (transfer_type, transfer_data) = fltr.parse_transferfrom(tx, eth_rpc)
-
-    assert transfer_type == 'transferfrom'
-
-
-def test_faucet_gift_to_tx(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        foo_token,
-        agent_roles,
-        contract_roles,
-        faucet,
-        account_registry,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    gas_oracle = OverrideGasOracle(conn=rpc, limit=800000)
-
-    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], rpc)
-    txf = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = txf.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
-    r = rpc.do(o)
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-   
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], rpc)
-    txf = MinterFaucet(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = txf.give_to(faucet, agent_roles['ALICE'], agent_roles['ALICE'])
-    r = rpc.do(o)
-
-    o = transaction(tx_hash_hex)
-    r = rpc.do(o)
-    tx_src = snake_and_camel(r)
-    tx = Tx(tx_src)
-
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-
-    rcpt = snake_and_camel(r)
-    tx.apply_receipt(rcpt)
-
-    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
-    (transfer_type, transfer_data) = fltr.parse_giftto(tx, eth_rpc)
-
-    assert transfer_type == 'tokengift'
-    assert transfer_data['token_address'] == foo_token
-
-
-def test_callback_filter(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        foo_token,
-        token_roles,
-        agent_roles,
-        contract_roles,
-        register_lookups,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
-    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
-   
-    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = txf.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
-    r = rpc.do(o)
-    
-    o = transaction(tx_hash_hex)
-    r = rpc.do(o)
-    logg.debug(r)
-
-    mockblock_src = {
-        'hash': add_0x(os.urandom(32).hex()),
-        'number': '0x2a',
-        'transactions': [tx_hash_hex],
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-            }
-    mockblock = Block(mockblock_src)
-
-    tx_src = snake_and_camel(r)
-    tx = Tx(tx_src, block=mockblock)
-
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-
-    rcpt = snake_and_camel(r)
-    tx.apply_receipt(rcpt)
-
-    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
-
-    class CallbackMock:
-
-        def __init__(self):
-            self.results = {}
-            self.queue = 'test'
-
-        def call_back(self, transfer_type, result):
-            self.results[transfer_type] = result
-            return self
-
-    mock = CallbackMock()
-    fltr.call_back = mock.call_back
-
-    fltr.filter(eth_rpc, mockblock, tx, init_database)
-
-    assert mock.results.get('transfer') != None
-    assert mock.results['transfer']['destination_token'] == foo_token

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -65,7 +65,6 @@ def test_tx(
     tx_hash_hex_orig = tx_hash_hex
 
     gas_oracle = OverrideGasOracle(price=1100000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
     queue_create(
             default_chain_spec,

apps/cic-eth/tests/fixtures_celery.py

@@ -34,7 +34,6 @@ def celery_includes():
         'cic_eth.admin.ctrl',
         'cic_eth.admin.nonce',
         'cic_eth.admin.debug',
-        'cic_eth.admin.token',
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',

apps/cic-eth/tests/fixtures_database.py

@@ -53,9 +53,6 @@ def init_database(
     alembic.command.downgrade(ac, 'base')
     alembic.command.upgrade(ac, 'head')
 
-    session.execute('DELETE FROM lock')
-    session.commit()
-
     yield session
     session.commit()
     session.close()

apps/cic-eth/tests/task/api/test_admin.py

@@ -273,7 +273,7 @@ def test_tx(
     eth_signer,
     agent_roles,
     contract_roles,
-    celery_session_worker,
+    celery_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)

apps/cic-eth/tests/task/api/test_list.py

@@ -35,7 +35,7 @@ def test_list_tx(
         foo_token,
         register_tokens,
         init_eth_tester,
-        celery_session_worker,
+        celery_worker,
         ):
 
     tx_hashes = []

apps/cic-eth/tests/unit/admin/test_default_token.py

@@ -1,21 +0,0 @@
-# external imports
-import celery
-
-
-def test_default_token(
-        default_token,
-        celery_session_worker,
-        foo_token,
-        foo_token_symbol,
-        ):
-      
-    s = celery.signature(
-            'cic_eth.admin.token.default_token',
-            [],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get()
-
-    assert r['address'] == foo_token
-    assert r['symbol'] == foo_token_symbol

apps/cic-meta/.gitignore

@@ -3,3 +3,4 @@ dist
 dist-web
 dist-server
 scratch
+tests

apps/cic-meta/package-lock.json

@@ -1,6 +1,6 @@
 {
 	"name": "cic-client-meta",
-	"version": "0.0.7-alpha.7",
+	"version": "0.0.7-alpha.2",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
@@ -852,75 +852,6 @@
 				"printj": "~1.1.0"
 			}
 		},
-		"crdt-meta": {
-			"version": "0.0.8",
-			"resolved": "https://registry.npmjs.org/crdt-meta/-/crdt-meta-0.0.8.tgz",
-			"integrity": "sha512-CS0sS0L2QWthz7vmu6vzl3p4kcpJ+IKILBJ4tbgN4A3iNG8wnBeuDIv/z3KFFQjcfuP4QAh6E9LywKUTxtDc3g==",
-			"requires": {
-				"automerge": "^0.14.2",
-				"ini": "^1.3.8",
-				"openpgp": "^4.10.8",
-				"pg": "^8.5.1",
-				"sqlite3": "^5.0.2"
-			},
-			"dependencies": {
-				"automerge": {
-					"version": "0.14.2",
-					"resolved": "https://registry.npmjs.org/automerge/-/automerge-0.14.2.tgz",
-					"integrity": "sha512-shiwuJHCbNRI23WZyIECLV4Ovf3WiAFJ7P9BH4l5gON1In/UUbjcSJKRygtIirObw2UQumeYxp3F2XBdSvQHnA==",
-					"requires": {
-						"immutable": "^3.8.2",
-						"transit-immutable-js": "^0.7.0",
-						"transit-js": "^0.8.861",
-						"uuid": "^3.4.0"
-					}
-				},
-				"node-addon-api": {
-					"version": "3.1.0",
-					"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz",
-					"integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw=="
-				},
-				"pg": {
-					"version": "8.6.0",
-					"resolved": "https://registry.npmjs.org/pg/-/pg-8.6.0.tgz",
-					"integrity": "sha512-qNS9u61lqljTDFvmk/N66EeGq3n6Ujzj0FFyNMGQr6XuEv4tgNTXvJQTfJdcvGit5p5/DWPu+wj920hAJFI+QQ==",
-					"requires": {
-						"buffer-writer": "2.0.0",
-						"packet-reader": "1.0.0",
-						"pg-connection-string": "^2.5.0",
-						"pg-pool": "^3.3.0",
-						"pg-protocol": "^1.5.0",
-						"pg-types": "^2.1.0",
-						"pgpass": "1.x"
-					}
-				},
-				"pg-connection-string": {
-					"version": "2.5.0",
-					"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
-					"integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ=="
-				},
-				"pg-pool": {
-					"version": "3.3.0",
-					"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.3.0.tgz",
-					"integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg=="
-				},
-				"pg-protocol": {
-					"version": "1.5.0",
-					"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz",
-					"integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ=="
-				},
-				"sqlite3": {
-					"version": "5.0.2",
-					"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.0.2.tgz",
-					"integrity": "sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA==",
-					"requires": {
-						"node-addon-api": "^3.0.0",
-						"node-gyp": "3.x",
-						"node-pre-gyp": "^0.11.0"
-					}
-				}
-			}
-		},
 		"create-hash": {
 			"version": "1.2.0",
 			"resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
@@ -1035,17 +966,17 @@
 			"dev": true
 		},
 		"elliptic": {
-			"version": "6.5.4",
-			"resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz",
-			"integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==",
+			"version": "6.5.3",
+			"resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz",
+			"integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==",
 			"requires": {
-				"bn.js": "^4.11.9",
-				"brorand": "^1.1.0",
+				"bn.js": "^4.4.0",
+				"brorand": "^1.0.1",
 				"hash.js": "^1.0.0",
-				"hmac-drbg": "^1.0.1",
-				"inherits": "^2.0.4",
-				"minimalistic-assert": "^1.0.1",
-				"minimalistic-crypto-utils": "^1.0.1"
+				"hmac-drbg": "^1.0.0",
+				"inherits": "^2.0.1",
+				"minimalistic-assert": "^1.0.0",
+				"minimalistic-crypto-utils": "^1.0.0"
 			}
 		},
 		"emoji-regex": {
@@ -1558,9 +1489,9 @@
 			"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
 		},
 		"ini": {
-			"version": "1.3.8",
-			"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-			"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
+			"version": "1.3.5",
+			"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+			"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
 		},
 		"interpret": {
 			"version": "2.2.0",
@@ -2026,9 +1957,9 @@
 					}
 				},
 				"y18n": {
-					"version": "4.0.3",
-					"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz",
-					"integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==",
+					"version": "4.0.0",
+					"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
+					"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
 					"dev": true
 				},
 				"yargs": {

apps/cic-meta/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cic-client-meta",
-	"version": "0.0.7-alpha.8",
+	"version": "0.0.7-alpha.6",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
@@ -15,9 +15,8 @@
 	"dependencies": {
 		"@ethereumjs/tx": "^3.0.0-beta.1",
 		"automerge": "^0.14.1",
-		"crdt-meta": "0.0.8",
 		"ethereumjs-wallet": "^1.0.1",
-		"ini": "^1.3.8",
+		"ini": "^1.3.5",
 		"openpgp": "^4.10.8",
 		"pg": "^8.4.2",
 		"sqlite3": "^5.0.0",
@@ -41,6 +40,6 @@
 	],
 	"license": "GPL-3.0-or-later",
 	"engines": {
-		"node": ">=14.16.1"
+		"node": "~14.16.1"
 	}
 }

apps/cic-meta/scripts/dumpconfig.js

@@ -1,4 +1,4 @@
-import { Config } from 'crdt-meta';
+const config = require('./src/config');
 const fs = require('fs');
 
 if (process.argv[2] === undefined) {
@@ -15,6 +15,6 @@ try {
 	process.exit(1);
 }
 
-const c = new Config(process.argv[2], process.env['CONFINI_ENV_PREFIX']);
+const c = new config.Config(process.argv[2], process.env['CONFINI_ENV_PREFIX']);
 c.process();
 process.stdout.write(c.toString());

apps/cic-meta/scripts/server/handlers.ts

@@ -1,7 +1,8 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
+import * as pg from 'pg';
 
-import { Envelope, Syncable } from 'crdt-meta';
+import { Envelope, Syncable } from '../../src/sync';
 
 
 function handleNoMergeGet(db, digest, keystore) {

apps/cic-meta/scripts/server/server.ts

@@ -1,11 +1,15 @@
 import * as http from 'http';
 import * as fs from 'fs';
 import * as path from 'path';
+import * as pgp from 'openpgp';
 
 import * as handlers from './handlers';
-import { PGPKeyStore, PGPSigner, Config, SqliteAdapter, PostgresAdapter } from 'crdt-meta';
+import { Envelope, Syncable } from '../../src/sync';
+import { PGPKeyStore, PGPSigner } from '../../src/auth';
 
 import { standardArgs } from './args';
+import { Config } from '../../src/config';
+import { SqliteAdapter, PostgresAdapter } from '../../src/db';
 
 let configPath = '/usr/local/etc/cic-meta';
 

apps/cic-meta/src/assets/phone.ts

@@ -1,4 +1,5 @@
-import { Syncable, Addressable, mergeKey } from 'crdt-meta';
+import { ArgPair, Syncable } from '../sync';
+import { Addressable, mergeKey } from '../digest';
 
 class Phone extends Syncable implements Addressable {
 

apps/cic-meta/src/assets/user.ts

@@ -1,4 +1,5 @@
-import { Syncable, Addressable, toAddressKey } from 'crdt-meta';
+import { ArgPair, Syncable } from '../sync';
+import { Addressable, addressToBytes, bytesToHex, toAddressKey } from '../digest';
 
 const keySalt = new TextEncoder().encode(':cic.person');
 class User extends Syncable implements Addressable {

apps/cic-meta/src/auth.ts

@@ -0,0 +1,191 @@
+import * as pgp from 'openpgp';
+import * as crypto from 'crypto';
+
+interface Signable {
+	digest():string;
+}
+
+type KeyGetter = () => any;
+
+type Signature = {
+	engine:string
+	algo:string
+	data:string
+	digest:string
+}
+
+interface Signer { 
+	prepare(Signable):boolean;
+	onsign(Signature):void;
+	onverify(boolean):void;
+	sign(digest:string):void
+	verify(digest:string, signature:Signature):void
+	fingerprint():string
+}
+
+interface Authoritative {
+}
+
+interface KeyStore {
+	getPrivateKey:		KeyGetter
+	getFingerprint:		() => string
+	getTrustedKeys: 	() => Array<any>
+	getTrustedActiveKeys: 	() => Array<any>
+	getEncryptKeys: 	() => Array<any>
+}
+
+class PGPKeyStore implements KeyStore {
+
+	fingerprint:	string
+	pk:		any
+
+	pubk = {
+		active: [],
+		trusted: [],
+		encrypt: [],
+	}
+	loads = 0x00;
+	loadsTarget = 0x0f;
+	onload: (k:KeyStore) => void;
+
+	constructor(passphrase:string, pkArmor:string, pubkActiveArmor:string, pubkTrustedArmor:string, pubkEncryptArmor:string, onload = (ks:KeyStore) => {}) {
+		this._readKey(pkArmor, undefined, 1, passphrase);
+		this._readKey(pubkActiveArmor, 'active', 2);
+		this._readKey(pubkTrustedArmor, 'trusted', 4);
+		this._readKey(pubkEncryptArmor, 'encrypt', 8);
+		this.onload = onload;
+	}
+
+	private _readKey(a:string, x:any, n:number, pass?:string) {
+		pgp.key.readArmored(a).then((k) => {
+			if (pass !== undefined) {
+				this.pk = k.keys[0];
+				this.pk.decrypt(pass).then(() => {
+					this.fingerprint = this.pk.getFingerprint();
+					console.log('private key (sign)', this.fingerprint);
+					this._registerLoad(n);
+				});
+			} else {
+				this.pubk[x] = k.keys;
+				k.keys.forEach((pubk) => {
+					console.log('public key (' + x + ')', pubk.getFingerprint());
+				});
+				this._registerLoad(n);
+			}
+		});
+	}
+
+	private _registerLoad(b:number) {
+		this.loads |= b;
+		if (this.loads == this.loadsTarget) {
+			this.onload(this);
+		}
+	}
+
+	public getTrustedKeys(): Array<any> {
+		return this.pubk['trusted'];
+	}
+
+	public getTrustedActiveKeys(): Array<any> {
+		return this.pubk['active'];
+
+	}
+
+	public getEncryptKeys(): Array<any> {
+		return this.pubk['encrypt'];
+
+	}
+
+	public getPrivateKey(): any {
+		return this.pk;
+	}
+
+	public getFingerprint(): string {
+		return this.fingerprint;
+	}
+}
+
+class PGPSigner implements Signer {
+
+	engine	= 'pgp'
+	algo	= 'sha256'
+	dgst:		string
+	signature:	Signature
+	keyStore:	KeyStore
+	onsign:		(Signature) => void
+	onverify:	(boolean) => void
+
+	constructor(keyStore:KeyStore) {
+		this.keyStore = keyStore
+		this.onsign = (string) => {};
+		this.onverify = (boolean) => {};
+	}
+
+	public fingerprint(): string {
+		return this.keyStore.getFingerprint();
+	}
+
+	public prepare(material:Signable):boolean {
+		this.dgst = material.digest();
+		return true;
+	}
+
+	public verify(digest:string, signature:Signature) {
+		pgp.signature.readArmored(signature.data).then((s) => {
+			const opts = {
+				message: pgp.cleartext.fromText(digest),
+				publicKeys: this.keyStore.getTrustedKeys(),
+				signature: s,
+			};
+			pgp.verify(opts).then((v) => {
+				let i = 0;
+				for (i = 0; i < v.signatures.length; i++) {
+					const s = v.signatures[i];
+					if (s.valid) {
+						this.onverify(s);
+						return;
+					}
+				}
+				console.error('checked ' + i + ' signature(s) but none valid');
+				this.onverify(false);
+			});
+		}).catch((e) => {
+			console.error(e);
+			this.onverify(false);
+		});
+	}
+
+	public sign(digest:string) {
+		const m = pgp.cleartext.fromText(digest);
+		const pk = this.keyStore.getPrivateKey();
+		const opts = {
+			message: m,
+			privateKeys: [pk],
+			detached: true,
+		}
+		pgp.sign(opts).then((s) => {
+			this.signature = {
+				engine: this.engine,
+				algo: this.algo,
+				data: s.signature,
+				// TODO: fix for browser later
+				digest: digest,
+			};
+			this.onsign(this.signature);
+		}).catch((e) => {
+			console.error(e);
+			this.onsign(undefined);
+		});
+	}
+}
+
+export {
+	Signature,
+	Authoritative,
+       	Signer,
+	KeyGetter,
+	Signable,
+	KeyStore,
+	PGPSigner,
+	PGPKeyStore,
+};

apps/cic-meta/src/config.ts

@@ -0,0 +1,71 @@
+import * as fs from 'fs';
+import * as ini from 'ini';
+import * as path from 'path';
+
+class Config {
+
+	filepath: 	string
+	store:		Object
+	censor:		Array<string>
+	require:	Array<string>
+	env_prefix:	string
+
+	constructor(filepath:string, env_prefix?:string) {
+		this.filepath = filepath;
+		this.store = {};
+		this.censor = [];
+		this.require = [];
+		this.env_prefix = '';
+		if (env_prefix !== undefined) {
+			this.env_prefix = env_prefix + "_";
+		}
+	}
+
+	public process() {
+		const d = fs.readdirSync(this.filepath);
+		
+		const r = /.*\.ini$/;
+		for (let i = 0; i < d.length; i++) {
+			const f = d[i];
+			if (!f.match(r)) {
+				return;
+			}
+
+			const fp = path.join(this.filepath, f);
+			const v = fs.readFileSync(fp, 'utf-8');
+			const inid = ini.decode(v);
+			const inik = Object.keys(inid);
+			for (let j = 0; j < inik.length; j++) {
+				const k_section = inik[j]
+				const k = k_section.toUpperCase();
+				Object.keys(inid[k_section]).forEach((k_directive) => {
+					const kk = k_directive.toUpperCase();
+					const kkk = k + '_' + kk;
+
+					let r = inid[k_section][k_directive];
+					const k_env = this.env_prefix + kkk
+					const env = process.env[k_env];
+					if (env !== undefined) {
+						console.debug('Environment variable ' + k_env + ' overrides ' + kkk);
+						r = env;
+					}
+					this.store[kkk] = r;
+				});
+			}
+		}
+	}
+
+	public get(s:string) {
+		return this.store[s];
+	}
+
+	public toString() {
+		let s = '';
+		Object.keys(this.store).forEach((k) => {
+			s += k + '=' + this.store[k] + '\n';
+		});
+		return s;
+	}	
+}
+
+export { Config };

apps/cic-meta/src/constants.ts

@@ -0,0 +1,38 @@
+import { JSONSerializable } from './format';
+
+const ENGINE_NAME = 'automerge';
+const ENGINE_VERSION = '0.14.1';
+
+const NETWORK_NAME = 'cic';
+const NETWORK_VERSION = '1';
+
+const CRYPTO_NAME = 'pgp';
+const CRYPTO_VERSION = '2';
+
+type VersionedSpec = {
+	name:		string
+	version: 	string
+	ext?:		Object
+}
+
+const engineSpec:VersionedSpec = {
+	name: ENGINE_NAME,
+	version: ENGINE_VERSION,			
+}
+
+const cryptoSpec:VersionedSpec = {
+	name: CRYPTO_NAME,
+	version: CRYPTO_VERSION,
+}
+
+const networkSpec:VersionedSpec = {
+	name: NETWORK_NAME,
+	version: NETWORK_VERSION,
+}
+
+export {
+	engineSpec,
+	cryptoSpec,
+	networkSpec,
+	VersionedSpec,
+};

apps/cic-meta/src/crypto.ts

@@ -0,0 +1,27 @@
+import * as crypto from 'crypto';
+
+const _algs = {
+	'SHA-256': 'sha256',
+}
+
+function cryptoWrapper() {
+}
+
+cryptoWrapper.prototype.digest = async function(s, d) {
+	const h = crypto.createHash(_algs[s]);
+	h.update(d);
+	return h.digest();
+}
+
+let subtle = undefined;
+if (typeof window !== 'undefined') {
+	subtle = window.crypto.subtle;
+} else {
+	subtle = new cryptoWrapper();
+}
+
+
+export {
+	subtle,
+}
+

apps/cic-meta/src/db.ts

@@ -0,0 +1,90 @@
+import * as pg from 'pg';
+import * as sqlite from 'sqlite3';
+
+type DbConfig = {
+	name:		string
+	host:		string
+	port:		number
+	user:		string
+	password:	string	
+}
+
+interface DbAdapter {
+	query:	(s:string, callback:(e:any, rs:any) => void) => void
+	close: 	() => void
+}
+
+const re_creatematch = /^(CREATE)/i
+const re_getmatch = /^(SELECT)/i;
+const re_setmatch = /^(INSERT|UPDATE)/i;
+
+class SqliteAdapter implements DbAdapter {
+
+	db:		any
+
+	constructor(dbConfig:DbConfig, callback?:(any) => void) {
+		this.db = new sqlite.Database(dbConfig.name); //, callback);
+	}
+
+	public query(s:string, callback:(e:any, rs?:any) => void): void {
+		const local_callback = (e, rs) => {
+			let r = undefined;
+			if (rs !== undefined) {
+				r = {
+					rowCount: rs.length,
+					rows: rs,
+				}
+			}
+			callback(e, r);
+		};
+		if (s.match(re_getmatch)) {
+			this.db.all(s, local_callback);
+		} else if (s.match(re_setmatch)) {
+			this.db.run(s, local_callback);
+		} else if (s.match(re_creatematch)) {
+			this.db.run(s, callback);
+		} else {
+			throw 'unhandled query';
+		}
+	}
+
+	public close() {
+		this.db.close();
+	}
+}
+
+class PostgresAdapter implements DbAdapter {
+
+	db:		any
+
+	constructor(dbConfig:DbConfig) {
+		let o = dbConfig;
+		o['database'] = o.name;
+		this.db = new pg.Pool(o);
+		return this.db;
+	}
+
+	public query(s:string, callback:(e:any, rs:any) => void): void {
+		this.db.query(s, (e, rs) => {
+			let r = {
+				length: rs.rowCount,
+			}
+			rs.length = rs.rowCount;
+			if (e === undefined) {
+				e = null;
+			}
+			console.debug(e, rs);
+			callback(e, rs);
+		});
+	}
+
+	public close() {
+		this.db.end();
+	}
+}
+
+export {
+	DbConfig,
+	SqliteAdapter,
+	PostgresAdapter,
+}

apps/cic-meta/src/digest.ts

@@ -0,0 +1,68 @@
+import * as crypto from './crypto';
+
+interface Addressable {
+	key(): string
+	digest(): string
+}
+
+function stringToBytes(s:string) {
+	const a = new Uint8Array(20);
+	let j = 2;
+	for (let i = 0; i < a.byteLength; i++) {
+		const n = parseInt(s.substring(j, j+2), 16);
+		a[i] = n;
+		j += 2;
+	}
+	return a;
+}
+
+function bytesToHex(a:Uint8Array) {
+	let s = '';
+	for (let i = 0; i < a.byteLength; i++) {
+		const h = '00' + a[i].toString(16);
+		s += h.slice(-2);
+	}
+	return s;
+}
+
+async function mergeKey(a:Uint8Array, s:Uint8Array) {
+	const y = new Uint8Array(a.byteLength + s.byteLength);
+	for (let i = 0; i < a.byteLength; i++) {
+		y[i] = a[i];
+	}
+	for (let i = 0; i < s.byteLength; i++) {
+		y[a.byteLength + i] = s[i];
+	}
+	const z = await crypto.subtle.digest('SHA-256', y);
+	return bytesToHex(new Uint8Array(z));
+}
+
+async function toKey(v:string, salt:string) {
+	const a = stringToBytes(v);
+	const s = new TextEncoder().encode(salt);
+	return await mergeKey(a, s);
+}
+
+
+async function toAddressKey(zeroExHex:string, salt:string) {
+	const a = addressToBytes(zeroExHex);
+	const s = new TextEncoder().encode(salt);
+	return await mergeKey(a, s);
+}
+
+const re_addrHex = /^0[xX][a-fA-F0-9]{40}$/;
+function addressToBytes(s:string) {
+	if (!s.match(re_addrHex)) {
+		throw 'invalid address hex';
+	}
+	return stringToBytes(s);
+}
+
+export {
+	toKey,
+	toAddressKey,
+	mergeKey,
+	bytesToHex,
+	addressToBytes,
+	Addressable,
+}

apps/cic-meta/src/dispatch.ts

@@ -0,0 +1,58 @@
+import { v4 as uuidv4 } from 'uuid';
+import { Syncable } from './sync';
+import { Store } from './store';
+import { PubSub } from './transport';
+
+function toIndexKey(id:string):string {
+	const d = Date.now();
+	return d + '_' + id + '_' + uuidv4();
+}
+
+const _re_indexKey = /^\d+_(.+)_[-\d\w]+$/;
+function fromIndexKey(s:string):string {
+	const m = s.match(_re_indexKey);
+	if (m === null) {
+		throw 'Invalid index key';
+	}
+	return m[1];
+}
+
+class Dispatcher {
+
+	idx:		Array<string>
+	syncer:		PubSub
+	store:		Store
+
+	constructor(store:Store, syncer:PubSub) {
+		this.idx = new Array<string>()
+		this.syncer = syncer;
+		this.store = store;
+	}
+
+	public isDirty(): boolean {
+		return this.idx.length > 0;
+	}
+
+	public add(id:string, item:Syncable): string {
+		const v = item.toJSON();
+		const k = toIndexKey(id);
+		this.store.put(k, v, true);
+		localStorage.setItem(k, v);
+		this.idx.push(k);
+		return k;
+	}
+
+	public sync(offset:number): number {
+		let i = 0;
+		this.idx.forEach((k) => {
+			const v = localStorage.getItem(k);
+			const k_id = fromIndexKey(k);
+			this.syncer.pub(v); // this must block until guaranteed delivery
+			localStorage.removeItem(k);
+			i++;
+		});
+		return i;
+	}
+}
+
+export { Dispatcher, toIndexKey, fromIndexKey }

apps/cic-meta/src/format.ts

@@ -0,0 +1,5 @@
+interface JSONSerializable {
+	toJSON(): string
+}
+
+export { JSONSerializable };

apps/cic-meta/src/index.ts

@@ -1,2 +1,5 @@
-export { User } from './user';
-export { Phone } from './phone';
+export { PGPSigner, PGPKeyStore, Signer, KeyStore } from './auth';
+export { ArgPair,  Envelope, Syncable } from './sync';
+export { User } from './assets/user';
+export { Phone } from './assets/phone';
+export { Config } from './config';

apps/cic-meta/src/store.ts

@@ -0,0 +1,9 @@
+import { Syncable } from './sync';
+
+interface Store {
+	put(string, Syncable, boolean?)
+	get(string):Syncable
+	delete(string)
+}
+
+export { Store };

apps/cic-meta/src/sync.ts

@@ -0,0 +1,266 @@
+import * as Automerge from 'automerge';
+
+import { JSONSerializable } from './format';
+
+import { Authoritative, Signer, PGPSigner, Signable, Signature } from './auth';
+
+import { engineSpec, cryptoSpec, networkSpec, VersionedSpec  } from './constants';
+
+const fullSpec:VersionedSpec = {
+	name: 'cic',
+	version: '1',
+	ext: {
+		network: cryptoSpec,
+		engine: engineSpec,
+	},
+}
+
+class Envelope {
+
+	o = fullSpec
+
+	constructor(payload:Object) {
+		this.set(payload);
+	}
+
+	public set(payload:Object) {
+		this.o['payload'] = payload
+	}
+
+	public get():string {
+		return this.o['payload'];
+	}
+
+	public toJSON() {
+		return JSON.stringify(this.o);
+	}
+
+	public static fromJSON(s:string): Envelope {
+		const e = new Envelope(undefined);
+		e.o = JSON.parse(s);
+		return e;
+	}
+
+	public unwrap(): Syncable {
+		return Syncable.fromJSON(this.o['payload']);
+	}
+}
+
+class ArgPair {
+
+	k:string
+	v:any
+
+	constructor(k:string, v:any) {
+		this.k = k;
+		this.v = v;
+	}
+}
+
+class SignablePart implements Signable {
+
+	s: string
+
+	constructor(s:string) {
+		this.s = s;
+	}
+
+	public digest():string {
+		return this.s;
+	}
+}
+
+function orderDict(src) {
+	let dst;
+	if (Array.isArray(src)) {
+		dst = [];
+		src.forEach((v) => {
+			if (typeof(v) == 'object') {
+				v = orderDict(v);
+			}
+			dst.push(v);
+		});
+	} else {
+		dst = {}
+		Object.keys(src).sort().forEach((k) => {
+			let v = src[k];
+			if (typeof(v) == 'object') {
+				v = orderDict(v);
+			}
+			dst[k] = v;
+		});
+	} 
+	return dst;
+}
+
+class Syncable implements JSONSerializable, Authoritative, Signable {
+
+	id:		string
+	timestamp:	number
+	m:		any // automerge object
+	e:		Envelope
+	signer:		Signer
+	onwrap: 	(string) => void
+	onauthenticate:	(boolean) => void
+
+	// TODO: Move data to sub-object so timestamp, id, signature don't collide
+	constructor(id:string, v:Object) {
+		this.id = id;
+		const o = {
+			'id': id,
+			'timestamp': Math.floor(Date.now() / 1000),
+			'data': v,
+		}
+		//this.m = Automerge.from(v)
+		this.m = Automerge.from(o)
+	}
+
+	public setSigner(signer:Signer) {
+		this.signer = signer;
+		this.signer.onsign = (s) => {
+			this.wrap(s);
+		};
+	}
+
+	// TODO: To keep integrity, the non-link key/value pairs for each step also need to be hashed
+	public digest(): string {
+		const links = [];
+		Automerge.getAllChanges(this.m).forEach((ch:Object) => {
+			const op:Array<any> = ch['ops'];
+			ch['ops'].forEach((op:Array<Object>) => {
+				if (op['action'] == 'link') {
+					//console.log('op link', op);
+					links.push([op['obj'], op['value']]);
+				}
+			});
+		});
+		//return JSON.stringify(links);
+		const j = JSON.stringify(links);
+		return Buffer.from(j).toString('base64');
+	}
+
+	private wrap(s:any) {
+		this.m = Automerge.change(this.m, 'sign', (doc) => {
+			doc['signature'] = s;
+		});
+		this.e = new Envelope(this.toJSON());
+		console.log('wrappin s', s, typeof(s));
+		this.e.o['digest'] = s.digest;
+		if (this.onwrap !== undefined) {
+			this.onwrap(this.e);
+		}
+
+	}
+
+//	private _verifyLoop(i:number, history:Array<any>, signable:Signable, result:boolean) {
+//		if (!result) {
+//			this.onauthenticate(false);
+//			return;
+//		} else if (history.length == 0) {
+//			this.onauthenticate(true);
+//			return;
+//		}
+//		const h = history.shift()
+//		if (i % 2 == 0) {
+//			i++;
+//			signable = {
+//				digest: () => {
+//					return Automerge.save(h.snapshot)
+//				},
+//			};
+//			this._verifyLoop(i, history, signable, true);
+//		} else {
+//			i++;
+//			const signature = h.snapshot['signature'];
+//			console.debug('signature', signature, signable.digest());
+//			this.signer.onverify = (v) => {
+//				this._verifyLoop(i, history, signable, v)
+//			}
+//			this.signer.verify(signable, signature);
+//		}
+//	}
+//
+//	// TODO: This should replay the graph and check signatures on each step
+//	public _authenticate(full:boolean=false) {
+//		let h = Automerge.getHistory(this.m);
+//		h.forEach((m) => {
+//			//console.debug(m.snapshot);
+//		});
+//		const signable = {
+//			digest: () => { return '' },
+//		}
+//		if (!full) {
+//			h = h.slice(h.length-2);
+//		}
+//		this._verifyLoop(0, h, signable, true);
+//	}
+
+	public authenticate(full:boolean=false) {
+		if (full) {
+			console.warn('only doing shallow authentication for now, sorry');			
+		}
+		//console.log('authenticating', signable.digest());
+		//console.log('signature', this.m.signature);
+		this.signer.onverify = (v) => {
+			//this._verifyLoop(i, history, signable, v)
+			this.onauthenticate(v);
+		}
+		this.signer.verify(this.m.signature.digest, this.m.signature);
+	}
+
+
+	public sign() {
+		//this.signer.prepare(this);
+		this.signer.sign(this.digest());
+	}
+
+	public update(changes:Array<ArgPair>, changesDescription:string) {
+		this.m = Automerge.change(this.m, changesDescription, (m) => {
+			changes.forEach((c) => {
+				let path = c.k.split('.');
+				let target = m['data'];
+				while (path.length > 1) {
+					const part = path.shift();
+					target = target[part];
+				}
+				target[path[0]] = c.v;
+			});
+			m['timestamp'] = Math.floor(Date.now() / 1000);
+		});
+	}
+
+	public replace(o:Object, changesDescription:string) {
+		this.m = Automerge.change(this.m, changesDescription, (m) => {
+			Object.keys(o).forEach((k) => {
+				m['data'][k] = o[k];
+			});
+			Object.keys(m).forEach((k) => {
+				if (o[k] == undefined) {
+					delete m['data'][k];
+				}
+			});
+			m['timestamp'] = Math.floor(Date.now() / 1000);
+		});
+	}
+
+	public merge(s:Syncable) {
+		this.m = Automerge.merge(s.m, this.m);
+	}
+
+	public toJSON(): string {
+		const s = Automerge.save(this.m);
+		const o = JSON.parse(s);
+		const oo = orderDict(o)
+		return JSON.stringify(oo);
+
+	}
+
+	public static fromJSON(s:string): Syncable {
+		const doc = Automerge.load(s);
+		let y = new Syncable(doc['id'], {});
+		y.m = doc
+		return y
+	}
+}
+
+export { JSONSerializable, Syncable, ArgPair, Envelope };

apps/cic-meta/src/transport.ts

@@ -0,0 +1,11 @@
+interface SubConsumer {
+	post(string)
+}
+
+interface PubSub {
+	pub(v:string):boolean
+	close()
+}
+
+export { PubSub, SubConsumer };
+

apps/cic-meta/tests/1_basic.ts

@@ -0,0 +1,50 @@
+import * as Automerge from 'automerge';
+import assert = require('assert');
+
+import { Dispatcher, toIndexKey, fromIndexKey } from '../src/dispatch';
+import { User } from '../src/assets/user';
+import { Syncable, ArgPair } from '../src/sync';
+
+import { MockSigner, MockStore } from './mock';
+
+describe('basic', () => {
+
+	it('store', () => {
+		const store = new MockStore('s');
+		assert.equal(store.name, 's');
+
+		const mockSigner = new MockSigner();
+		const v = new Syncable('foo', {baz: 42});
+		v.setSigner(mockSigner);
+		store.put('foo', v);
+		const one = store.get('foo').toJSON();
+		const vv = new Syncable('bar', {baz: 666});
+		vv.setSigner(mockSigner);
+		assert.throws(() => {
+			store.put('foo', vv)
+		});
+		store.put('foo', vv, true);
+		const other = store.get('foo').toJSON();
+		assert.notEqual(one, other);
+		store.delete('foo');
+		assert.equal(store.get('foo'), undefined);
+	});
+
+	it('add_doc_to_dispatcher', () => {
+		const store = new MockStore('s');
+		//const syncer = new MockSyncer();
+		const dispatcher = new Dispatcher(store, undefined);
+		const user = new User('foo'); 
+		dispatcher.add(user.id, user);
+		assert(dispatcher.isDirty());
+	});
+
+	it('dispatch_keyindex', () => {
+		const s = 'foo';
+		const k = toIndexKey(s);
+		const v = fromIndexKey(k);
+		assert.equal(s, v);
+	});
+
+
+});

apps/cic-meta/tests/2_sync.ts

@@ -0,0 +1,212 @@
+import * as Automerge from 'automerge';
+import assert = require('assert');
+
+import * as pgp from 'openpgp';
+import * as fs from 'fs';
+
+import { PGPSigner } from '../src/auth';
+
+import { Syncable, ArgPair } from '../src/sync';
+
+import { MockKeyStore, MockSigner } from './mock';
+
+
+describe('sync', async () => {
+	it('sync_merge', () => {
+		const mockSigner = new MockSigner();
+		const s = new Syncable('foo', {
+			bar: 'baz',
+		});
+		s.setSigner(mockSigner);
+		const changePair = new ArgPair('xyzzy', 42);
+		s.update([changePair], 'ch-ch-cha-changes');
+		assert.equal(s.m.data['xyzzy'], 42)
+		assert.equal(s.m.data['bar'], 'baz')
+		assert.equal(s.m['id'], 'foo')
+		assert.equal(Automerge.getHistory(s.m).length, 2);
+	});
+
+	it('sync_serialize', () => {
+		const mockSigner = new MockSigner();
+		const s = new Syncable('foo', {
+			bar: 'baz',
+		});
+		s.setSigner(mockSigner);
+		const j = s.toJSON();
+		const ss = Syncable.fromJSON(j);
+		assert.equal(ss.m['id'], 'foo');
+		assert.equal(ss.m['data']['bar'], 'baz');
+		assert.equal(Automerge.getHistory(ss.m).length, 1);
+	});
+
+	it('sync_sign_and_wrap', () => {
+		const mockSigner = new MockSigner();
+		const s = new Syncable('foo', {
+			bar: 'baz',
+		});
+		s.setSigner(mockSigner);
+		s.onwrap = (e) => {
+			const j = e.toJSON();
+			const v = JSON.parse(j);
+			assert.deepEqual(v.payload, e.o.payload);
+
+		}
+		s.sign();
+	});
+	it('sync_verify_success', async () => {
+		const pksa = fs.readFileSync(__dirname + '/privatekeys.asc');
+		const pks = await pgp.key.readArmored(pksa);
+		await pks.keys[0].decrypt('merman');
+		await pks.keys[1].decrypt('beastman');
+
+		const pubksa = fs.readFileSync(__dirname + '/publickeys.asc');
+		const pubks = await pgp.key.readArmored(pubksa);
+
+		const oneStore = new MockKeyStore(pks.keys[0], pubks.keys);
+		const twoStore = new MockKeyStore(pks.keys[1], pubks.keys);
+		const threeStore = new MockKeyStore(pks.keys[2], [pubks.keys[0], pubks.keys[2]]);
+
+		const oneSigner = new PGPSigner(oneStore);
+		const twoSigner = new PGPSigner(twoStore);
+		const threeSigner = new PGPSigner(threeStore);
+
+		const x = new Syncable('foo', {
+			bar: 'baz',		  
+		});
+		x.setSigner(oneSigner);
+
+		// TODO: make this look better
+		x.onwrap = (e) => {
+			let updateData = new ArgPair('bar', 'xyzzy');
+			x.update([updateData], 'change one');
+
+			x.onwrap = (e) => {
+				x.setSigner(twoSigner);
+				updateData = new ArgPair('bar', 42);
+				x.update([updateData], 'change two');
+
+				x.onwrap = (e) => {
+					const p = e.unwrap();
+					p.setSigner(twoSigner);
+					p.onauthenticate = (v) => {
+						assert(v);
+					}
+					p.authenticate();
+				}
+
+				x.sign();
+			};
+
+			x.sign();
+		}
+
+		x.sign();
+
+	});
+
+	it('sync_verify_fail', async () => {
+		const pksa = fs.readFileSync(__dirname + '/privatekeys.asc');
+		const pks = await pgp.key.readArmored(pksa);
+		await pks.keys[0].decrypt('merman');
+		await pks.keys[1].decrypt('beastman');
+
+		const pubksa = fs.readFileSync(__dirname + '/publickeys.asc');
+		const pubks = await pgp.key.readArmored(pubksa);
+
+		const oneStore = new MockKeyStore(pks.keys[0], pubks.keys);
+		const twoStore = new MockKeyStore(pks.keys[1], pubks.keys);
+		const threeStore = new MockKeyStore(pks.keys[2], [pubks.keys[0], pubks.keys[2]]);
+
+		const oneSigner = new PGPSigner(oneStore);
+		const twoSigner = new PGPSigner(twoStore);
+		const threeSigner = new PGPSigner(threeStore);
+
+		const x = new Syncable('foo', {
+			bar: 'baz',		  
+		});
+		x.setSigner(oneSigner);
+
+		// TODO: make this look better
+		x.onwrap = (e) => {
+			let updateData = new ArgPair('bar', 'xyzzy');
+			x.update([updateData], 'change one');
+
+			x.onwrap = (e) => {
+				x.setSigner(twoSigner);
+				updateData = new ArgPair('bar', 42);
+				x.update([updateData], 'change two');
+
+				x.onwrap = (e) => {
+					const p = e.unwrap();
+					p.setSigner(threeSigner);
+					p.onauthenticate = (v) => {
+						assert(!v);
+					}
+					p.authenticate();
+				}
+
+				x.sign();
+			};
+
+			x.sign();
+		}
+
+		x.sign();
+
+	});
+
+	xit('sync_verify_shallow_tricked', async () => {
+		const pksa = fs.readFileSync(__dirname + '/privatekeys.asc');
+		const pks = await pgp.key.readArmored(pksa);
+		await pks.keys[0].decrypt('merman');
+		await pks.keys[1].decrypt('beastman');
+
+		const pubksa = fs.readFileSync(__dirname + '/publickeys.asc');
+		const pubks = await pgp.key.readArmored(pubksa);
+
+		const oneStore = new MockKeyStore(pks.keys[0], pubks.keys);
+		const twoStore = new MockKeyStore(pks.keys[1], pubks.keys);
+		const threeStore = new MockKeyStore(pks.keys[2], [pubks.keys[0], pubks.keys[2]]);
+
+		const oneSigner = new PGPSigner(oneStore);
+		const twoSigner = new PGPSigner(twoStore);
+		const threeSigner = new PGPSigner(threeStore);
+
+		const x = new Syncable('foo', {
+			bar: 'baz',		  
+		});
+		x.setSigner(twoSigner);
+
+		// TODO: make this look better
+		x.onwrap = (e) => {
+			let updateData = new ArgPair('bar', 'xyzzy');
+			x.update([updateData], 'change one');
+
+			x.onwrap = (e) => {
+				updateData = new ArgPair('bar', 42);
+				x.update([updateData], 'change two');
+				x.setSigner(oneSigner);
+
+				x.onwrap = (e) => {
+					const p = e.unwrap();
+					p.setSigner(threeSigner);
+					p.onauthenticate = (v) => {
+						assert(v);
+						p.onauthenticate = (v) => {
+							assert(!v);
+						}
+						p.authenticate(true);
+					}
+					p.authenticate();
+				}
+
+				x.sign();
+			};
+
+			x.sign();
+		}
+
+		x.sign();
+
+	});
+});

apps/cic-meta/tests/3_transport.ts

@@ -0,0 +1,14 @@
+import * as assert from 'assert';
+
+import { MockPubSub, MockConsumer } from './mock';
+
+describe('transport', () => {
+	it('pub_sub', () => {
+		const c = new MockConsumer();
+		const ps = new MockPubSub('foo', c);
+		ps.pub('foo');	
+		ps.pub('bar');
+		ps.flush();
+		assert.deepEqual(c.omnoms, ['foo', 'bar']);
+	});
+});

apps/cic-meta/tests/4_auth.ts

@@ -0,0 +1,46 @@
+import assert = require('assert');
+import pgp = require('openpgp');
+import crypto  = require('crypto');
+
+import { Syncable, ArgPair } from '../src/sync';
+
+import { MockKeyStore, MockSignable } from './mock';
+
+import { PGPSigner } from '../src/auth';
+
+
+describe('auth', async () => {
+	await it('digest', async () => {
+		const opts = {
+			userIds: [
+				{
+					name: 'John Marston',
+					email: 'red@dead.com',
+				},
+			],
+			numBits: 2048,
+			passphrase: 'foo',
+		};
+		const pkgen = await pgp.generateKey(opts);
+		const pka = pkgen.privateKeyArmored;
+		const pks = await pgp.key.readArmored(pka);
+		await pks.keys[0].decrypt('foo');
+		const pubka = pkgen.publicKeyArmored;
+		const pubks = await pgp.key.readArmored(pubka);
+		const keyStore = new MockKeyStore(pks.keys[0], pubks.keys);
+		const s = new PGPSigner(keyStore);
+
+		const message = await pgp.cleartext.fromText('foo');
+		s.onverify = (ok) => {
+			assert(ok);
+		}
+		s.onsign = (signature) => {
+			s.onverify((v) => {
+				console.log('bar', v);
+			});
+			s.verify('foo', signature);
+		}
+		
+		await s.sign('foo');
+	});
+});	

apps/cic-meta/tests/999_functional.ts

@@ -0,0 +1,47 @@
+import * as assert from 'assert';
+import * as pgp from 'openpgp';
+
+import { Dispatcher } from '../src/dispatch';
+import { User } from '../src/assets/user';
+import { PGPSigner, KeyStore } from '../src/auth';
+import { SubConsumer } from '../src/transport';
+
+import { MockStore, MockPubSub, MockConsumer, MockKeyStore } from './mock';
+
+async function createKeyStore() {
+	const opts = {
+		userIds: [
+			{
+				name: 'John Marston',
+				email: 'red@dead.com',
+			},
+		],
+		numBits: 2048,
+		passphrase: 'foo',
+	};
+	const pkgen = await pgp.generateKey(opts);
+	const pka = pkgen.privateKeyArmored;
+	const pks = await pgp.key.readArmored(pka);
+	await pks.keys[0].decrypt('foo');
+	return new MockKeyStore(pks.keys[0], []);
+}
+
+describe('fullchain', async () => {
+	it('dispatch_and_publish_user', async () => {
+		const g = await createKeyStore();
+		const n = new PGPSigner(g);
+		const u = new User('u1', {});
+		u.setSigner(n);
+		u.setName('Nico', 'Bellic');
+		const s = new MockStore('fooStore');
+		const c = new MockConsumer();
+		const p = new MockPubSub('fooPubSub', c);
+		const d = new Dispatcher(s, p);
+		u.onwrap = (e) => {
+			d.add(u.id, e);
+			d.sync(0);
+			assert.equal(p.pubs.length, 1);
+		};
+		u.sign();
+	});
+});

apps/cic-meta/tests/mock.ts

@@ -0,0 +1,150 @@
+import * as crypto from 'crypto';
+
+import { Signable, Signature, KeyStore } from '../src/auth';
+import { Store } from '../src/store';
+import { PubSub, SubConsumer } from '../src/transport';
+import { Syncable } from '../src/sync';
+
+class MockStore implements Store {
+	
+	contents:	Object
+	name:		string
+
+	constructor(name:string) {
+		this.name = name;
+		this.contents = {};
+	}
+
+	public put(k:string, v:Syncable, existsOk = false) {
+		if (!existsOk && this.contents[k] !== undefined) {
+			throw '"' + k + '" already exists in store ' + this.name;
+		} 
+		this.contents[k] = v;
+	}
+
+	public get(k:string): Syncable {
+		return this.contents[k];
+	}
+
+	public delete(k:string) {
+		delete this.contents[k];
+	}
+}
+
+class MockSigner {
+	onsign: (string) => void
+	onverify: (boolean) => void
+	public verify(src:string, signature:Signature) {
+		return true;
+	}
+
+	public sign(s:string):boolean {
+		this.onsign('there would be a signature here');
+		return true;
+	}
+
+	public prepare(m:Signable):boolean {
+		return true;
+	}
+
+	public fingerprint():string {
+		return '';
+	}
+}
+
+class MockConsumer implements SubConsumer {
+
+	omnoms:	Array<string>
+
+	constructor() {
+		this.omnoms = Array<string>();
+	}
+
+	public post(v:string) {
+		this.omnoms.push(v);
+	}
+}
+
+class MockPubSub implements PubSub {
+
+	pubs:		Array<string>
+	consumer:	SubConsumer
+
+	constructor(name:string, consumer:SubConsumer) {
+		this.pubs = Array<string>();	
+		this.consumer = consumer;
+	}
+	
+	public pub(v:string): boolean {
+		this.pubs.push(v);
+		return true;
+	}
+
+	public flush() {
+		while (this.pubs.length > 0) {
+			const s = this.pubs.shift();
+			this.consumer.post(s);
+		}
+	}
+
+	public close() {
+	}
+}
+
+class MockSignable implements Signable {
+
+	src:	string
+	dst:	string
+
+	constructor(src:string) {
+		this.src = src;
+	}
+
+	public digest():string {
+		const h = crypto.createHash('sha256');
+		h.update(this.src);
+		this.dst= h.digest('hex');
+		return this.dst;
+	}
+
+}
+
+class MockKeyStore implements KeyStore {
+
+	pk: any
+	pubks: Array<any>
+
+	constructor(pk:any, pubks:Array<any>) {
+		this.pk = pk;
+		this.pubks = pubks;	
+	}
+
+	public getPrivateKey(): any {
+		return this.pk;
+	}
+
+	public getTrustedKeys(): Array<any> {
+		return this.pubks;
+	}
+
+	public getTrustedActiveKeys(): Array<any> {
+		return [];
+	}
+
+	public getEncryptKeys(): Array<any> {
+		return [];
+	}
+
+	public getFingerprint(): string {
+		return '';
+	}
+}
+
+export {
+	MockStore,
+	MockPubSub,
+	MockConsumer,
+	MockSignable,
+	MockKeyStore,
+	MockSigner,
+};

apps/cic-meta/tests/server.ts

@@ -1,10 +1,13 @@
+import Automerge = require('automerge');
 import assert = require('assert');
 import fs = require('fs');
 import pgp = require('openpgp');
 import sqlite = require('sqlite3');
 
 import * as handlers from '../scripts/server/handlers';
-import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer, SqliteAdapter } from 'crdt-meta';
+import { Envelope, Syncable, ArgPair } from '../src/sync';
+import { PGPKeyStore, PGPSigner, KeyStore, Signer } from '../src/auth';
+import { SqliteAdapter } from '../src/db';
 
 function createKeystore() {
 	const pksa = fs.readFileSync(__dirname + '/privatekeys.asc', 'utf-8');

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -33,9 +33,7 @@ elif args.v:
 
 config = confini.Config(args.c, args.env_prefix)
 config.process()
-config.add(args.q, '_CELERY_QUEUE', True)
 config.censor('PASSWORD', 'DATABASE')
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 dsn = dsn_from_config(config)

apps/cic-notify/cic_notify/version.py

@@ -6,10 +6,11 @@ import time
 import semver
 
 # local imports
+from cic_notify.error import PleaseCommitFirstError
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.4')
+version = (0, 4, 0, 'alpha.3')
 
 version_object = semver.VersionInfo(
         major=version[0],
@@ -17,4 +18,27 @@ version_object = semver.VersionInfo(
         patch=version[2],
         prerelease=version[3],
         )
+
 version_string = str(version_object)
+
+
+def git_hash():
+    import subprocess
+
+    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+    return git_hash_brief
+
+
+try:
+    version_git = git_hash()
+    version_string += '+build.{}'.format(version_git)
+except FileNotFoundError:
+    time_string_pair = str(time.time()).split('.')
+    version_string += '+build.{}{:<09d}'.format(
+        time_string_pair[0],
+        int(time_string_pair[1]),
+    )
+logg.info(f'Final version string will be {version_string}')
+
+__version_string__ = version_string

apps/cic-notify/setup.cfg

@@ -1,5 +1,6 @@
 [metadata]
 name = cic-notify
+version= 0.4.0a3
 description = CIC notifications service
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-notify/setup.py

@@ -1,31 +1,9 @@
 # standard imports
-import logging
-import subprocess
-import time
 from setuptools import setup
 
+# third-party imports
+
 # local imports
-from cic_notify.version import version_string
-
-logg = logging.getLogger()
-
-
-def git_hash():
-    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-    return git_hash_brief
-
-
-try:
-    version_git = git_hash()
-    version_string += '+build.{}'.format(version_git)
-except FileNotFoundError:
-    time_string_pair = str(time.time()).split('.')
-    version_string += '+build.{}{:<09d}'.format(
-            time_string_pair[0],
-            int(time_string_pair[1]),
-            )
-logg.info(f'Final version string will be {version_string}')
 
 
 requirements = []
@@ -47,6 +25,6 @@ while True:
 test_requirements_file.close()
 
 setup(
-    version=version_string,
     install_requires=requirements,
-    tests_require=test_requirements)
+    tests_require=test_requirements,
+)

apps/cic-ussd/.config/app.ini

@@ -11,14 +11,7 @@ REGION=KE
 
 [ussd]
 MENU_FILE=/usr/src/data/ussd_menu.json
-user = 
-pass = 
 
 [statemachine]
 STATES=/usr/src/cic-ussd/states/
 TRANSITIONS=/usr/src/cic-ussd/transitions/
-
-[client]
-host =
-port = 
-ssl =

apps/cic-ussd/.config/redis.ini

@@ -1,6 +1,6 @@
 [celery]
-BROKER_URL=redis://
-RESULT_URL=redis://
+BROKER_URL=redis://redis:6379
+RESULT_URL=redis://redis:6379
 
 [redis]
 HOSTNAME=redis

apps/cic-ussd/cic_ussd/account.py

@@ -8,12 +8,12 @@ from cic_types.processor import generate_metadata_pointer
 
 # local imports
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.user import User
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.redis import get_cached_data
 
 
-def define_account_tx_metadata(user: Account):
+def define_account_tx_metadata(user: User):
     # get sender metadata
     identifier = blockchain_address_to_metadata_pointer(
         blockchain_address=user.blockchain_address

apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py

@@ -1,4 +1,4 @@
-"""Create account table
+"""Create user table
 
 Revision ID: f289e8510444
 Revises: 
@@ -17,7 +17,7 @@ depends_on = None
 
 
 def upgrade():
-    op.create_table('account',
+    op.create_table('user',
                     sa.Column('id', sa.Integer(), nullable=False),
                     sa.Column('blockchain_address', sa.String(), nullable=False),
                     sa.Column('phone_number', sa.String(), nullable=False),
@@ -29,11 +29,11 @@ def upgrade():
                     sa.Column('updated', sa.DateTime(), nullable=False),
                     sa.PrimaryKeyConstraint('id')
                     )
-    op.create_index(op.f('ix_account_phone_number'), 'account', ['phone_number'], unique=True)
-    op.create_index(op.f('ix_account_blockchain_address'), 'account', ['blockchain_address'], unique=True)
+    op.create_index(op.f('ix_user_phone_number'), 'user', ['phone_number'], unique=True)
+    op.create_index(op.f('ix_user_blockchain_address'), 'user', ['blockchain_address'], unique=True)
 
 
 def downgrade():
-    op.drop_index(op.f('ix_account_blockchain_address'), table_name='account')
-    op.drop_index(op.f('ix_account_phone_number'), table_name='account')
-    op.drop_table('account')
+    op.drop_index(op.f('ix_user_blockchain_address'), table_name='user')
+    op.drop_index(op.f('ix_user_phone_number'), table_name='user')
+    op.drop_table('user')

apps/cic-ussd/cic_ussd/db/models/user.py

@@ -16,12 +16,12 @@ class AccountStatus(IntEnum):
     RESET = 4
 
 
-class Account(SessionBase):
+class User(SessionBase):
     """
     This class defines a user record along with functions responsible for hashing the user's corresponding password and
      subsequently verifying a password's validity given an input to compare against the persisted hash.
     """
-    __tablename__ = 'account'
+    __tablename__ = 'user'
 
     blockchain_address = Column(String)
     phone_number = Column(String)
@@ -38,7 +38,7 @@ class Account(SessionBase):
         self.account_status = AccountStatus.PENDING.value
 
     def __repr__(self):
-        return f'<Account: {self.blockchain_address}>'
+        return f'<User: {self.blockchain_address}>'
 
     def create_password(self, password):
         """This method takes a password value and hashes the value before assigning it to the corresponding

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -1,126 +1,7 @@
-# standard imports
-import json
-import logging
-import os
-from typing import Dict, Union
-
-# third-part imports
-import requests
-from cic_types.models.person import generate_metadata_pointer, Person
-
-# local imports
-from cic_ussd.metadata import make_request
-from cic_ussd.metadata.signer import Signer
-from cic_ussd.redis import cache_data
-from cic_ussd.error import MetadataStoreError
-
-
-logg = logging.getLogger().getChild(__name__)
-
-
 class Metadata:
     """
-    :cvar base_url: The base url or the metadata server.
-    :type base_url: str
+    :cvar base_url:
+    :type base_url:
     """
 
     base_url = None
-
-
-def metadata_http_error_handler(result: requests.Response):
-    """ This function handles and appropriately raises errors from http requests interacting with the metadata server.
-    :param result: The response object from a http request.
-    :type result: requests.Response
-    """
-    status_code = result.status_code
-
-    if 100 <= status_code < 200:
-        raise MetadataStoreError(f'Informational errors: {status_code}, reason: {result.reason}')
-
-    elif 300 <= status_code < 400:
-        raise MetadataStoreError(f'Redirect Issues: {status_code}, reason: {result.reason}')
-
-    elif 400 <= status_code < 500:
-        raise MetadataStoreError(f'Client Error: {status_code}, reason: {result.reason}')
-
-    elif 500 <= status_code < 600:
-        raise MetadataStoreError(f'Server Error: {status_code}, reason: {result.reason}')
-
-
-class MetadataRequestsHandler(Metadata):
-
-    def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'):
-        """
-        :param cic_type: The salt value with which to hash a specific metadata identifier.
-        :type cic_type: str
-        :param engine: Encryption used for sending data to the metadata server.
-        :type engine: str
-        :param identifier: A unique element of data in bytes necessary for creating a metadata pointer.
-        :type identifier: bytes
-        """
-        self.cic_type = cic_type
-        self.engine = engine
-        self.headers = {
-            'X-CIC-AUTOMERGE': 'server',
-            'Content-Type': 'application/json'
-        }
-        self.identifier = identifier
-        self.metadata_pointer = generate_metadata_pointer(
-            identifier=self.identifier,
-            cic_type=self.cic_type
-        )
-        if self.base_url:
-            self.url = os.path.join(self.base_url, self.metadata_pointer)
-
-    def create(self, data: Union[Dict, str]):
-        """ This function is responsible for posting data to the metadata server with a corresponding metadata pointer
-        for storage.
-        :param data: The data to be stored in the metadata server.
-        :type data: dict|str
-        """
-        data = json.dumps(data).encode('utf-8')
-        result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
-        metadata_http_error_handler(result=result)
-        metadata = result.content
-        self.edit(data=metadata)
-
-    def edit(self, data: bytes):
-        """ This function is responsible for editing data in the metadata server corresponding to a unique pointer.
-        :param data: The data to be edited in the metadata server.
-        :type data: bytes
-        """
-        cic_meta_signer = Signer()
-        signature = cic_meta_signer.sign_digest(data=data)
-        algorithm = cic_meta_signer.get_operational_key().get('algo')
-        decoded_data = data.decode('utf-8')
-        formatted_data = {
-            'm': data.decode('utf-8'),
-            's': {
-                'engine': self.engine,
-                'algo': algorithm,
-                'data': signature,
-                'digest': json.loads(data).get('digest'),
-            }
-        }
-        formatted_data = json.dumps(formatted_data).encode('utf-8')
-        result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
-        logg.info(f'signed metadata submission status: {result.status_code}.')
-        metadata_http_error_handler(result=result)
-        try:
-            decoded_identifier = self.identifier.decode("utf-8")
-        except UnicodeDecodeError:
-            decoded_identifier = self.identifier.hex()
-        logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {decoded_data}.')
-
-    def query(self):
-        """This function is responsible for querying the metadata server for data corresponding to a unique pointer."""
-        result = make_request(method='GET', url=self.url)
-        metadata_http_error_handler(result=result)
-        response_data = result.content
-        data = json.loads(response_data.decode('utf-8'))
-        if result.status_code == 200 and self.cic_type == 'cic.person':
-            person = Person()
-            deserialized_person = person.deserialize(person_data=json.loads(data))
-            data = json.dumps(deserialized_person.serialize())
-            cache_data(self.metadata_pointer, data=data)
-            logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -1,12 +0,0 @@
-# standard imports
-
-# third-party imports
-
-# local imports
-from .base import MetadataRequestsHandler
-
-
-class PersonMetadata(MetadataRequestsHandler):
-
-    def __init__(self, identifier: bytes):
-        super().__init__(cic_type='cic.person', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -1,13 +1,85 @@
 # standard imports
+import json
 import logging
+import os
 
 # external imports
+import requests
+from cic_types.models.person import generate_metadata_pointer
+from cic_ussd.metadata import make_request
+from cic_ussd.metadata.signer import Signer
 
 # local imports
-from .base import MetadataRequestsHandler
+from cic_ussd.error import MetadataStoreError
+from .base import Metadata
+
+logg = logging.getLogger().getChild(__name__)
 
 
-class PhonePointerMetadata(MetadataRequestsHandler):
+class PhonePointerMetadata(Metadata):
+
+    def __init__(self, identifier: bytes, engine: str):
+        """
+        :param identifier:
+        :type identifier:
+        """
+    
+        self.headers = {
+            'X-CIC-AUTOMERGE': 'server',
+            'Content-Type': 'application/json'
+        }
+        self.identifier = identifier
+        self.metadata_pointer = generate_metadata_pointer(
+                identifier=self.identifier,
+                cic_type=':cic.phone'
+        )
+        if self.base_url:
+            self.url = os.path.join(self.base_url, self.metadata_pointer)
+
+        self.engine = engine
+
+
+    def create(self, data: str):
+        try:
+            data = json.dumps(data).encode('utf-8')
+            result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
+            metadata = result.content
+            logg.debug('data {} meta {} resp {} stats {}'.format(data, metadata, result.reason, result.status_code))
+            self.edit(data=metadata, engine=self.engine)
+            result.raise_for_status()
+        except requests.exceptions.HTTPError as error:
+            raise MetadataStoreError(error)
+
+
+    def edit(self, data: bytes, engine: str):
+        """
+        :param data:
+        :type data:
+        :param engine:
+        :type engine:
+        :return:
+        :rtype:
+        """
+        cic_meta_signer = Signer()
+        signature = cic_meta_signer.sign_digest(data=data)
+        algorithm = cic_meta_signer.get_operational_key().get('algo')
+        decoded_data = data.decode('utf-8')
+        formatted_data = {
+            'm': decoded_data,
+            's': {
+                'engine': engine,
+                'algo': algorithm,
+                'data': signature,
+                'digest': json.loads(data).get('digest'),
+            }
+        }
+        formatted_data = json.dumps(formatted_data).encode('utf-8')
+
+        try:
+            result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
+            logg.debug(f'signed phone pointer metadata submission status: {result.status_code}.')
+            result.raise_for_status()
+            logg.info('phone {} metadata pointer {} set to {}'.format(self.identifier.decode('utf-8'), self.metadata_pointer, decoded_data))
+        except requests.exceptions.HTTPError as error:
+            raise MetadataStoreError(error)
 
-    def __init__(self, identifier: bytes):
-        super().__init__(cic_type='cic.msisdn', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/user.py

@@ -0,0 +1,100 @@
+# standard imports
+import json
+import logging
+import os
+
+# third-party imports
+import requests
+from cic_types.models.person import generate_metadata_pointer, Person
+
+# local imports
+from cic_ussd.chain import Chain
+from cic_ussd.metadata import make_request
+from cic_ussd.metadata.signer import Signer
+from cic_ussd.redis import cache_data
+from cic_ussd.error import MetadataStoreError
+from .base import Metadata
+
+logg = logging.getLogger()
+
+
+class UserMetadata(Metadata):
+    
+
+    def __init__(self, identifier: bytes):
+        """
+        :param identifier:
+        :type identifier:
+        """
+        self.headers = {
+            'X-CIC-AUTOMERGE': 'server',
+            'Content-Type': 'application/json'
+        }
+        self.identifier = identifier
+        self.metadata_pointer = generate_metadata_pointer(
+                identifier=self.identifier,
+                cic_type=':cic.person'
+        )
+        if self.base_url:
+            self.url = os.path.join(self.base_url, self.metadata_pointer)
+
+    def create(self, data: dict):
+        try:
+            data = json.dumps(data).encode('utf-8')
+            result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
+            metadata = result.content
+            self.edit(data=metadata, engine='pgp')
+            logg.info(f'Get sign material response status: {result.status_code}')
+            result.raise_for_status()
+        except requests.exceptions.HTTPError as error:
+            raise MetadataStoreError(error)
+
+    def edit(self, data: bytes, engine: str):
+        """
+        :param data:
+        :type data:
+        :param engine:
+        :type engine:
+        :return:
+        :rtype:
+        """
+        cic_meta_signer = Signer()
+        signature = cic_meta_signer.sign_digest(data=data)
+        algorithm = cic_meta_signer.get_operational_key().get('algo')
+        formatted_data = {
+            'm': data.decode('utf-8'),
+            's': {
+                'engine': engine,
+                'algo': algorithm,
+                'data': signature,
+                'digest': json.loads(data).get('digest'),
+            }
+        }
+        formatted_data = json.dumps(formatted_data).encode('utf-8')
+
+        try:
+            result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
+            logg.debug(f'signed user metadata submission status: {result.status_code}.')
+            result.raise_for_status()
+        except requests.exceptions.HTTPError as error:
+            raise MetadataStoreError(error)
+
+    def query(self):
+        result = make_request(method='GET', url=self.url)
+        status = result.status_code
+        logg.info(f'Get latest data status: {status}')
+        try:
+            if status == 200:
+                response_data = result.content
+                data = json.loads(response_data.decode())
+
+                # validate data
+                person = Person()
+                deserialized_person = person.deserialize(person_data=json.loads(data))
+
+                cache_data(key=self.metadata_pointer, data=json.dumps(deserialized_person.serialize()))
+            elif status == 404:
+                logg.info('The data is not available and might need to be added.')
+            result.raise_for_status()
+        except requests.exceptions.HTTPError as error:
+            raise MetadataNotFoundError(error)

apps/cic-ussd/cic_ussd/operations.py

@@ -10,7 +10,7 @@ from tinydb.table import Document
 from typing import Optional
 
 # local imports
-from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.user import User
 from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.menu.ussd_menu import UssdMenu
@@ -143,10 +143,10 @@ def get_account_status(phone_number) -> str:
     :return: The user account status.
     :rtype: str
     """
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
+    user = User.session.query(User).filter_by(phone_number=phone_number).first()
     status = user.get_account_status()
-    Account.session.add(user)
-    Account.session.commit()
+    User.session.add(user)
+    User.session.commit()
 
     return status
 
@@ -269,12 +269,12 @@ def cache_account_creation_task_id(phone_number: str, task_id: str):
     redis_cache.persist(name=task_id)
 
 
-def process_current_menu(ussd_session: Optional[dict], user: Account, user_input: str) -> Document:
+def process_current_menu(ussd_session: Optional[dict], user: User, user_input: str) -> Document:
     """This function checks user input and returns a corresponding ussd menu
     :param ussd_session: An in db ussd session object.
     :type ussd_session: UssdSession
     :param user: A user object.
-    :type user: Account
+    :type user: User
     :param user_input: The user's input.
     :type user_input: str
     :return: An in memory ussd menu object.
@@ -324,7 +324,7 @@ def process_menu_interaction_requests(chain_str: str,
 
     else:
         # get user
-        user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
+        user = User.session.query(User).filter_by(phone_number=phone_number).first()
 
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
@@ -390,10 +390,10 @@ def reset_pin(phone_number: str) -> str:
     :return: The status of the pin reset.
     :rtype: str
     """
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
+    user = User.session.query(User).filter_by(phone_number=phone_number).first()
     user.reset_account_pin()
-    Account.session.add(user)
-    Account.session.commit()
+    User.session.add(user)
+    User.session.commit()
 
     response = f'Pin reset for user {phone_number} is successful!'
     return response

apps/cic-ussd/cic_ussd/phone_number.py

@@ -5,7 +5,7 @@ from typing import Optional
 import phonenumbers
 
 # local imports
-from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.user import User
 
 
 def process_phone_number(phone_number: str, region: str):
@@ -30,14 +30,14 @@ def process_phone_number(phone_number: str, region: str):
     return parsed_phone_number
 
 
-def get_user_by_phone_number(phone_number: str) -> Optional[Account]:
+def get_user_by_phone_number(phone_number: str) -> Optional[User]:
     """This function queries the database for a user based on the provided phone number.
     :param phone_number: A valid phone number.
     :type phone_number: str
     :return: A user object matching a given phone number
-    :rtype: Account|None
+    :rtype: User|None
     """
     # consider adding region to user's metadata
     phone_number = process_phone_number(phone_number=phone_number, region='KE')
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
+    user = User.session.query(User).filter_by(phone_number=phone_number).first()
     return user

apps/cic-ussd/cic_ussd/processor.py

@@ -13,7 +13,7 @@ from tinydb.table import Document
 from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement
 from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.account import AccountStatus, Account
+from cic_ussd.db.models.user import AccountStatus, User
 from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.error import MetadataNotFoundError
 from cic_ussd.menu.ussd_menu import UssdMenu
@@ -28,13 +28,13 @@ from cic_types.models.person import generate_metadata_pointer, get_contact_data_
 logg = logging.getLogger(__name__)
 
 
-def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
+def process_pin_authorization(display_key: str, user: User, **kwargs) -> str:
     """
     This method provides translation for all ussd menu entries that follow the pin authorization pattern.
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param user: The user in a running USSD session.
-    :type user: Account
+    :type user: User
     :param kwargs: Any additional information required by the text values in the internationalization files.
     :type kwargs
     :return: A string value corresponding the ussd menu's text value.
@@ -55,13 +55,13 @@ def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
         )
 
 
-def process_exit_insufficient_balance(display_key: str, user: Account, ussd_session: dict):
+def process_exit_insufficient_balance(display_key: str, user: User, ussd_session: dict):
     """This function processes the exit menu letting users their account balance is insufficient to perform a specific
     transaction.
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :type user: User
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
@@ -90,12 +90,12 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     )
 
 
-def process_exit_successful_transaction(display_key: str, user: Account, ussd_session: dict):
+def process_exit_successful_transaction(display_key: str, user: User, ussd_session: dict):
     """This function processes the exit menu after a successful initiation for a transfer of tokens.
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :type user: User
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
@@ -118,11 +118,11 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     )
 
 
-def process_transaction_pin_authorization(user: Account, display_key: str, ussd_session: dict):
+def process_transaction_pin_authorization(user: User, display_key: str, ussd_session: dict):
     """This function processes pin authorization where making a transaction is concerned. It constructs a
     pre-transaction response menu that shows the details of the transaction.
     :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :type user: User
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param ussd_session: The USSD session determining what user data needs to be extracted and added to the menu's
@@ -151,7 +151,7 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     )
 
 
-def process_account_balances(user: Account, display_key: str, ussd_session: dict):
+def process_account_balances(user: User, display_key: str, ussd_session: dict):
     """
     :param user:
     :type user:
@@ -205,7 +205,7 @@ def format_transactions(transactions: list, preferred_language: str):
         return formatted_transactions
 
 
-def process_display_user_metadata(user: Account, display_key: str):
+def process_display_user_metadata(user: User, display_key: str):
     """
     :param user:
     :type user:
@@ -235,10 +235,10 @@ def process_display_user_metadata(user: Account, display_key: str):
             products=products
         )
     else:
-        raise MetadataNotFoundError(f'Expected person metadata but found none in cache for key: {key}')
+        raise MetadataNotFoundError(f'Expected user metadata but found none in cache for key: {user.blockchain_address}')
 
 
-def process_account_statement(user: Account, display_key: str, ussd_session: dict):
+def process_account_statement(user: User, display_key: str, ussd_session: dict):
     """
     :param user:
     :type user:
@@ -301,12 +301,12 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
         )
 
 
-def process_start_menu(display_key: str, user: Account):
+def process_start_menu(display_key: str, user: User):
     """This function gets data on an account's balance and token in order to append it to the start of the start menu's
     title. It passes said arguments to the translation function and returns the appropriate corresponding text from the
     translation files.
     :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :type user: User
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :return: Corresponding translation text response
@@ -331,11 +331,11 @@ def process_start_menu(display_key: str, user: Account):
     operational_balance = compute_operational_balance(balances=balances_data)
 
     # retrieve and cache account's metadata
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
+    s_query_user_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.query_user_metadata',
         [blockchain_address]
     )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
+    s_query_user_metadata.apply_async(queue='cic-ussd')
 
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
@@ -361,13 +361,13 @@ def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession:
     return last_ussd_session
 
 
-def process_request(user_input: str, user: Account, ussd_session: Optional[dict] = None) -> Document:
+def process_request(user_input: str, user: User, ussd_session: Optional[dict] = None) -> Document:
     """This function assesses a request based on the user from the request comes, the session_id and the user's
     input. It determines whether the request translates to a return to an existing session by checking whether the
     provided  session id exists in the database or whether the creation of a new ussd session object is warranted.
     It then returns the appropriate ussd menu text values.
     :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :type user: User
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :param ussd_session: A JSON serialized in-memory ussd session object
@@ -389,11 +389,14 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
                 identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
                 cic_type='cic.person'
             )
-            person_metadata = get_cached_data(key=key)
+            logg.debug(f'METADATA POINTER: {key}')
+            user_metadata = get_cached_data(key=key)
+            logg.debug(f'METADATA: {user_metadata}')
 
             if last_ussd_session:
                 # get last state
                 last_state = last_ussd_session.state
+                logg.debug(f'LAST USSD SESSION STATE: {last_state}')
                 # if last state is account_creation_prompt and metadata exists, show start menu
                 if last_state in [
                     'account_creation_prompt',
@@ -402,7 +405,7 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
                     'exit_invalid_new_pin',
                     'exit_pin_mismatch',
                     'exit_invalid_request'
-                ] and person_metadata is not None:
+                ] and user_metadata is not None:
                     return UssdMenu.find_by_name(name='start')
                 else:
                     return UssdMenu.find_by_name(name=last_state)
@@ -415,14 +418,14 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
                 return UssdMenu.find_by_name(name='initial_pin_entry')
 
 
-def next_state(ussd_session: dict, user: Account, user_input: str) -> str:
+def next_state(ussd_session: dict, user: User, user_input: str) -> str:
     """This function navigates the state machine based on the ussd session object and user inputs it receives.
     It checks the user input and provides the successive state in the state machine. It then updates the session's
     state attribute with the new state.
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :type user: User
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :return: A string value corresponding the successive give a specific state in the state machine.
@@ -438,7 +441,7 @@ def custom_display_text(
         display_key: str,
         menu_name: str,
         ussd_session: dict,
-        user: Account) -> str:
+        user: User) -> str:
     """This function extracts the appropriate session data based on the current menu name. It then inserts them as
     keywords in the i18n function.
     :param display_key: The path in the translation files defining an appropriate ussd response
@@ -446,7 +449,7 @@ def custom_display_text(
     :param menu_name: The name by which a specific menu can be identified.
     :type menu_name: str
     :param user: The user in a running USSD session.
-    :type user: Account
+    :type user: User
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: A string value corresponding the ussd menu's text value.

apps/cic-ussd/cic_ussd/requests.py

@@ -10,7 +10,7 @@ from urllib.parse import urlparse, parse_qs
 from sqlalchemy import desc
 
 # local imports
-from cic_ussd.db.models.account import AccountStatus, Account
+from cic_ussd.db.models.user import AccountStatus, User
 from cic_ussd.operations import get_account_status, reset_pin
 from cic_ussd.validator import check_known_user
 
@@ -123,9 +123,9 @@ def process_locked_accounts_requests(env: dict) -> tuple:
             else:
                 limit = r[1]
 
-        locked_accounts = Account.session.query(Account.blockchain_address).filter(
-            Account.account_status == AccountStatus.LOCKED.value,
-            Account.failed_pin_attempts >= 3).order_by(desc(Account.updated)).offset(offset).limit(limit).all()
+        locked_accounts = User.session.query(User.blockchain_address).filter(
+            User.account_status == AccountStatus.LOCKED.value,
+            User.failed_pin_attempts >= 3).order_by(desc(User.updated)).offset(offset).limit(limit).all()
 
         # convert lists to scalar blockchain addresses
         locked_accounts = [blockchain_address for (blockchain_address, ) in locked_accounts]

apps/cic-ussd/cic_ussd/runnable/server.py

@@ -57,17 +57,19 @@ arg_parser.add_argument('--env-prefix',
                         help='environment prefix for variables to overwrite configuration')
 args = arg_parser.parse_args()
 
+# parse config
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+
 # define log levels
 if args.vv:
     logging.getLogger().setLevel(logging.DEBUG)
 elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
-# parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
+# log config vars
+logg.debug(config)
 
 # initialize elements
 # set up translations

apps/cic-ussd/cic_ussd/runnable/tasker.py

@@ -6,7 +6,6 @@ import tempfile
 
 # third party imports
 import celery
-import i18n
 import redis
 from confini import Config
 
@@ -34,18 +33,18 @@ arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
 arg_parser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 args = arg_parser.parse_args()
 
+# parse config
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+
 # define log levels
 if args.vv:
     logging.getLogger().setLevel(logging.DEBUG)
 elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
-# parse config
-config = Config(args.c, args.env_prefix)
-config.process()
-config.add(args.q, '_CELERY_QUEUE', True)
-config.censor('PASSWORD', 'DATABASE')
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
+logg.debug(config)
 
 # connect to database
 data_source_name = dsn_from_config(config)
@@ -78,10 +77,6 @@ if key_file_path:
     validate_presence(path=key_file_path)
 Signer.key_file_path = key_file_path
 
-# set up translations
-i18n.load_path.append(config.get('APP_LOCALE_PATH'))
-i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
-
 # set up celery
 current_app = celery.Celery(__name__)