Updates verification steps.

should have dropped the user flag

See merge request grassrootseconomics/cic-internal-integration!136

apps/cic-cache/.config/database.ini

@@ -6,3 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
+DEBUG=

apps/cic-cache/.config/test/database.ini

@@ -6,3 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
+DEBUG=

apps/cic-cache/cic_cache/db/__init__.py

@@ -2,9 +2,14 @@
 import logging
 
 # local imports
-from .list import list_transactions_mined
-from .list import list_transactions_account_mined
-from .list import add_transaction
+from .list import (
+        list_transactions_mined,
+        list_transactions_account_mined,
+        add_transaction,
+        tag_transaction,
+        add_tag,
+    )
+
 
 logg = logging.getLogger()
 

apps/cic-cache/cic_cache/db/list.py

@@ -2,8 +2,9 @@
 import logging
 import datetime
 
-# third-party imports
+# external imports
 from cic_cache.db.models.base import SessionBase
+from sqlalchemy import text
 
 logg = logging.getLogger()
 
@@ -50,7 +51,8 @@ def list_transactions_account_mined(
 
 
 def add_transaction(
-        session, tx_hash,
+        session,
+        tx_hash,
         block_number,
         tx_index,
         sender,
@@ -62,6 +64,33 @@ def add_transaction(
         success,
         timestamp,
         ):
+    """Adds a single transaction to the cache persistent storage. Sensible interpretation of all fields is the responsibility of the caller.
+
+    :param session: Persistent storage session object
+    :type session: SQLAlchemy session
+    :param tx_hash: Transaction hash
+    :type tx_hash: str, 0x-hex
+    :param block_number: Block number
+    :type block_number: int
+    :param tx_index: Transaction index in block
+    :type tx_index: int
+    :param sender: Ethereum address of effective sender
+    :type sender: str, 0x-hex
+    :param receiver: Ethereum address of effective recipient
+    :type receiver: str, 0x-hex
+    :param source_token: Ethereum address of token used by sender
+    :type source_token: str, 0x-hex
+    :param destination_token: Ethereum address of token received by recipient
+    :type destination_token: str, 0x-hex
+    :param from_value: Source token value spent in transaction
+    :type from_value: int
+    :param to_value: Destination token value received in transaction
+    :type to_value: int
+    :param success: True if code execution on network was successful
+    :type success: bool
+    :param date_block: Block timestamp
+    :type date_block: datetime
+    """
     date_block = datetime.datetime.fromtimestamp(timestamp)
     s = "INSERT INTO tx (tx_hash, block_number, tx_index, sender, recipient, source_token, destination_token, from_value, to_value, success, date_block) VALUES ('{}', {}, {}, '{}', '{}', '{}', '{}', {}, {}, {}, '{}')".format(
             tx_hash,
@@ -77,3 +106,74 @@ def add_transaction(
             date_block,
             )
     session.execute(s)
+
+
+
+def tag_transaction(
+        session,
+        tx_hash,
+        name,
+        domain=None,
+        ):
+    """Tag a single transaction with a single tag.
+
+    Tag must already exist in storage.
+
+    :param session: Persistent storage session object
+    :type session: SQLAlchemy session
+    :param tx_hash: Transaction hash
+    :type tx_hash: str, 0x-hex
+    :param name: Tag value
+    :type name: str
+    :param domain: Tag domain
+    :type domain: str
+    :raises ValueError: Unknown tag or transaction hash
+
+    """
+
+    s = text("SELECT id from tx where tx_hash = :a")
+    r = session.execute(s, {'a': tx_hash}).fetchall()
+    tx_id = r[0].values()[0]
+
+    if tx_id == None:
+        raise ValueError('unknown tx hash {}'.format(tx_hash))
+
+    #s = text("SELECT id from tag where value = :a and domain = :b")
+    if domain == None:
+        s = text("SELECT id from tag where value = :a")
+    else:
+        s = text("SELECT id from tag where value = :a and domain = :b")
+    r = session.execute(s, {'a': name, 'b': domain}).fetchall()
+    tag_id = r[0].values()[0]
+
+    logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
+
+    if tag_id == None:
+        raise ValueError('unknown tag name {} domain {}'.format(name, domain))
+
+    s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
+    r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
+
+
+def add_tag(
+        session,
+        name,
+        domain=None,
+        ):
+    """Add a single tag to storage.
+
+    :param session: Persistent storage session object
+    :type session: SQLAlchemy session
+    :param name: Tag value
+    :type name: str
+    :param domain: Tag domain
+    :type domain: str
+    :raises sqlalchemy.exc.IntegrityError: Tag already exists
+    """
+
+    s = None
+    if domain == None: 
+        s = text("INSERT INTO tag (value) VALUES (:b)")
+    else:
+        s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
+    session.execute(s, {'a': domain, 'b': name})

apps/cic-cache/cic_cache/db/migrations/default/versions/aaf2bdce7d6e_transaction_tags.py

@@ -0,0 +1,38 @@
+"""Transaction tags
+
+Revision ID: aaf2bdce7d6e
+Revises: 6604de4203e2
+Create Date: 2021-05-01 09:20:20.775082
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'aaf2bdce7d6e'
+down_revision = '6604de4203e2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'tag',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column('domain', sa.String(), nullable=True),
+            sa.Column('value', sa.String(), nullable=False),
+            )
+    op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
+
+    op.create_table(
+            'tag_tx_link',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
+            sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
+                )
+
+def downgrade():
+    op.drop_table('tag_tx_link')
+    op.drop_index('idx_tag_domain_value')
+    op.drop_table('tag')

apps/cic-cache/cic_cache/runnable/daemons/filters/base.py

@@ -1,2 +1,27 @@
-class SyncFilter:
-    pass
+class TagSyncFilter:
+    """Holds tag name and domain for an implementing filter.
+
+    :param name: Tag value 
+    :type name: str
+    :param domain: Tag domain
+    :type domain: str
+    """
+
+    def __init__(self, name, domain=None):
+        self.tag_name = name
+        self.tag_domain = domain
+
+
+    def tag(self):
+        """Return tag value/domain.
+
+        :rtype: Tuple
+        :returns: tag value/domain.
+        """
+        return (self.tag_name, self.tag_domain)
+
+
+    def __str__(self):
+        if self.tag_domain == None:
+            return self.tag_name
+        return '{}.{}'.format(self.tag_domain, self.tag_name)

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -2,7 +2,6 @@
 import logging
 
 # external imports
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.address import (
         to_checksum_address,
         )
@@ -13,17 +12,19 @@ from cic_eth_registry.error import (
         NotAContractError,
         ContractMismatchError,
         )
+from eth_erc20 import ERC20
 
 # local imports
-from .base import SyncFilter
+from .base import TagSyncFilter
 from cic_cache import db as cic_cache_db
 
 logg = logging.getLogger().getChild(__name__)
 
 
-class ERC20TransferFilter(SyncFilter):
+class ERC20TransferFilter(TagSyncFilter):
 
     def __init__(self, chain_spec):
+        super(ERC20TransferFilter, self).__init__('transfer', domain='erc20')
         self.chain_spec = chain_spec
 
 
@@ -46,6 +47,9 @@ class ERC20TransferFilter(SyncFilter):
         except RequestMismatchException:
             logg.debug('erc20 match but not a transfer, skipping')
             return False
+        except ValueError:
+            logg.debug('erc20 match but bogus data, skipping')
+            return False
 
         token_sender = tx.outputs[0]
         token_recipient = transfer_data[0]
@@ -67,7 +71,13 @@ class ERC20TransferFilter(SyncFilter):
                 tx.status == Status.SUCCESS,
                 block.timestamp,
                 )
-        #db_session.flush()
+        db_session.flush()
+        cic_cache_db.tag_transaction(
+                db_session,
+                tx.hash,
+                self.tag_name,
+                domain=self.tag_domain,
+                )
         db_session.commit()
 
         return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -7,9 +7,10 @@ import argparse
 import sys
 import re
 
-# third-party imports
+# external imports
 import confini
 import celery
+import sqlalchemy
 import rlp
 import cic_base.config
 import cic_base.log
@@ -34,7 +35,10 @@ from chainsyncer.driver import (
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
-from cic_cache.db import dsn_from_config
+from cic_cache.db import (
+        dsn_from_config,
+        add_tag,
+        )
 from cic_cache.runnable.daemons.filters import (
         ERC20TransferFilter,
         )
@@ -59,6 +63,17 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+def register_filter_tags(filters, session):
+    for f in filters:
+        tag = f.tag()
+        try:
+            add_tag(session, tag[0], domain=tag[1])
+            session.commit()
+            logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
+        except sqlalchemy.exc.IntegrityError:
+            logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
+
+
 def main():
     # Connect to blockchain with chainlib
     rpc = RPCConnection.connect(chain_spec, 'default')
@@ -98,10 +113,19 @@ def main():
 
     erc20_transfer_filter = ERC20TransferFilter(chain_spec)
 
+    filters = [
+        erc20_transfer_filter,
+            ]
+
+    session = SessionBase.create_session()
+    register_filter_tags(filters, session)
+    session.close()
+
     i = 0
     for syncer in syncers:
         logg.debug('running syncer index {}'.format(i))
-        syncer.add_filter(erc20_transfer_filter)
+        for f in filters:
+            syncer.add_filter(f)
         r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 

apps/cic-cache/config/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=
+DEBUG=0

apps/cic-cache/config/test/cic.ini

@@ -1,2 +1,4 @@
 [cic]
 registry_address = 
+chain_spec = 
+trust_address =

apps/cic-cache/config/test/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=
+DEBUG=1

apps/cic-cache/docker/Dockerfile

@@ -43,10 +43,6 @@ COPY cic-cache/config/ /usr/local/etc/cic-cache/
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
 COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
-	mkdir -p /usr/local/share/cic/solidity && \
-	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
-
 COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
 COPY cic-cache/docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh

apps/cic-cache/requirements.txt

@@ -1,12 +1,12 @@
-cic-base~=0.1.2a77
+cic-base~=0.1.2b8
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.4a16
+cic-eth-registry~=0.5.5a4
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a2
+chainsyncer[sql]~=0.0.2a4

apps/cic-cache/test_requirements.txt

@@ -4,3 +4,8 @@ pytest-mock==3.3.1
 pysqlite3==0.4.3
 sqlparse==0.4.1
 pytest-celery==0.0.0a1
+eth_tester==0.5.0b3
+py-evm==0.3.0a20
+web3==5.12.2
+cic-eth-registry~=0.5.5a3
+cic-base[full]==0.1.2b8

apps/cic-cache/tests/conftest.py

@@ -3,7 +3,7 @@ import os
 import sys
 import datetime
 
-# third-party imports
+# external imports
 import pytest
 
 # local imports
@@ -84,3 +84,7 @@ def txs(
     
     session.commit()
 
+    return [
+            tx_hash_first,
+            tx_hash_second,
+            ]

apps/cic-cache/tests/filters/conftest.py

@@ -0,0 +1,3 @@
+from chainlib.eth.pytest import *
+from cic_eth_registry.pytest.fixtures_tokens import *
+

apps/cic-cache/tests/filters/test_erc20.py

@@ -0,0 +1,69 @@
+# standard imports
+import os
+import datetime
+import logging
+import json
+
+# external imports
+import pytest
+from sqlalchemy import text
+from chainlib.eth.tx import Tx
+from chainlib.eth.block import Block
+from chainlib.chain import ChainSpec
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+
+# local imports
+from cic_cache.db import add_tag
+from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
+
+logg = logging.getLogger()
+
+
+def test_cache(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    data = 'a9059cbb'
+    data += strip_0x(list_actors['alice'])
+    data += '1000'.ljust(64, '0')
+
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': foo_token,
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    r = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    assert r
+
+    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
+    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
+    assert r[0] == tx.hash

apps/cic-cache/tests/fixtures_config.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import confini
 
@@ -13,7 +13,7 @@ logg = logging.getLogger(__file__)
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.path.join(root_dir, '.config/test')
+    config_dir = os.path.join(root_dir, 'config/test')
     conf = confini.Config(config_dir, 'CICTEST')
     conf.process()
     logg.debug('config {}'.format(conf))

apps/cic-cache/tests/fixtures_database.py

@@ -3,13 +3,16 @@ import os
 import logging
 import re
 
-# third-party imports
+# external imports
 import pytest
 import sqlparse
+import alembic
+from alembic.config import Config as AlembicConfig
 
 # local imports
 from cic_cache.db.models.base import SessionBase
 from cic_cache.db import dsn_from_config
+from cic_cache.db import add_tag
 
 logg = logging.getLogger(__file__)
 
@@ -26,11 +29,10 @@ def database_engine(
         except FileNotFoundError:
             pass
     dsn = dsn_from_config(load_config)
-    SessionBase.connect(dsn)
+    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
     return dsn
 
 
-# TODO: use alembic instead to migrate db, here we have to keep separate schema than migration script in script/migrate.py
 @pytest.fixture(scope='function')
 def init_database(
         load_config,
@@ -38,52 +40,23 @@ def init_database(
         ):
 
     rootdir = os.path.dirname(os.path.dirname(__file__))
-    schemadir = os.path.join(rootdir, 'db', load_config.get('DATABASE_DRIVER'))
-
-    if load_config.get('DATABASE_ENGINE') == 'sqlite':
-        rconn = SessionBase.engine.raw_connection()
-        f = open(os.path.join(schemadir, 'db.sql'))
-        s = f.read()
-        f.close()
-        rconn.executescript(s)
-
-    else:
-        rconn = SessionBase.engine.raw_connection()
-        rcursor = rconn.cursor()
-
-        #rcursor.execute('DROP FUNCTION IF EXISTS public.transaction_list')
-        #rcursor.execute('DROP FUNCTION IF EXISTS public.balances')
-
-        f = open(os.path.join(schemadir, 'db.sql'))
-        s = f.read()
-        f.close()
-        r = re.compile(r'^[A-Z]', re.MULTILINE)
-        for l in sqlparse.parse(s):
-            strl = str(l)
-            # we need to check for empty query lines, as sqlparse doesn't do that on its own (and psycopg complains when it gets them)
-            if not re.search(r, strl):
-                logg.warning('skipping parsed query line {}'.format(strl))
-                continue
-            rcursor.execute(strl)
-        rconn.commit()
-
-        rcursor.execute('SET search_path TO public')
-
-# this doesn't work when run separately, no idea why
-# functions have been manually added to original schema from cic-eth 
-#        f = open(os.path.join(schemadir, 'proc_transaction_list.sql'))
-#        s = f.read()
-#        f.close()
-#        rcursor.execute(s)
-# 
-#        f = open(os.path.join(schemadir, 'proc_balances.sql'))
-#        s = f.read()
-#        f.close()
-#        rcursor.execute(s)
-
-        rcursor.close()
+    dbdir = os.path.join(rootdir, 'cic_cache', 'db')
+    migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
+    if not os.path.isdir(migrationsdir):
+        migrationsdir = os.path.join(dbdir, 'migrations', 'default')
+    logg.info('using migrations directory {}'.format(migrationsdir))
 
     session = SessionBase.create_session()
+
+    ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini'))
+    ac.set_main_option('sqlalchemy.url', database_engine)
+    ac.set_main_option('script_location', migrationsdir)
+
+    alembic.command.downgrade(ac, 'base')
+    alembic.command.upgrade(ac, 'head')
+
+    session.commit()
+
     yield session
     session.commit()
     session.close()
@@ -116,3 +89,14 @@ def list_defaults(
     return {
         'block': 420000,
     }
+
+
+@pytest.fixture(scope='function')
+def tags(
+        init_database,
+        ):
+
+    add_tag(init_database, 'foo') 
+    add_tag(init_database, 'baz', domain='bar') 
+    add_tag(init_database, 'xyzzy', domain='bar') 
+    init_database.commit()

apps/cic-cache/tests/test_cache.py

@@ -4,7 +4,7 @@ import datetime
 import logging
 import json
 
-# third-party imports
+# external imports
 import pytest
 
 # local imports

apps/cic-cache/tests/test_tag.py

@@ -0,0 +1,37 @@
+import os
+import datetime
+import logging
+import json
+
+# external imports
+import pytest
+
+# local imports
+from cic_cache.db import tag_transaction
+
+logg = logging.getLogger()
+
+
+def test_cache(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        tags,
+        ):
+
+    tag_transaction(init_database, txs[0], 'foo')
+    tag_transaction(init_database, txs[0], 'baz', domain='bar')
+    tag_transaction(init_database, txs[1], 'xyzzy', domain='bar')
+
+    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.value = 'foo'").fetchall()
+    assert r[0][0] == txs[0]
+
+
+    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'baz'").fetchall()
+    assert r[0][0] == txs[0]
+
+    
+    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'xyzzy'").fetchall()
+    assert r[0][0] == txs[1]

apps/cic-eth/cic_eth/eth/account.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from erc20_faucet import Faucet
 from hexathon import (
         strip_0x,
         )
@@ -20,8 +20,8 @@ from chainlib.eth.tx import (
         )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
-from eth_accounts_index import AccountRegistry
-from sarafu_faucet import MinterFaucet as Faucet
+from eth_accounts_index.registry import AccountRegistry # TODO, use interface module instead (needs gas limit method)
+from sarafu_faucet import MinterFaucet
 from chainqueue.db.models.tx import TxCache
 
 # local import
@@ -127,13 +127,13 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     if writer_address == ZERO_ADDRESS:
         session.close()
         raise RoleMissingError('call address for resgistering {}'.format(account_address))
-    account_registry_address = registry.by_name('AccountRegistry', sender_address=call_address)
+    account_registry_address = registry.by_name('AccountsIndex', sender_address=call_address)
    
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
-    account_registry = AccountRegistry(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    gas_oracle = self.create_gas_oracle(rpc, AccountsIndex.gas)
+    account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
 
@@ -185,7 +185,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -338,7 +338,7 @@ def cache_account_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
     tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = AccountRegistry.parse_add_request(tx['data'])
+    tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_cache = TxCache(

apps/cic-eth/cic_eth/eth/erc20.py

@@ -6,7 +6,6 @@ import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -16,6 +15,7 @@ from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
 from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase

apps/cic-eth/cic_eth/ext/address.py

@@ -7,7 +7,7 @@ from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth_registry import CICRegistry
-from eth_address_declarator import AddressDeclarator
+from eth_address_declarator import Declarator
 
 # local imports
 from cic_eth.task import BaseTask
@@ -23,12 +23,12 @@ def translate_address(address, trusted_addresses, chain_spec, sender_address=ZER
     registry = CICRegistry(chain_spec, rpc)
     
     declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    c = AddressDeclarator(chain_spec)
+    c = Declarator(chain_spec)
 
     for trusted_address in trusted_addresses:
         o = c.declaration(declarator_address, trusted_address, address, sender_address=sender_address)
         r = rpc.do(o)
-        declaration_hex = AddressDeclarator.parse_declaration(r)
+        declaration_hex = Declarator.parse_declaration(r)
         declaration_hex = declaration_hex[0].rstrip('0')
         declaration_bytes = bytes.fromhex(declaration_hex)
         declaration = None

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,13 +14,13 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
-from chainlib.eth.erc20 import ERC20
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.query import get_tx_cache
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.queue.time import tx_times
@@ -114,7 +114,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
-                        token = ERC20Token(rpc, tx['to'])
+                        token = ERC20Token(chain_spec, rpc, tx['to'])
                         token_symbol = token.symbol
                         token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)

apps/cic-eth/cic_eth/queue/time.py

@@ -12,6 +12,7 @@ from chainqueue.error import NotLocalTxError
 
 # local imports
 from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
+from cic_eth.db.models.base import SessionBase
 
 celery_app = celery.current_app
 

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -13,9 +13,7 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-# TODO: use sarafu_Faucet for both when inheritance has been implemented
-from erc20_single_shot_faucet import SingleShotFaucet
-from sarafu_faucet import MinterFaucet as Faucet
+from erc20_faucet import Faucet
 
 # local imports
 from .base import SyncFilter
@@ -71,14 +69,13 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        c = SingleShotFaucet(self.chain_spec)
-        o = c.token(faucet_contract, sender_address=self.caller_address)
+        o = Faucet.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 
-        o = c.amount(faucet_contract, sender_address=self.caller_address)
+        o = c.token_amount(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
-        transfer_data['value'] = c.parse_amount(r)
+        transfer_data['value'] = c.parse_token_amount(r)
 
         return ('tokengift', transfer_data)
 

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -36,6 +36,7 @@ from cic_eth.eth import (
 from cic_eth.admin import (
         debug,
         ctrl,
+        token
         )
 from cic_eth.queue import (
         query,

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.11',
+        'beta.12',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b8
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./
@@ -50,8 +50,4 @@ COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
-	mkdir -p /usr/local/share/cic/solidity && \
-	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
-
 COPY util/liveness/health.sh /usr/local/bin/health.sh

apps/cic-eth/requirements.txt

@@ -1,25 +1,24 @@
-cic-base==0.1.2b3
+cic-base==0.1.2b8
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.4a16
-#cic-bancor~=0.0.6
+cic-eth-registry~=0.5.5a4
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a9
-erc20-transfer-authorization~=0.3.1a5
+eth_accounts_index~=0.0.11a11
+erc20-transfer-authorization~=0.3.1a6
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a9
-chainlib~=0.0.2a18
+eth-address-index~=0.1.1a11
+chainlib~=0.0.3a1
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a2
-chainqueue~=0.0.1a7
-pysha3==1.0.2
+chainsyncer[sql]~=0.0.2a4
+chainqueue~=0.0.2a2
+sarafu-faucet==0.0.3a1
 coincurve==15.0.0
-sarafu-faucet==0.0.2a28
+sarafu-faucet==0.0.3a2
 potaahto~=0.0.1a1

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -210,9 +210,11 @@ def test_callback_filter(
 
         def __init__(self):
             self.results = {}
+            self.queue = 'test'
 
         def call_back(self, transfer_type, result):
             self.results[transfer_type] = result
+            return self
 
     mock = CallbackMock()
     fltr.call_back = mock.call_back

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -65,6 +65,7 @@ def test_tx(
     tx_hash_hex_orig = tx_hash_hex
 
     gas_oracle = OverrideGasOracle(price=1100000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
     queue_create(
             default_chain_spec,

apps/cic-eth/tests/fixtures_database.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -53,6 +53,9 @@ def init_database(
     alembic.command.downgrade(ac, 'base')
     alembic.command.upgrade(ac, 'head')
 
+    session.execute('DELETE FROM lock')
+    session.commit()
+
     yield session
     session.commit()
     session.close()

apps/cic-eth/tests/task/api/test_admin.py

@@ -273,7 +273,7 @@ def test_tx(
     eth_signer,
     agent_roles,
     contract_roles,
-    celery_worker,
+    celery_session_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)

apps/cic-eth/tests/task/api/test_list.py

@@ -35,7 +35,7 @@ def test_list_tx(
         foo_token,
         register_tokens,
         init_eth_tester,
-        celery_worker,
+        celery_session_worker,
         ):
 
     tx_hashes = []

apps/cic-meta/.config/pgp.ini

@@ -1,5 +1,5 @@
 [pgp]
-exports_dir = pgp
+exports_dir = /root/pgp
 privatekey_file = privatekeys.asc
 passphrase = merman
 publickey_trusted_file = publickeys.asc

apps/cic-meta/.gitignore

@@ -3,4 +3,3 @@ dist
 dist-web
 dist-server
 scratch
-tests

apps/cic-meta/docker/Dockerfile

@@ -2,26 +2,31 @@ FROM node:15.3.0-alpine3.10
 
 WORKDIR /tmp/src/cic-meta
 
+RUN apk add --no-cache postgresql bash
+
 COPY cic-meta/package.json \
      ./
 
 COPY cic-meta/src/ src/
 COPY cic-meta/tests/ tests/
 COPY cic-meta/scripts/ scripts/
-#COPY docker/*.sh /root/
-
-RUN alias tsc=node_modules/typescript/bin/tsc 
 
 RUN npm install
 
+# see exports_dir gpg.ini
+COPY cic-meta/tests/*.asc /root/pgp/
+RUN alias tsc=node_modules/typescript/bin/tsc 
+
+
 COPY cic-meta/.config/ /usr/local/etc/cic-meta/
 # COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql
 
 COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
 
-RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
-ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
+#RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
+#ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
 
-# COPY cic-meta/docker/start_server.sh ./start_server.sh
-# RUN chmod 755 ./start_server.sh
+COPY cic-meta/docker/start_server.sh ./start_server.sh
+RUN chmod 755 ./start_server.sh
+ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/db.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
+set -e 
+
+PGPASSWORD=$DATABASE_PASSWORD psql -v ON_ERROR_STOP=1 -U $DATABASE_USER -h $DATABASE_HOST -p $DATABASE_PORT -d $DATABASE_NAME -f $SCHEMA_SQL_PATH 
+
 
-PGPASSWORD=$DATABASE_PASSWORD psql -U $DATABASE_USER -h $DATABASE_HOST -p $DATABASE_PORT -d $DATABASE_NAME /usr/local/share/cic-meta/sql/server.sql

apps/cic-meta/docker/start_server.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+# db migration
 sh ./db.sh
 
-/usr/local/bin/node /usr/local/bin/cic-meta-server $@
+# /usr/local/bin/node /usr/local/bin/cic-meta-server $@
+# ./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts $@
+npm run start "$@"

apps/cic-meta/package-lock.json

@@ -1,6 +1,6 @@
 {
 	"name": "cic-client-meta",
-	"version": "0.0.7-alpha.2",
+	"version": "0.0.7-alpha.7",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
@@ -852,6 +852,75 @@
 				"printj": "~1.1.0"
 			}
 		},
+		"crdt-meta": {
+			"version": "0.0.8",
+			"resolved": "https://registry.npmjs.org/crdt-meta/-/crdt-meta-0.0.8.tgz",
+			"integrity": "sha512-CS0sS0L2QWthz7vmu6vzl3p4kcpJ+IKILBJ4tbgN4A3iNG8wnBeuDIv/z3KFFQjcfuP4QAh6E9LywKUTxtDc3g==",
+			"requires": {
+				"automerge": "^0.14.2",
+				"ini": "^1.3.8",
+				"openpgp": "^4.10.8",
+				"pg": "^8.5.1",
+				"sqlite3": "^5.0.2"
+			},
+			"dependencies": {
+				"automerge": {
+					"version": "0.14.2",
+					"resolved": "https://registry.npmjs.org/automerge/-/automerge-0.14.2.tgz",
+					"integrity": "sha512-shiwuJHCbNRI23WZyIECLV4Ovf3WiAFJ7P9BH4l5gON1In/UUbjcSJKRygtIirObw2UQumeYxp3F2XBdSvQHnA==",
+					"requires": {
+						"immutable": "^3.8.2",
+						"transit-immutable-js": "^0.7.0",
+						"transit-js": "^0.8.861",
+						"uuid": "^3.4.0"
+					}
+				},
+				"node-addon-api": {
+					"version": "3.1.0",
+					"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz",
+					"integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw=="
+				},
+				"pg": {
+					"version": "8.6.0",
+					"resolved": "https://registry.npmjs.org/pg/-/pg-8.6.0.tgz",
+					"integrity": "sha512-qNS9u61lqljTDFvmk/N66EeGq3n6Ujzj0FFyNMGQr6XuEv4tgNTXvJQTfJdcvGit5p5/DWPu+wj920hAJFI+QQ==",
+					"requires": {
+						"buffer-writer": "2.0.0",
+						"packet-reader": "1.0.0",
+						"pg-connection-string": "^2.5.0",
+						"pg-pool": "^3.3.0",
+						"pg-protocol": "^1.5.0",
+						"pg-types": "^2.1.0",
+						"pgpass": "1.x"
+					}
+				},
+				"pg-connection-string": {
+					"version": "2.5.0",
+					"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
+					"integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ=="
+				},
+				"pg-pool": {
+					"version": "3.3.0",
+					"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.3.0.tgz",
+					"integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg=="
+				},
+				"pg-protocol": {
+					"version": "1.5.0",
+					"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz",
+					"integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ=="
+				},
+				"sqlite3": {
+					"version": "5.0.2",
+					"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.0.2.tgz",
+					"integrity": "sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA==",
+					"requires": {
+						"node-addon-api": "^3.0.0",
+						"node-gyp": "3.x",
+						"node-pre-gyp": "^0.11.0"
+					}
+				}
+			}
+		},
 		"create-hash": {
 			"version": "1.2.0",
 			"resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
@@ -966,17 +1035,17 @@
 			"dev": true
 		},
 		"elliptic": {
-			"version": "6.5.3",
-			"resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz",
-			"integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==",
+			"version": "6.5.4",
+			"resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz",
+			"integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==",
 			"requires": {
-				"bn.js": "^4.4.0",
-				"brorand": "^1.0.1",
+				"bn.js": "^4.11.9",
+				"brorand": "^1.1.0",
 				"hash.js": "^1.0.0",
-				"hmac-drbg": "^1.0.0",
-				"inherits": "^2.0.1",
-				"minimalistic-assert": "^1.0.0",
-				"minimalistic-crypto-utils": "^1.0.0"
+				"hmac-drbg": "^1.0.1",
+				"inherits": "^2.0.4",
+				"minimalistic-assert": "^1.0.1",
+				"minimalistic-crypto-utils": "^1.0.1"
 			}
 		},
 		"emoji-regex": {
@@ -1489,9 +1558,9 @@
 			"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
 		},
 		"ini": {
-			"version": "1.3.5",
-			"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-			"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
+			"version": "1.3.8",
+			"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+			"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
 		},
 		"interpret": {
 			"version": "2.2.0",
@@ -1957,9 +2026,9 @@
 					}
 				},
 				"y18n": {
-					"version": "4.0.0",
-					"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
-					"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
+					"version": "4.0.3",
+					"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz",
+					"integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==",
 					"dev": true
 				},
 				"yargs": {

apps/cic-meta/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cic-client-meta",
-	"version": "0.0.7-alpha.6",
+	"version": "0.0.7-alpha.8",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
@@ -10,13 +10,15 @@
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
 		"clean": "rm -rf dist",
-		"prepare": "npm run build && npm run build-server"
+		"prepare": "npm run build && npm run build-server",
+                "start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts" 
 	},
 	"dependencies": {
 		"@ethereumjs/tx": "^3.0.0-beta.1",
 		"automerge": "^0.14.1",
+		"crdt-meta": "0.0.8",
 		"ethereumjs-wallet": "^1.0.1",
-		"ini": "^1.3.5",
+		"ini": "^1.3.8",
 		"openpgp": "^4.10.8",
 		"pg": "^8.4.2",
 		"sqlite3": "^5.0.0",
@@ -40,6 +42,6 @@
 	],
 	"license": "GPL-3.0-or-later",
 	"engines": {
-		"node": "~14.16.1"
+		"node": ">=14.16.1"
 	}
 }

apps/cic-meta/scripts/dumpconfig.js

@@ -1,4 +1,4 @@
-const config = require('./src/config');
+import { Config } from 'crdt-meta';
 const fs = require('fs');
 
 if (process.argv[2] === undefined) {
@@ -15,6 +15,6 @@ try {
 	process.exit(1);
 }
 
-const c = new config.Config(process.argv[2], process.env['CONFINI_ENV_PREFIX']);
+const c = new Config(process.argv[2], process.env['CONFINI_ENV_PREFIX']);
 c.process();
 process.stdout.write(c.toString());

apps/cic-meta/scripts/initdb/postgresql.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-psql -v ON_ERROR_STOP=1 --username grassroots --dbname cic_meta <<-EOSQL
-    create table if not exists store (
-        id serial primary key not null,
-        owner_fingerprint text not null,
-        hash char(64) not null unique,
-        content text not null
-    );
-
-    create index if not exists idx_fp on store ((lower(owner_fingerprint)));
-EOSQL
-
-

apps/cic-meta/scripts/initdb/server.postgres.sql

@@ -1,4 +1,4 @@
-create table if not exists cic_meta.store (
+create table if not exists store (
 	id serial primary key not null,
 	owner_fingerprint text not null,
 	hash char(64) not null unique,

apps/cic-meta/scripts/server/handlers.ts

@@ -1,8 +1,7 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
-import * as pg from 'pg';
 
-import { Envelope, Syncable } from '../../src/sync';
+import { Envelope, Syncable } from 'crdt-meta';
 
 
 function handleNoMergeGet(db, digest, keystore) {

apps/cic-meta/scripts/server/server.ts

@@ -1,15 +1,11 @@
 import * as http from 'http';
 import * as fs from 'fs';
 import * as path from 'path';
-import * as pgp from 'openpgp';
 
 import * as handlers from './handlers';
-import { Envelope, Syncable } from '../../src/sync';
-import { PGPKeyStore, PGPSigner } from '../../src/auth';
+import { PGPKeyStore, PGPSigner, Config, SqliteAdapter, PostgresAdapter } from 'crdt-meta';
 
 import { standardArgs } from './args';
-import { Config } from '../../src/config';
-import { SqliteAdapter, PostgresAdapter } from '../../src/db';
 
 let configPath = '/usr/local/etc/cic-meta';
 

apps/cic-meta/src/auth.ts

@@ -1,191 +0,0 @@
-import * as pgp from 'openpgp';
-import * as crypto from 'crypto';
-
-interface Signable {
-	digest():string;
-}
-
-type KeyGetter = () => any;
-
-type Signature = {
-	engine:string
-	algo:string
-	data:string
-	digest:string
-}
-
-interface Signer { 
-	prepare(Signable):boolean;
-	onsign(Signature):void;
-	onverify(boolean):void;
-	sign(digest:string):void
-	verify(digest:string, signature:Signature):void
-	fingerprint():string
-}
-
-interface Authoritative {
-}
-
-interface KeyStore {
-	getPrivateKey:		KeyGetter
-	getFingerprint:		() => string
-	getTrustedKeys: 	() => Array<any>
-	getTrustedActiveKeys: 	() => Array<any>
-	getEncryptKeys: 	() => Array<any>
-}
-
-class PGPKeyStore implements KeyStore {
-
-	fingerprint:	string
-	pk:		any
-
-	pubk = {
-		active: [],
-		trusted: [],
-		encrypt: [],
-	}
-	loads = 0x00;
-	loadsTarget = 0x0f;
-	onload: (k:KeyStore) => void;
-
-	constructor(passphrase:string, pkArmor:string, pubkActiveArmor:string, pubkTrustedArmor:string, pubkEncryptArmor:string, onload = (ks:KeyStore) => {}) {
-		this._readKey(pkArmor, undefined, 1, passphrase);
-		this._readKey(pubkActiveArmor, 'active', 2);
-		this._readKey(pubkTrustedArmor, 'trusted', 4);
-		this._readKey(pubkEncryptArmor, 'encrypt', 8);
-		this.onload = onload;
-	}
-
-	private _readKey(a:string, x:any, n:number, pass?:string) {
-		pgp.key.readArmored(a).then((k) => {
-			if (pass !== undefined) {
-				this.pk = k.keys[0];
-				this.pk.decrypt(pass).then(() => {
-					this.fingerprint = this.pk.getFingerprint();
-					console.log('private key (sign)', this.fingerprint);
-					this._registerLoad(n);
-				});
-			} else {
-				this.pubk[x] = k.keys;
-				k.keys.forEach((pubk) => {
-					console.log('public key (' + x + ')', pubk.getFingerprint());
-				});
-				this._registerLoad(n);
-			}
-		});
-	}
-
-	private _registerLoad(b:number) {
-		this.loads |= b;
-		if (this.loads == this.loadsTarget) {
-			this.onload(this);
-		}
-	}
-
-	public getTrustedKeys(): Array<any> {
-		return this.pubk['trusted'];
-	}
-
-	public getTrustedActiveKeys(): Array<any> {
-		return this.pubk['active'];
-
-	}
-
-	public getEncryptKeys(): Array<any> {
-		return this.pubk['encrypt'];
-
-	}
-
-	public getPrivateKey(): any {
-		return this.pk;
-	}
-
-	public getFingerprint(): string {
-		return this.fingerprint;
-	}
-}
-
-class PGPSigner implements Signer {
-
-	engine	= 'pgp'
-	algo	= 'sha256'
-	dgst:		string
-	signature:	Signature
-	keyStore:	KeyStore
-	onsign:		(Signature) => void
-	onverify:	(boolean) => void
-
-	constructor(keyStore:KeyStore) {
-		this.keyStore = keyStore
-		this.onsign = (string) => {};
-		this.onverify = (boolean) => {};
-	}
-
-	public fingerprint(): string {
-		return this.keyStore.getFingerprint();
-	}
-
-	public prepare(material:Signable):boolean {
-		this.dgst = material.digest();
-		return true;
-	}
-
-	public verify(digest:string, signature:Signature) {
-		pgp.signature.readArmored(signature.data).then((s) => {
-			const opts = {
-				message: pgp.cleartext.fromText(digest),
-				publicKeys: this.keyStore.getTrustedKeys(),
-				signature: s,
-			};
-			pgp.verify(opts).then((v) => {
-				let i = 0;
-				for (i = 0; i < v.signatures.length; i++) {
-					const s = v.signatures[i];
-					if (s.valid) {
-						this.onverify(s);
-						return;
-					}
-				}
-				console.error('checked ' + i + ' signature(s) but none valid');
-				this.onverify(false);
-			});
-		}).catch((e) => {
-			console.error(e);
-			this.onverify(false);
-		});
-	}
-
-	public sign(digest:string) {
-		const m = pgp.cleartext.fromText(digest);
-		const pk = this.keyStore.getPrivateKey();
-		const opts = {
-			message: m,
-			privateKeys: [pk],
-			detached: true,
-		}
-		pgp.sign(opts).then((s) => {
-			this.signature = {
-				engine: this.engine,
-				algo: this.algo,
-				data: s.signature,
-				// TODO: fix for browser later
-				digest: digest,
-			};
-			this.onsign(this.signature);
-		}).catch((e) => {
-			console.error(e);
-			this.onsign(undefined);
-		});
-	}
-}
-
-export {
-	Signature,
-	Authoritative,
-       	Signer,
-	KeyGetter,
-	Signable,
-	KeyStore,
-	PGPSigner,
-	PGPKeyStore,
-};

apps/cic-meta/src/config.ts

@@ -1,71 +0,0 @@
-import * as fs from 'fs';
-import * as ini from 'ini';
-import * as path from 'path';
-
-class Config {
-
-	filepath: 	string
-	store:		Object
-	censor:		Array<string>
-	require:	Array<string>
-	env_prefix:	string
-
-	constructor(filepath:string, env_prefix?:string) {
-		this.filepath = filepath;
-		this.store = {};
-		this.censor = [];
-		this.require = [];
-		this.env_prefix = '';
-		if (env_prefix !== undefined) {
-			this.env_prefix = env_prefix + "_";
-		}
-	}
-
-	public process() {
-		const d = fs.readdirSync(this.filepath);
-		
-		const r = /.*\.ini$/;
-		for (let i = 0; i < d.length; i++) {
-			const f = d[i];
-			if (!f.match(r)) {
-				return;
-			}
-
-			const fp = path.join(this.filepath, f);
-			const v = fs.readFileSync(fp, 'utf-8');
-			const inid = ini.decode(v);
-			const inik = Object.keys(inid);
-			for (let j = 0; j < inik.length; j++) {
-				const k_section = inik[j]
-				const k = k_section.toUpperCase();
-				Object.keys(inid[k_section]).forEach((k_directive) => {
-					const kk = k_directive.toUpperCase();
-					const kkk = k + '_' + kk;
-
-					let r = inid[k_section][k_directive];
-					const k_env = this.env_prefix + kkk
-					const env = process.env[k_env];
-					if (env !== undefined) {
-						console.debug('Environment variable ' + k_env + ' overrides ' + kkk);
-						r = env;
-					}
-					this.store[kkk] = r;
-				});
-			}
-		}
-	}
-
-	public get(s:string) {
-		return this.store[s];
-	}
-
-	public toString() {
-		let s = '';
-		Object.keys(this.store).forEach((k) => {
-			s += k + '=' + this.store[k] + '\n';
-		});
-		return s;
-	}	
-}
-
-export { Config };

apps/cic-meta/src/constants.ts

@@ -1,38 +0,0 @@
-import { JSONSerializable } from './format';
-
-const ENGINE_NAME = 'automerge';
-const ENGINE_VERSION = '0.14.1';
-
-const NETWORK_NAME = 'cic';
-const NETWORK_VERSION = '1';
-
-const CRYPTO_NAME = 'pgp';
-const CRYPTO_VERSION = '2';
-
-type VersionedSpec = {
-	name:		string
-	version: 	string
-	ext?:		Object
-}
-
-const engineSpec:VersionedSpec = {
-	name: ENGINE_NAME,
-	version: ENGINE_VERSION,			
-}
-
-const cryptoSpec:VersionedSpec = {
-	name: CRYPTO_NAME,
-	version: CRYPTO_VERSION,
-}
-
-const networkSpec:VersionedSpec = {
-	name: NETWORK_NAME,
-	version: NETWORK_VERSION,
-}
-
-export {
-	engineSpec,
-	cryptoSpec,
-	networkSpec,
-	VersionedSpec,
-};

apps/cic-meta/src/crypto.ts

@@ -1,27 +0,0 @@
-import * as crypto from 'crypto';
-
-const _algs = {
-	'SHA-256': 'sha256',
-}
-
-function cryptoWrapper() {
-}
-
-cryptoWrapper.prototype.digest = async function(s, d) {
-	const h = crypto.createHash(_algs[s]);
-	h.update(d);
-	return h.digest();
-}
-
-let subtle = undefined;
-if (typeof window !== 'undefined') {
-	subtle = window.crypto.subtle;
-} else {
-	subtle = new cryptoWrapper();
-}
-
-
-export {
-	subtle,
-}
-

apps/cic-meta/src/db.ts

@@ -1,90 +0,0 @@
-import * as pg from 'pg';
-import * as sqlite from 'sqlite3';
-
-type DbConfig = {
-	name:		string
-	host:		string
-	port:		number
-	user:		string
-	password:	string	
-}
-
-interface DbAdapter {
-	query:	(s:string, callback:(e:any, rs:any) => void) => void
-	close: 	() => void
-}
-
-const re_creatematch = /^(CREATE)/i
-const re_getmatch = /^(SELECT)/i;
-const re_setmatch = /^(INSERT|UPDATE)/i;
-
-class SqliteAdapter implements DbAdapter {
-
-	db:		any
-
-	constructor(dbConfig:DbConfig, callback?:(any) => void) {
-		this.db = new sqlite.Database(dbConfig.name); //, callback);
-	}
-
-	public query(s:string, callback:(e:any, rs?:any) => void): void {
-		const local_callback = (e, rs) => {
-			let r = undefined;
-			if (rs !== undefined) {
-				r = {
-					rowCount: rs.length,
-					rows: rs,
-				}
-			}
-			callback(e, r);
-		};
-		if (s.match(re_getmatch)) {
-			this.db.all(s, local_callback);
-		} else if (s.match(re_setmatch)) {
-			this.db.run(s, local_callback);
-		} else if (s.match(re_creatematch)) {
-			this.db.run(s, callback);
-		} else {
-			throw 'unhandled query';
-		}
-	}
-
-	public close() {
-		this.db.close();
-	}
-}
-
-class PostgresAdapter implements DbAdapter {
-
-	db:		any
-
-	constructor(dbConfig:DbConfig) {
-		let o = dbConfig;
-		o['database'] = o.name;
-		this.db = new pg.Pool(o);
-		return this.db;
-	}
-
-	public query(s:string, callback:(e:any, rs:any) => void): void {
-		this.db.query(s, (e, rs) => {
-			let r = {
-				length: rs.rowCount,
-			}
-			rs.length = rs.rowCount;
-			if (e === undefined) {
-				e = null;
-			}
-			console.debug(e, rs);
-			callback(e, rs);
-		});
-	}
-
-	public close() {
-		this.db.end();
-	}
-}
-
-export {
-	DbConfig,
-	SqliteAdapter,
-	PostgresAdapter,
-}

apps/cic-meta/src/digest.ts

@@ -1,68 +0,0 @@
-import * as crypto from './crypto';
-
-interface Addressable {
-	key(): string
-	digest(): string
-}
-
-function stringToBytes(s:string) {
-	const a = new Uint8Array(20);
-	let j = 2;
-	for (let i = 0; i < a.byteLength; i++) {
-		const n = parseInt(s.substring(j, j+2), 16);
-		a[i] = n;
-		j += 2;
-	}
-	return a;
-}
-
-function bytesToHex(a:Uint8Array) {
-	let s = '';
-	for (let i = 0; i < a.byteLength; i++) {
-		const h = '00' + a[i].toString(16);
-		s += h.slice(-2);
-	}
-	return s;
-}
-
-async function mergeKey(a:Uint8Array, s:Uint8Array) {
-	const y = new Uint8Array(a.byteLength + s.byteLength);
-	for (let i = 0; i < a.byteLength; i++) {
-		y[i] = a[i];
-	}
-	for (let i = 0; i < s.byteLength; i++) {
-		y[a.byteLength + i] = s[i];
-	}
-	const z = await crypto.subtle.digest('SHA-256', y);
-	return bytesToHex(new Uint8Array(z));
-}
-
-async function toKey(v:string, salt:string) {
-	const a = stringToBytes(v);
-	const s = new TextEncoder().encode(salt);
-	return await mergeKey(a, s);
-}
-
-
-async function toAddressKey(zeroExHex:string, salt:string) {
-	const a = addressToBytes(zeroExHex);
-	const s = new TextEncoder().encode(salt);
-	return await mergeKey(a, s);
-}
-
-const re_addrHex = /^0[xX][a-fA-F0-9]{40}$/;
-function addressToBytes(s:string) {
-	if (!s.match(re_addrHex)) {
-		throw 'invalid address hex';
-	}
-	return stringToBytes(s);
-}
-
-export {
-	toKey,
-	toAddressKey,
-	mergeKey,
-	bytesToHex,
-	addressToBytes,
-	Addressable,
-}

apps/cic-meta/src/dispatch.ts

@@ -1,58 +0,0 @@
-import { v4 as uuidv4 } from 'uuid';
-import { Syncable } from './sync';
-import { Store } from './store';
-import { PubSub } from './transport';
-
-function toIndexKey(id:string):string {
-	const d = Date.now();
-	return d + '_' + id + '_' + uuidv4();
-}
-
-const _re_indexKey = /^\d+_(.+)_[-\d\w]+$/;
-function fromIndexKey(s:string):string {
-	const m = s.match(_re_indexKey);
-	if (m === null) {
-		throw 'Invalid index key';
-	}
-	return m[1];
-}
-
-class Dispatcher {
-
-	idx:		Array<string>
-	syncer:		PubSub
-	store:		Store
-
-	constructor(store:Store, syncer:PubSub) {
-		this.idx = new Array<string>()
-		this.syncer = syncer;
-		this.store = store;
-	}
-
-	public isDirty(): boolean {
-		return this.idx.length > 0;
-	}
-
-	public add(id:string, item:Syncable): string {
-		const v = item.toJSON();
-		const k = toIndexKey(id);
-		this.store.put(k, v, true);
-		localStorage.setItem(k, v);
-		this.idx.push(k);
-		return k;
-	}
-
-	public sync(offset:number): number {
-		let i = 0;
-		this.idx.forEach((k) => {
-			const v = localStorage.getItem(k);
-			const k_id = fromIndexKey(k);
-			this.syncer.pub(v); // this must block until guaranteed delivery
-			localStorage.removeItem(k);
-			i++;
-		});
-		return i;
-	}
-}
-
-export { Dispatcher, toIndexKey, fromIndexKey }

apps/cic-meta/src/format.ts

@@ -1,5 +0,0 @@
-interface JSONSerializable {
-	toJSON(): string
-}
-
-export { JSONSerializable };

apps/cic-meta/src/index.ts

@@ -1,5 +1,2 @@
-export { PGPSigner, PGPKeyStore, Signer, KeyStore } from './auth';
-export { ArgPair,  Envelope, Syncable } from './sync';
-export { User } from './assets/user';
-export { Phone } from './assets/phone';
-export { Config } from './config';
+export { User } from './user';
+export { Phone } from './phone';

apps/cic-meta/src/phone.ts

@@ -1,5 +1,4 @@
-import { ArgPair, Syncable } from '../sync';
-import { Addressable, mergeKey } from '../digest';
+import { Syncable, Addressable, mergeKey } from 'crdt-meta';
 
 class Phone extends Syncable implements Addressable {
 

apps/cic-meta/src/store.ts

@@ -1,9 +0,0 @@
-import { Syncable } from './sync';
-
-interface Store {
-	put(string, Syncable, boolean?)
-	get(string):Syncable
-	delete(string)
-}
-
-export { Store };

apps/cic-meta/src/sync.ts

@@ -1,266 +0,0 @@
-import * as Automerge from 'automerge';
-
-import { JSONSerializable } from './format';
-
-import { Authoritative, Signer, PGPSigner, Signable, Signature } from './auth';
-
-import { engineSpec, cryptoSpec, networkSpec, VersionedSpec  } from './constants';
-
-const fullSpec:VersionedSpec = {
-	name: 'cic',
-	version: '1',
-	ext: {
-		network: cryptoSpec,
-		engine: engineSpec,
-	},
-}
-
-class Envelope {
-
-	o = fullSpec
-
-	constructor(payload:Object) {
-		this.set(payload);
-	}
-
-	public set(payload:Object) {
-		this.o['payload'] = payload
-	}
-
-	public get():string {
-		return this.o['payload'];
-	}
-
-	public toJSON() {
-		return JSON.stringify(this.o);
-	}
-
-	public static fromJSON(s:string): Envelope {
-		const e = new Envelope(undefined);
-		e.o = JSON.parse(s);
-		return e;
-	}
-
-	public unwrap(): Syncable {
-		return Syncable.fromJSON(this.o['payload']);
-	}
-}
-
-class ArgPair {
-
-	k:string
-	v:any
-
-	constructor(k:string, v:any) {
-		this.k = k;
-		this.v = v;
-	}
-}
-
-class SignablePart implements Signable {
-
-	s: string
-
-	constructor(s:string) {
-		this.s = s;
-	}
-
-	public digest():string {
-		return this.s;
-	}
-}
-
-function orderDict(src) {
-	let dst;
-	if (Array.isArray(src)) {
-		dst = [];
-		src.forEach((v) => {
-			if (typeof(v) == 'object') {
-				v = orderDict(v);
-			}
-			dst.push(v);
-		});
-	} else {
-		dst = {}
-		Object.keys(src).sort().forEach((k) => {
-			let v = src[k];
-			if (typeof(v) == 'object') {
-				v = orderDict(v);
-			}
-			dst[k] = v;
-		});
-	} 
-	return dst;
-}
-
-class Syncable implements JSONSerializable, Authoritative, Signable {
-
-	id:		string
-	timestamp:	number
-	m:		any // automerge object
-	e:		Envelope
-	signer:		Signer
-	onwrap: 	(string) => void
-	onauthenticate:	(boolean) => void
-
-	// TODO: Move data to sub-object so timestamp, id, signature don't collide
-	constructor(id:string, v:Object) {
-		this.id = id;
-		const o = {
-			'id': id,
-			'timestamp': Math.floor(Date.now() / 1000),
-			'data': v,
-		}
-		//this.m = Automerge.from(v)
-		this.m = Automerge.from(o)
-	}
-
-	public setSigner(signer:Signer) {
-		this.signer = signer;
-		this.signer.onsign = (s) => {
-			this.wrap(s);
-		};
-	}
-
-	// TODO: To keep integrity, the non-link key/value pairs for each step also need to be hashed
-	public digest(): string {
-		const links = [];
-		Automerge.getAllChanges(this.m).forEach((ch:Object) => {
-			const op:Array<any> = ch['ops'];
-			ch['ops'].forEach((op:Array<Object>) => {
-				if (op['action'] == 'link') {
-					//console.log('op link', op);
-					links.push([op['obj'], op['value']]);
-				}
-			});
-		});
-		//return JSON.stringify(links);
-		const j = JSON.stringify(links);
-		return Buffer.from(j).toString('base64');
-	}
-
-	private wrap(s:any) {
-		this.m = Automerge.change(this.m, 'sign', (doc) => {
-			doc['signature'] = s;
-		});
-		this.e = new Envelope(this.toJSON());
-		console.log('wrappin s', s, typeof(s));
-		this.e.o['digest'] = s.digest;
-		if (this.onwrap !== undefined) {
-			this.onwrap(this.e);
-		}
-
-	}
-
-//	private _verifyLoop(i:number, history:Array<any>, signable:Signable, result:boolean) {
-//		if (!result) {
-//			this.onauthenticate(false);
-//			return;
-//		} else if (history.length == 0) {
-//			this.onauthenticate(true);
-//			return;
-//		}
-//		const h = history.shift()
-//		if (i % 2 == 0) {
-//			i++;
-//			signable = {
-//				digest: () => {
-//					return Automerge.save(h.snapshot)
-//				},
-//			};
-//			this._verifyLoop(i, history, signable, true);
-//		} else {
-//			i++;
-//			const signature = h.snapshot['signature'];
-//			console.debug('signature', signature, signable.digest());
-//			this.signer.onverify = (v) => {
-//				this._verifyLoop(i, history, signable, v)
-//			}
-//			this.signer.verify(signable, signature);
-//		}
-//	}
-//
-//	// TODO: This should replay the graph and check signatures on each step
-//	public _authenticate(full:boolean=false) {
-//		let h = Automerge.getHistory(this.m);
-//		h.forEach((m) => {
-//			//console.debug(m.snapshot);
-//		});
-//		const signable = {
-//			digest: () => { return '' },
-//		}
-//		if (!full) {
-//			h = h.slice(h.length-2);
-//		}
-//		this._verifyLoop(0, h, signable, true);
-//	}
-
-	public authenticate(full:boolean=false) {
-		if (full) {
-			console.warn('only doing shallow authentication for now, sorry');			
-		}
-		//console.log('authenticating', signable.digest());
-		//console.log('signature', this.m.signature);
-		this.signer.onverify = (v) => {
-			//this._verifyLoop(i, history, signable, v)
-			this.onauthenticate(v);
-		}
-		this.signer.verify(this.m.signature.digest, this.m.signature);
-	}
-
-
-	public sign() {
-		//this.signer.prepare(this);
-		this.signer.sign(this.digest());
-	}
-
-	public update(changes:Array<ArgPair>, changesDescription:string) {
-		this.m = Automerge.change(this.m, changesDescription, (m) => {
-			changes.forEach((c) => {
-				let path = c.k.split('.');
-				let target = m['data'];
-				while (path.length > 1) {
-					const part = path.shift();
-					target = target[part];
-				}
-				target[path[0]] = c.v;
-			});
-			m['timestamp'] = Math.floor(Date.now() / 1000);
-		});
-	}
-
-	public replace(o:Object, changesDescription:string) {
-		this.m = Automerge.change(this.m, changesDescription, (m) => {
-			Object.keys(o).forEach((k) => {
-				m['data'][k] = o[k];
-			});
-			Object.keys(m).forEach((k) => {
-				if (o[k] == undefined) {
-					delete m['data'][k];
-				}
-			});
-			m['timestamp'] = Math.floor(Date.now() / 1000);
-		});
-	}
-
-	public merge(s:Syncable) {
-		this.m = Automerge.merge(s.m, this.m);
-	}
-
-	public toJSON(): string {
-		const s = Automerge.save(this.m);
-		const o = JSON.parse(s);
-		const oo = orderDict(o)
-		return JSON.stringify(oo);
-
-	}
-
-	public static fromJSON(s:string): Syncable {
-		const doc = Automerge.load(s);
-		let y = new Syncable(doc['id'], {});
-		y.m = doc
-		return y
-	}
-}
-
-export { JSONSerializable, Syncable, ArgPair, Envelope };

apps/cic-meta/src/transport.ts

@@ -1,11 +0,0 @@
-interface SubConsumer {
-	post(string)
-}
-
-interface PubSub {
-	pub(v:string):boolean
-	close()
-}
-
-export { PubSub, SubConsumer };
-

apps/cic-meta/src/user.ts

@@ -1,5 +1,4 @@
-import { ArgPair, Syncable } from '../sync';
-import { Addressable, addressToBytes, bytesToHex, toAddressKey } from '../digest';
+import { Syncable, Addressable, toAddressKey } from 'crdt-meta';
 
 const keySalt = new TextEncoder().encode(':cic.person');
 class User extends Syncable implements Addressable {

apps/cic-meta/tests/1_basic.ts

@@ -1,50 +0,0 @@
-import * as Automerge from 'automerge';
-import assert = require('assert');
-
-import { Dispatcher, toIndexKey, fromIndexKey } from '../src/dispatch';
-import { User } from '../src/assets/user';
-import { Syncable, ArgPair } from '../src/sync';
-
-import { MockSigner, MockStore } from './mock';
-
-describe('basic', () => {
-
-	it('store', () => {
-		const store = new MockStore('s');
-		assert.equal(store.name, 's');
-
-		const mockSigner = new MockSigner();
-		const v = new Syncable('foo', {baz: 42});
-		v.setSigner(mockSigner);
-		store.put('foo', v);
-		const one = store.get('foo').toJSON();
-		const vv = new Syncable('bar', {baz: 666});
-		vv.setSigner(mockSigner);
-		assert.throws(() => {
-			store.put('foo', vv)
-		});
-		store.put('foo', vv, true);
-		const other = store.get('foo').toJSON();
-		assert.notEqual(one, other);
-		store.delete('foo');
-		assert.equal(store.get('foo'), undefined);
-	});
-
-	it('add_doc_to_dispatcher', () => {
-		const store = new MockStore('s');
-		//const syncer = new MockSyncer();
-		const dispatcher = new Dispatcher(store, undefined);
-		const user = new User('foo'); 
-		dispatcher.add(user.id, user);
-		assert(dispatcher.isDirty());
-	});
-
-	it('dispatch_keyindex', () => {
-		const s = 'foo';
-		const k = toIndexKey(s);
-		const v = fromIndexKey(k);
-		assert.equal(s, v);
-	});
-
-
-});

apps/cic-meta/tests/2_sync.ts

@@ -1,212 +0,0 @@
-import * as Automerge from 'automerge';
-import assert = require('assert');
-
-import * as pgp from 'openpgp';
-import * as fs from 'fs';
-
-import { PGPSigner } from '../src/auth';
-
-import { Syncable, ArgPair } from '../src/sync';
-
-import { MockKeyStore, MockSigner } from './mock';
-
-
-describe('sync', async () => {
-	it('sync_merge', () => {
-		const mockSigner = new MockSigner();
-		const s = new Syncable('foo', {
-			bar: 'baz',
-		});
-		s.setSigner(mockSigner);
-		const changePair = new ArgPair('xyzzy', 42);
-		s.update([changePair], 'ch-ch-cha-changes');
-		assert.equal(s.m.data['xyzzy'], 42)
-		assert.equal(s.m.data['bar'], 'baz')
-		assert.equal(s.m['id'], 'foo')
-		assert.equal(Automerge.getHistory(s.m).length, 2);
-	});
-
-	it('sync_serialize', () => {
-		const mockSigner = new MockSigner();
-		const s = new Syncable('foo', {
-			bar: 'baz',
-		});
-		s.setSigner(mockSigner);
-		const j = s.toJSON();
-		const ss = Syncable.fromJSON(j);
-		assert.equal(ss.m['id'], 'foo');
-		assert.equal(ss.m['data']['bar'], 'baz');
-		assert.equal(Automerge.getHistory(ss.m).length, 1);
-	});
-
-	it('sync_sign_and_wrap', () => {
-		const mockSigner = new MockSigner();
-		const s = new Syncable('foo', {
-			bar: 'baz',
-		});
-		s.setSigner(mockSigner);
-		s.onwrap = (e) => {
-			const j = e.toJSON();
-			const v = JSON.parse(j);
-			assert.deepEqual(v.payload, e.o.payload);
-
-		}
-		s.sign();
-	});
-	it('sync_verify_success', async () => {
-		const pksa = fs.readFileSync(__dirname + '/privatekeys.asc');
-		const pks = await pgp.key.readArmored(pksa);
-		await pks.keys[0].decrypt('merman');
-		await pks.keys[1].decrypt('beastman');
-
-		const pubksa = fs.readFileSync(__dirname + '/publickeys.asc');
-		const pubks = await pgp.key.readArmored(pubksa);
-
-		const oneStore = new MockKeyStore(pks.keys[0], pubks.keys);
-		const twoStore = new MockKeyStore(pks.keys[1], pubks.keys);
-		const threeStore = new MockKeyStore(pks.keys[2], [pubks.keys[0], pubks.keys[2]]);
-
-		const oneSigner = new PGPSigner(oneStore);
-		const twoSigner = new PGPSigner(twoStore);
-		const threeSigner = new PGPSigner(threeStore);
-
-		const x = new Syncable('foo', {
-			bar: 'baz',		  
-		});
-		x.setSigner(oneSigner);
-
-		// TODO: make this look better
-		x.onwrap = (e) => {
-			let updateData = new ArgPair('bar', 'xyzzy');
-			x.update([updateData], 'change one');
-
-			x.onwrap = (e) => {
-				x.setSigner(twoSigner);
-				updateData = new ArgPair('bar', 42);
-				x.update([updateData], 'change two');
-
-				x.onwrap = (e) => {
-					const p = e.unwrap();
-					p.setSigner(twoSigner);
-					p.onauthenticate = (v) => {
-						assert(v);
-					}
-					p.authenticate();
-				}
-
-				x.sign();
-			};
-
-			x.sign();
-		}
-
-		x.sign();
-
-	});
-
-	it('sync_verify_fail', async () => {
-		const pksa = fs.readFileSync(__dirname + '/privatekeys.asc');
-		const pks = await pgp.key.readArmored(pksa);
-		await pks.keys[0].decrypt('merman');
-		await pks.keys[1].decrypt('beastman');
-
-		const pubksa = fs.readFileSync(__dirname + '/publickeys.asc');
-		const pubks = await pgp.key.readArmored(pubksa);
-
-		const oneStore = new MockKeyStore(pks.keys[0], pubks.keys);
-		const twoStore = new MockKeyStore(pks.keys[1], pubks.keys);
-		const threeStore = new MockKeyStore(pks.keys[2], [pubks.keys[0], pubks.keys[2]]);
-
-		const oneSigner = new PGPSigner(oneStore);
-		const twoSigner = new PGPSigner(twoStore);
-		const threeSigner = new PGPSigner(threeStore);
-
-		const x = new Syncable('foo', {
-			bar: 'baz',		  
-		});
-		x.setSigner(oneSigner);
-
-		// TODO: make this look better
-		x.onwrap = (e) => {
-			let updateData = new ArgPair('bar', 'xyzzy');
-			x.update([updateData], 'change one');
-
-			x.onwrap = (e) => {
-				x.setSigner(twoSigner);
-				updateData = new ArgPair('bar', 42);
-				x.update([updateData], 'change two');
-
-				x.onwrap = (e) => {
-					const p = e.unwrap();
-					p.setSigner(threeSigner);
-					p.onauthenticate = (v) => {
-						assert(!v);
-					}
-					p.authenticate();
-				}
-
-				x.sign();
-			};
-
-			x.sign();
-		}
-
-		x.sign();
-
-	});
-
-	xit('sync_verify_shallow_tricked', async () => {
-		const pksa = fs.readFileSync(__dirname + '/privatekeys.asc');
-		const pks = await pgp.key.readArmored(pksa);
-		await pks.keys[0].decrypt('merman');
-		await pks.keys[1].decrypt('beastman');
-
-		const pubksa = fs.readFileSync(__dirname + '/publickeys.asc');
-		const pubks = await pgp.key.readArmored(pubksa);
-
-		const oneStore = new MockKeyStore(pks.keys[0], pubks.keys);
-		const twoStore = new MockKeyStore(pks.keys[1], pubks.keys);
-		const threeStore = new MockKeyStore(pks.keys[2], [pubks.keys[0], pubks.keys[2]]);
-
-		const oneSigner = new PGPSigner(oneStore);
-		const twoSigner = new PGPSigner(twoStore);
-		const threeSigner = new PGPSigner(threeStore);
-
-		const x = new Syncable('foo', {
-			bar: 'baz',		  
-		});
-		x.setSigner(twoSigner);
-
-		// TODO: make this look better
-		x.onwrap = (e) => {
-			let updateData = new ArgPair('bar', 'xyzzy');
-			x.update([updateData], 'change one');
-
-			x.onwrap = (e) => {
-				updateData = new ArgPair('bar', 42);
-				x.update([updateData], 'change two');
-				x.setSigner(oneSigner);
-
-				x.onwrap = (e) => {
-					const p = e.unwrap();
-					p.setSigner(threeSigner);
-					p.onauthenticate = (v) => {
-						assert(v);
-						p.onauthenticate = (v) => {
-							assert(!v);
-						}
-						p.authenticate(true);
-					}
-					p.authenticate();
-				}
-
-				x.sign();
-			};
-
-			x.sign();
-		}
-
-		x.sign();
-
-	});
-});

apps/cic-meta/tests/3_transport.ts

@@ -1,14 +0,0 @@
-import * as assert from 'assert';
-
-import { MockPubSub, MockConsumer } from './mock';
-
-describe('transport', () => {
-	it('pub_sub', () => {
-		const c = new MockConsumer();
-		const ps = new MockPubSub('foo', c);
-		ps.pub('foo');	
-		ps.pub('bar');
-		ps.flush();
-		assert.deepEqual(c.omnoms, ['foo', 'bar']);
-	});
-});

apps/cic-meta/tests/4_auth.ts

@@ -1,46 +0,0 @@
-import assert = require('assert');
-import pgp = require('openpgp');
-import crypto  = require('crypto');
-
-import { Syncable, ArgPair } from '../src/sync';
-
-import { MockKeyStore, MockSignable } from './mock';
-
-import { PGPSigner } from '../src/auth';
-
-
-describe('auth', async () => {
-	await it('digest', async () => {
-		const opts = {
-			userIds: [
-				{
-					name: 'John Marston',
-					email: 'red@dead.com',
-				},
-			],
-			numBits: 2048,
-			passphrase: 'foo',
-		};
-		const pkgen = await pgp.generateKey(opts);
-		const pka = pkgen.privateKeyArmored;
-		const pks = await pgp.key.readArmored(pka);
-		await pks.keys[0].decrypt('foo');
-		const pubka = pkgen.publicKeyArmored;
-		const pubks = await pgp.key.readArmored(pubka);
-		const keyStore = new MockKeyStore(pks.keys[0], pubks.keys);
-		const s = new PGPSigner(keyStore);
-
-		const message = await pgp.cleartext.fromText('foo');
-		s.onverify = (ok) => {
-			assert(ok);
-		}
-		s.onsign = (signature) => {
-			s.onverify((v) => {
-				console.log('bar', v);
-			});
-			s.verify('foo', signature);
-		}
-		
-		await s.sign('foo');
-	});
-});	

apps/cic-meta/tests/999_functional.ts

@@ -1,47 +0,0 @@
-import * as assert from 'assert';
-import * as pgp from 'openpgp';
-
-import { Dispatcher } from '../src/dispatch';
-import { User } from '../src/assets/user';
-import { PGPSigner, KeyStore } from '../src/auth';
-import { SubConsumer } from '../src/transport';
-
-import { MockStore, MockPubSub, MockConsumer, MockKeyStore } from './mock';
-
-async function createKeyStore() {
-	const opts = {
-		userIds: [
-			{
-				name: 'John Marston',
-				email: 'red@dead.com',
-			},
-		],
-		numBits: 2048,
-		passphrase: 'foo',
-	};
-	const pkgen = await pgp.generateKey(opts);
-	const pka = pkgen.privateKeyArmored;
-	const pks = await pgp.key.readArmored(pka);
-	await pks.keys[0].decrypt('foo');
-	return new MockKeyStore(pks.keys[0], []);
-}
-
-describe('fullchain', async () => {
-	it('dispatch_and_publish_user', async () => {
-		const g = await createKeyStore();
-		const n = new PGPSigner(g);
-		const u = new User('u1', {});
-		u.setSigner(n);
-		u.setName('Nico', 'Bellic');
-		const s = new MockStore('fooStore');
-		const c = new MockConsumer();
-		const p = new MockPubSub('fooPubSub', c);
-		const d = new Dispatcher(s, p);
-		u.onwrap = (e) => {
-			d.add(u.id, e);
-			d.sync(0);
-			assert.equal(p.pubs.length, 1);
-		};
-		u.sign();
-	});
-});

apps/cic-meta/tests/mock.ts

@@ -1,150 +0,0 @@
-import * as crypto from 'crypto';
-
-import { Signable, Signature, KeyStore } from '../src/auth';
-import { Store } from '../src/store';
-import { PubSub, SubConsumer } from '../src/transport';
-import { Syncable } from '../src/sync';
-
-class MockStore implements Store {
-	
-	contents:	Object
-	name:		string
-
-	constructor(name:string) {
-		this.name = name;
-		this.contents = {};
-	}
-
-	public put(k:string, v:Syncable, existsOk = false) {
-		if (!existsOk && this.contents[k] !== undefined) {
-			throw '"' + k + '" already exists in store ' + this.name;
-		} 
-		this.contents[k] = v;
-	}
-
-	public get(k:string): Syncable {
-		return this.contents[k];
-	}
-
-	public delete(k:string) {
-		delete this.contents[k];
-	}
-}
-
-class MockSigner {
-	onsign: (string) => void
-	onverify: (boolean) => void
-	public verify(src:string, signature:Signature) {
-		return true;
-	}
-
-	public sign(s:string):boolean {
-		this.onsign('there would be a signature here');
-		return true;
-	}
-
-	public prepare(m:Signable):boolean {
-		return true;
-	}
-
-	public fingerprint():string {
-		return '';
-	}
-}
-
-class MockConsumer implements SubConsumer {
-
-	omnoms:	Array<string>
-
-	constructor() {
-		this.omnoms = Array<string>();
-	}
-
-	public post(v:string) {
-		this.omnoms.push(v);
-	}
-}
-
-class MockPubSub implements PubSub {
-
-	pubs:		Array<string>
-	consumer:	SubConsumer
-
-	constructor(name:string, consumer:SubConsumer) {
-		this.pubs = Array<string>();	
-		this.consumer = consumer;
-	}
-	
-	public pub(v:string): boolean {
-		this.pubs.push(v);
-		return true;
-	}
-
-	public flush() {
-		while (this.pubs.length > 0) {
-			const s = this.pubs.shift();
-			this.consumer.post(s);
-		}
-	}
-
-	public close() {
-	}
-}
-
-class MockSignable implements Signable {
-
-	src:	string
-	dst:	string
-
-	constructor(src:string) {
-		this.src = src;
-	}
-
-	public digest():string {
-		const h = crypto.createHash('sha256');
-		h.update(this.src);
-		this.dst= h.digest('hex');
-		return this.dst;
-	}
-
-}
-
-class MockKeyStore implements KeyStore {
-
-	pk: any
-	pubks: Array<any>
-
-	constructor(pk:any, pubks:Array<any>) {
-		this.pk = pk;
-		this.pubks = pubks;	
-	}
-
-	public getPrivateKey(): any {
-		return this.pk;
-	}
-
-	public getTrustedKeys(): Array<any> {
-		return this.pubks;
-	}
-
-	public getTrustedActiveKeys(): Array<any> {
-		return [];
-	}
-
-	public getEncryptKeys(): Array<any> {
-		return [];
-	}
-
-	public getFingerprint(): string {
-		return '';
-	}
-}
-
-export {
-	MockStore,
-	MockPubSub,
-	MockConsumer,
-	MockSignable,
-	MockKeyStore,
-	MockSigner,
-};

apps/cic-meta/tests/server.ts

@@ -1,13 +1,10 @@
-import Automerge = require('automerge');
 import assert = require('assert');
 import fs = require('fs');
 import pgp = require('openpgp');
 import sqlite = require('sqlite3');
 
 import * as handlers from '../scripts/server/handlers';
-import { Envelope, Syncable, ArgPair } from '../src/sync';
-import { PGPKeyStore, PGPSigner, KeyStore, Signer } from '../src/auth';
-import { SqliteAdapter } from '../src/db';
+import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer, SqliteAdapter } from 'crdt-meta';
 
 function createKeystore() {
 	const pksa = fs.readFileSync(__dirname + '/privatekeys.asc', 'utf-8');

apps/cic-ussd/cic_ussd/account.py

@@ -20,7 +20,7 @@ def define_account_tx_metadata(user: Account):
     )
     key = generate_metadata_pointer(
         identifier=identifier,
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     account_metadata = get_cached_data(key=key)
 

apps/cic-ussd/cic_ussd/balance.py

@@ -80,7 +80,7 @@ def get_cached_operational_balance(blockchain_address: str):
     """
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),
-        salt='cic.balances_data'
+        salt=':cic.balances_data'
     )
     cached_balance = get_cached_data(key=key)
     if cached_balance:

apps/cic-ussd/cic_ussd/error.py

@@ -38,3 +38,13 @@ class MetadataStoreError(Exception):
     pass
 
 
+class SeppukuError(Exception):
+    """Exception base class for all errors that should cause system shutdown"""
+    pass
+
+
+class InitializationError(Exception):
+    """Exception raised when initialization state is insufficient to run component"""
+    pass
+
+

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -118,9 +118,9 @@ class MetadataRequestsHandler(Metadata):
         metadata_http_error_handler(result=result)
         response_data = result.content
         data = json.loads(response_data.decode('utf-8'))
-        if result.status_code == 200 and self.cic_type == 'cic.person':
+        if result.status_code == 200 and self.cic_type == ':cic.person':
             person = Person()
-            deserialized_person = person.deserialize(person_data=json.loads(data))
+            deserialized_person = person.deserialize(person_data=data)
             data = json.dumps(deserialized_person.serialize())
             cache_data(self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -9,4 +9,4 @@ from .base import MetadataRequestsHandler
 class PersonMetadata(MetadataRequestsHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type='cic.person', identifier=identifier)
+        super().__init__(cic_type=':cic.person', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -10,4 +10,4 @@ from .base import MetadataRequestsHandler
 class PhonePointerMetadata(MetadataRequestsHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type='cic.msisdn', identifier=identifier)
+        super().__init__(cic_type=':cic.phone', identifier=identifier)

apps/cic-ussd/cic_ussd/operations.py

@@ -48,10 +48,9 @@ def define_response_with_content(headers: list, response: str) -> tuple:
     content_length_header = ('Content-Length', str(content_length))
     # check for content length defaulted to zero in error headers
     for position, header in enumerate(headers):
-        if header[0] == 'Content-Length':
-            headers[position] = content_length_header
-        else:
-            headers.append(content_length_header)
+        if 'Content-Length' in header:
+            headers.pop(position)
+    headers.append(content_length_header)
     return response_bytes, headers
 
 
@@ -326,6 +325,14 @@ def process_menu_interaction_requests(chain_str: str,
         # get user
         user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
+        # retrieve and cache user's metadata
+        blockchain_address = user.blockchain_address
+        s_query_person_metadata = celery.signature(
+            'cic_ussd.tasks.metadata.query_person_metadata',
+            [blockchain_address]
+        )
+        s_query_person_metadata.apply_async(queue='cic-ussd')
+
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
             external_session_id=external_session_id).first()

apps/cic-ussd/cic_ussd/processor.py

@@ -7,6 +7,7 @@ from typing import Optional
 # third party imports
 import celery
 from sqlalchemy import desc
+from cic_eth.api import Api
 from tinydb.table import Document
 
 # local imports
@@ -15,7 +16,7 @@ from cic_ussd.balance import BalanceManager, compute_operational_balance, get_ca
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.db.models.ussd_session import UssdSession
-from cic_ussd.error import MetadataNotFoundError
+from cic_ussd.error import MetadataNotFoundError, SeppukuError
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.phone_number import get_user_by_phone_number
@@ -28,6 +29,38 @@ from cic_types.models.person import generate_metadata_pointer, get_contact_data_
 logg = logging.getLogger(__name__)
 
 
+def get_default_token_data():
+    chain_str = Chain.spec.__str__()
+    cic_eth_api = Api(chain_str=chain_str)
+    default_token_request_task = cic_eth_api.default_token()
+    default_token_data = default_token_request_task.get()
+    return default_token_data
+
+
+def retrieve_token_symbol(chain_str: str = Chain.spec.__str__()):
+    """
+    :param chain_str:
+    :type chain_str:
+    :return:
+    :rtype:
+    """
+    cache_key = create_cached_data_key(
+        identifier=chain_str.encode('utf-8'),
+        salt=':cic.default_token_data'
+    )
+    cached_data = get_cached_data(key=cache_key)
+    if cached_data:
+        default_token_data = json.loads(cached_data)
+        return default_token_data.get('symbol')
+    else:
+        logg.warning('Cached default token data not found. Attempting retrieval from default token API')
+        default_token_data = get_default_token_data()
+        if default_token_data:
+            return default_token_data.get('symbol')
+        else:
+            raise SeppukuError(f'Could not retrieve default token for: {chain_str}')
+
+
 def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
     """
     This method provides translation for all ussd menu entries that follow the pin authorization pattern.
@@ -73,7 +106,9 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     # compile response data
     user_input = ussd_session.get('user_input').split('*')[-1]
     transaction_amount = to_wei(value=int(user_input))
-    token_symbol = 'SRF'
+
+    # get default data
+    token_symbol = retrieve_token_symbol()
 
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
@@ -102,7 +137,7 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     :rtype: str
     """
     transaction_amount = to_wei(int(ussd_session.get('session_data').get('transaction_amount')))
-    token_symbol = 'SRF'
+    token_symbol = retrieve_token_symbol()
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
     tx_recipient_information = define_account_tx_metadata(user=recipient)
@@ -137,7 +172,7 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     tx_recipient_information = define_account_tx_metadata(user=recipient)
     tx_sender_information = define_account_tx_metadata(user=user)
 
-    token_symbol = 'SRF'
+    token_symbol = retrieve_token_symbol()
     user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
     logg.debug('Requires integration to determine user tokens.')
@@ -168,18 +203,18 @@ def process_account_balances(user: Account, display_key: str, ussd_session: dict
     logg.debug('Requires call to retrieve tax and bonus amounts')
     tax = ''
     bonus = ''
-
+    token_symbol = retrieve_token_symbol()
     return translation_for(
         key=display_key,
         preferred_language=user.preferred_language,
         operational_balance=operational_balance,
         tax=tax,
         bonus=bonus,
-        token_symbol='SRF'
+        token_symbol=token_symbol
     )
 
 
-def format_transactions(transactions: list, preferred_language: str):
+def format_transactions(transactions: list, preferred_language: str, token_symbol: str):
     
     formatted_transactions = ''
     if len(transactions) > 0:
@@ -190,7 +225,7 @@ def format_transactions(transactions: list, preferred_language: str):
             timestamp = transaction.get('timestamp')
             action_tag = transaction.get('action_tag')
             direction = transaction.get('direction')
-            token_symbol = 'SRF'
+            token_symbol = token_symbol
 
             if action_tag == 'SENT' or action_tag == 'ULITUMA':
                 formatted_transactions += f'{action_tag} {value} {token_symbol} {direction} {recipient_phone_number} {timestamp}.\n'
@@ -214,7 +249,7 @@ def process_display_user_metadata(user: Account, display_key: str):
     """
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     user_metadata = get_cached_data(key)
     if user_metadata:
@@ -251,9 +286,11 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
     """
     # retrieve cached statement
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address)
-    key = create_cached_data_key(identifier=identifier, salt='cic.statement')
+    key = create_cached_data_key(identifier=identifier, salt=':cic.statement')
     transactions = get_cached_data(key=key)
 
+    token_symbol = retrieve_token_symbol()
+
     first_transaction_set = []
     middle_transaction_set = []
     last_transaction_set = []
@@ -277,7 +314,8 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             first_transaction_set=format_transactions(
                 transactions=first_transaction_set,
-                preferred_language=user.preferred_language
+                preferred_language=user.preferred_language,
+                token_symbol=token_symbol
             )
         )
     elif display_key == 'ussd.kenya.middle_transaction_set':
@@ -286,7 +324,8 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             middle_transaction_set=format_transactions(
                 transactions=middle_transaction_set,
-                preferred_language=user.preferred_language
+                preferred_language=user.preferred_language,
+                token_symbol=token_symbol
             )
         )
 
@@ -296,7 +335,8 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             last_transaction_set=format_transactions(
                 transactions=last_transaction_set,
-                preferred_language=user.preferred_language
+                preferred_language=user.preferred_language,
+                token_symbol=token_symbol
             )
         )
 
@@ -312,37 +352,28 @@ def process_start_menu(display_key: str, user: Account):
     :return: Corresponding translation text response
     :rtype: str
     """
+    token_symbol = retrieve_token_symbol()
     chain_str = Chain.spec.__str__()
     blockchain_address = user.blockchain_address
     balance_manager = BalanceManager(address=blockchain_address,
                                      chain_str=chain_str,
-                                     token_symbol='SRF')
+                                     token_symbol=token_symbol)
 
     # get balances synchronously for display on start menu
     balances_data = balance_manager.get_balances()
 
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),
-        salt='cic.balances_data'
+        salt=':cic.balances_data'
     )
     cache_data(key=key, data=json.dumps(balances_data))
 
     # get operational balance
     operational_balance = compute_operational_balance(balances=balances_data)
 
-    # retrieve and cache account's metadata
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
-        [blockchain_address]
-    )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
-
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 
-    # TODO [Philip]: figure out how to get token symbol from a metadata layer of sorts.
-    token_symbol = 'SRF'
-
     return translation_for(
         key=display_key,
         preferred_language=user.preferred_language,
@@ -375,6 +406,13 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
     :return: A ussd menu's corresponding text value.
     :rtype: Document
     """
+    # retrieve metadata before any transition
+    key = generate_metadata_pointer(
+        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
+        cic_type=':cic.person'
+    )
+    person_metadata = get_cached_data(key=key)
+
     if ussd_session:
         if user_input == "0":
             return UssdMenu.parent_menu(menu_name=ussd_session.get('state'))
@@ -385,12 +423,6 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
         if user.has_valid_pin():
             last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
 
-            key = generate_metadata_pointer(
-                identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-                cic_type='cic.person'
-            )
-            person_metadata = get_cached_data(key=key)
-
             if last_ussd_session:
                 # get last state
                 last_state = last_ussd_session.state

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -0,0 +1,73 @@
+"""
+This module handles requests originating from CICADA or any other management client for custodial wallets, processing
+requests offering control of user account states to a staff behind the client.
+"""
+
+# standard imports
+import logging
+from urllib.parse import quote_plus
+
+# third-party imports
+from confini import Config
+
+# local imports
+from cic_ussd.db import dsn_from_config
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.operations import define_response_with_content
+from cic_ussd.requests import (get_request_endpoint,
+                               get_query_parameters,
+                               process_pin_reset_requests,
+                               process_locked_accounts_requests)
+from cic_ussd.runnable.server_base import exportable_parser, logg
+args = exportable_parser.parse_args()
+
+# define log levels
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+# parse config
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
+
+# set up db
+data_source_name = dsn_from_config(config)
+SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
+# create session for the life time of http request
+SessionBase.session = SessionBase.create_session()
+
+
+# handle requests from CICADA
+def application(env, start_response):
+    """Loads python code for application to be accessible over web server
+    :param env: Object containing server and request information
+    :type env: dict
+    :param start_response: Callable to define responses.
+    :type start_response: any
+    :return: a list containing a bytes representation of the response object
+    :rtype: list
+    """
+
+    # define headers
+    errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
+    headers = [('Content-Type', 'text/plain')]
+
+    if get_request_endpoint(env) == '/pin':
+        phone_number = get_query_parameters(env=env, query_name='phoneNumber')
+        phone_number = quote_plus(phone_number)
+        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
+        response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
+        SessionBase.session.close()
+        start_response(message, headers)
+        return [response_bytes]
+
+    # handle requests for locked accounts
+    response, message = process_locked_accounts_requests(env=env)
+    response_bytes, headers = define_response_with_content(headers=headers, response=response)
+    start_response(message, headers)
+    SessionBase.session.close()
+    return [response_bytes]
+

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -1,25 +1,23 @@
-"""Functions defining WSGI interaction with external http requests
-Defines an application function essential for the uWSGI python loader to run th python application code.
+"""This module handles requests originating from the ussd service provider.
 """
+
 # standard imports
-import argparse
-import celery
-import i18n
 import json
 import logging
-import os
-import redis
 
 # third-party imports
-from confini import Config
+import celery
+import i18n
+import redis
 from chainlib.chain import ChainSpec
-from urllib.parse import quote_plus
+from confini import Config
 
 # local imports
 from cic_ussd.chain import Chain
 from cic_ussd.db import dsn_from_config
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.encoder import PasswordEncoder
+from cic_ussd.error import InitializationError
 from cic_ussd.files.local_files import create_local_file_data_stores, json_file_parser
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata.signer import Signer
@@ -28,34 +26,17 @@ from cic_ussd.operations import (define_response_with_content,
                                  process_menu_interaction_requests,
                                  define_multilingual_responses)
 from cic_ussd.phone_number import process_phone_number
-from cic_ussd.redis import InMemoryStore
+from cic_ussd.processor import get_default_token_data
+from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore
 from cic_ussd.requests import (get_request_endpoint,
-                               get_request_method,
-                               get_query_parameters,
-                               process_locked_accounts_requests,
-                               process_pin_reset_requests)
+                               get_request_method)
+from cic_ussd.runnable.server_base import exportable_parser, logg
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.state_machine import UssdStateMachine
 from cic_ussd.validator import check_ip, check_request_content_length, check_service_code, validate_phone_number, \
     validate_presence
 
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-config_directory = '/usr/local/etc/cic-ussd/'
-
-# define arguments
-arg_parser = argparse.ArgumentParser()
-arg_parser.add_argument('-c', type=str, default=config_directory, help='config directory.')
-arg_parser.add_argument('-q', type=str, default='cic-ussd', help='queue name for worker tasks')
-arg_parser.add_argument('-v', action='store_true', help='be verbose')
-arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
-arg_parser.add_argument('--env-prefix',
-                        default=os.environ.get('CONFINI_ENV_PREFIX'),
-                        dest='env_prefix',
-                        type=str,
-                        help='environment prefix for variables to overwrite configuration')
-args = arg_parser.parse_args()
+args = exportable_parser.parse_args()
 
 # define log levels
 if args.vv:
@@ -69,7 +50,14 @@ config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-# initialize elements
+# set up db
+data_source_name = dsn_from_config(config)
+SessionBase.connect(data_source_name,
+                    pool_size=int(config.get('DATABASE_POOL_SIZE')),
+                    debug=config.true('DATABASE_DEBUG'))
+# create session for the life time of http request
+SessionBase.session = SessionBase.create_session()
+
 # set up translations
 i18n.load_path.append(config.get('APP_LOCALE_PATH'))
 i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
@@ -82,12 +70,6 @@ ussd_menu_db = create_local_file_data_stores(file_location=config.get('USSD_MENU
                                              table_name='ussd_menu')
 UssdMenu.ussd_menu_db = ussd_menu_db
 
-# set up db
-data_source_name = dsn_from_config(config)
-SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
-# create session for the life time of http request
-SessionBase.session = SessionBase.create_session()
-
 # define universal redis cache access
 InMemoryStore.cache = redis.StrictRedis(host=config.get('REDIS_HOSTNAME'),
                                         port=config.get('REDIS_PORT'),
@@ -127,6 +109,20 @@ Chain.spec = chain_spec
 UssdStateMachine.states = states
 UssdStateMachine.transitions = transitions
 
+# retrieve default token data
+default_token_data = get_default_token_data()
+chain_str = Chain.spec.__str__()
+
+# cache default token for re-usability
+if default_token_data:
+    cache_key = create_cached_data_key(
+        identifier=chain_str.encode('utf-8'),
+        salt=':cic.default_token_data'
+    )
+    cache_data(key=cache_key, data=json.dumps(default_token_data))
+else:
+    raise InitializationError(f'Default token data for: {chain_str} not found.')
+
 
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
@@ -134,6 +130,8 @@ def application(env, start_response):
     :type env: dict
     :param start_response: Callable to define responses.
     :type start_response: any
+    :return: a list containing a bytes representation of the response object
+    :rtype: list
     """
     # define headers
     errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
@@ -194,20 +192,3 @@ def application(env, start_response):
         start_response('200 OK,', headers)
         SessionBase.session.close()
         return [response_bytes]
-
-    # handle pin requests
-    if get_request_endpoint(env) == '/pin':
-        phone_number = get_query_parameters(env=env, query_name='phoneNumber')
-        phone_number = quote_plus(phone_number)
-        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
-        response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
-        SessionBase.session.close()
-        start_response(message, headers)
-        return [response_bytes]
-
-    # handle requests for locked accounts
-    response, message = process_locked_accounts_requests(env=env)
-    response_bytes, headers = define_response_with_content(headers=headers, response=response)
-    start_response(message, headers)
-    SessionBase.session.close()
-    return [response_bytes]

apps/cic-ussd/cic_ussd/runnable/server_base.py

@@ -0,0 +1,38 @@
+"""This module handles generic wsgi server configurations that can then be subsumed by different server flavors for the
+cic-ussd component.
+"""
+
+# standard imports
+import logging
+import os
+from argparse import ArgumentParser
+
+# third-party imports
+
+# local imports
+
+# define a logging system
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+# define default config directory as would be defined in docker
+default_config_dir = '/usr/local/etc/cic-ussd/'
+
+# define args parser
+arg_parser = ArgumentParser(description='CLI for handling cic-ussd server applications.')
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+arg_parser.add_argument('-v', help='be verbose', action='store_true')
+arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
+arg_parser.add_argument('-q', type=str, default='cic-ussd', help='queue name for worker tasks')
+arg_parser.add_argument('--env-prefix',
+                        default=os.environ.get('CONFINI_ENV_PREFIX'),
+                        dest='env_prefix',
+                        type=str,
+                        help='environment prefix for variables to overwrite configuration')
+exportable_parser = arg_parser
+
+
+
+
+
+

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -64,7 +64,7 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> boo
     # get cached balance
     key = create_cached_data_key(
         identifier=bytes.fromhex(user.blockchain_address[2:]),
-        salt='cic.balances_data'
+        salt=':cic.balances_data'
     )
     cached_balance = get_cached_data(key=key)
     operational_balance = compute_operational_balance(balances=json.loads(cached_balance))

apps/cic-ussd/cic_ussd/state_machine/logic/user.py

@@ -176,7 +176,7 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
     blockchain_address = user.blockchain_address
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     user_metadata = get_cached_data(key=key)
 

apps/cic-ussd/cic_ussd/state_machine/logic/validator.py

@@ -23,7 +23,7 @@ def has_cached_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     # check for user metadata in cache
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     user_metadata = get_cached_data(key=key)
     return user_metadata is not None

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -136,7 +136,7 @@ def process_balances_callback(result: list, param: str, status_code: int):
         blockchain_address = balances_data.get('address')
         key = create_cached_data_key(
             identifier=bytes.fromhex(blockchain_address[2:]),
-            salt='cic.balances_data'
+            salt=':cic.balances_data'
         )
         cache_data(key=key, data=json.dumps(balances_data))
     else:
@@ -226,7 +226,7 @@ def process_statement_callback(result, param: str, status_code: int):
 
         # cache account statement
         identifier = bytes.fromhex(param[2:])
-        key = create_cached_data_key(identifier=identifier, salt='cic.statement')
+        key = create_cached_data_key(identifier=identifier, salt=':cic.statement')
         data = json.dumps(processed_transactions)
 
         # cache statement data

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.9')
+version = (0, 3, 0, 'alpha.10')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/docker/Dockerfile

@@ -38,8 +38,9 @@ COPY cic-ussd/transitions/ cic-ussd/transitions/
 COPY cic-ussd/var/ cic-ussd/var/
 
 COPY cic-ussd/docker/db.sh \
-     cic-ussd/docker/start_tasker.sh \
-     cic-ussd/docker/start_uwsgi.sh \
+     cic-ussd/docker/start_cic_user_tasker.sh \
+     cic-ussd/docker/start_cic_user_ussd_server.sh\
+     cic-ussd/docker/start_cic_user_server.sh\
      /root/
 
 RUN chmod +x /root/*.sh

apps/cic-ussd/docker/start_cic_user_server.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. /root/db.sh
+
+user_server_port=${SERVER_PORT:-9500}
+
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_cic_user_tasker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. /root/db.sh
+
+/usr/local/bin/cic-user-tasker "$@"

apps/cic-ussd/docker/start_cic_user_ussd_server.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. /root/db.sh
+
+user_ussd_server_port=${SERVER_PORT:-9000}
+
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_tasker.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-. /root/db.sh
-
-/usr/local/bin/cic-ussd-tasker $@

apps/cic-ussd/docker/start_uwsgi.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-. /root/db.sh
-
-server_port=${SERVER_PORT:-9000}
-
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/server.py --http :$server_port --pyargv "$@"

apps/cic-ussd/setup.cfg

@@ -35,6 +35,7 @@ packages =
 	cic_ussd.menu
 	cic_ussd.metadata
 	cic_ussd.runnable
+	cic_ussd.runnable.daemons
 	cic_ussd.session
 	cic_ussd.state_machine
 	cic_ussd.state_machine.logic
@@ -44,5 +45,5 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-ussd-tasker = cic_ussd.runnable.tasker:main
+	cic-user-tasker = cic_ussd.runnable.daemons.cic_user_tasker:main
 	cic-ussd-client = cic_ussd.runnable.client:main

apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py

@@ -105,7 +105,7 @@ def test_get_user_metadata(caplog,
         assert 'Get latest data status: 200' in caplog.text
     key = generate_metadata_pointer(
         identifier=identifier,
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     cached_user_metadata = get_cached_data(key=key)
     assert cached_user_metadata

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_validator_logic.py

@@ -36,7 +36,7 @@ def test_has_cached_user_metadata(create_in_db_ussd_session,
     user = create_activated_user
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     cache_data(key=key, data=json.dumps(person_metadata))
     result = has_cached_user_metadata(state_machine_data=state_machine_data)

apps/cic-ussd/tests/fixtures/user.py

@@ -115,6 +115,6 @@ def cached_user_metadata(create_activated_user, init_redis_cache, person_metadat
     user_metadata = json.dumps(person_metadata)
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     cache_data(key=key, data=user_metadata)

apps/contract-migration/docker/Dockerfile

@@ -4,9 +4,13 @@ FROM python:3.8.6-slim-buster as compile-image
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
 
-RUN apt-get install -y software-properties-common
-RUN add-apt-repository ppa:ethereum/ethereum
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C52189C923F6CA9
+RUN touch /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' > /etc/apt/sources.list.d/ethereum.list
+RUN echo 'deb-src http://ppa.launchpad.net/ethereum/ethereum/ubuntu bionic main' >> /etc/apt/sources.list.d/ethereum.list
+
+RUN cat etc/apt/sources.list.d/ethereum.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
+
 RUN apt-get update
 RUN apt-get install solc
 RUN pip install --upgrade pip
@@ -27,15 +31,6 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-ARG cic_contracts_commit=698ef3a30fde8d7f2c498f1208fb0ff45d665501
-ARG cic_contracts_url=https://gitlab.com/grassrootseconomics/cic-contracts.git/
-RUN echo Install ABI collection for solidity interfaces used across all components && \
-	git clone --depth 1 $cic_contracts_url cic-contracts && \
-	cd cic-contracts && \
-	git fetch --depth 1 origin $cic_contracts_commit && \
-	git checkout $cic_contracts_commit && \
-	make install
-
 # Install nvm with node and npm
 # https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
 ENV NVM_DIR /root/.nvm
@@ -52,54 +47,61 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh |
 ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
 ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
 
-RUN useradd --create-home grassroots
-WORKDIR /home/grassroots
-USER grassroots
+#RUN useradd --create-home grassroots
+# WORKDIR /home/grassroots
+# USER grassroots
 
+ARG pip_extra_args=""
+ARG pip_index_url=https://pypi.org/simple
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a79
-ARG cic_eth_version=0.11.0b8+build.c2286e5c
-ARG sarafu_faucet_version=0.0.2a28
-ARG sarafu_token_version==0.0.1a6
-ARG cic_contracts_version=0.0.2a2
-RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
+ARG cic_base_version=0.1.2b8
+ARG cic_eth_version=0.11.0b12
+ARG sarafu_token_version=0.0.1a8
+ARG sarafu_faucet_version=0.0.3a2
+RUN pip install --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url \
+	cic-base[full_graph]==$cic_base_version \
 	cic-eth==$cic_eth_version \
-	cic-contracts==$cic_contracts_version \
 	sarafu-faucet==$sarafu_faucet_version \
-	sarafu-token==$sarafu_token_version
+	sarafu-token==$sarafu_token_version \
+	cic-eth==$cic_eth_version 
 
+# -------------- begin runtime container ---------------- 
 FROM python:3.8.6-slim-buster as runtime-image
 
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
-RUN apt-get install -y --no-install-recommends jq
+RUN apt-get install -y jq bash iputils-ping socat
 
 COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
 COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
+COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
 
-RUN useradd --create-home grassroots
-WORKDIR /home/grassroots
-# COPY python dependencies to user dir
-COPY --from=compile-image /home/grassroots/.local .local
-ENV PATH=/home/grassroots/.local/bin:$PATH
+ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
+# RUN useradd -u 1001 --create-home grassroots
+# RUN adduser grassroots sudo && \
+#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+# WORKDIR /home/grassroots
 
 COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
 COPY contract-migration/envlist .
+COPY contract-migration/scripts scripts/ 
 
-# RUN chown grassroots:grassroots .local/
-
-RUN mkdir -p /tmp/cic/config
-RUN chown grassroots:grassroots /tmp/cic/config
 # A shared output dir for environment configs
+RUN mkdir -p /tmp/cic/config
+# RUN chown grassroots:grassroots /tmp/cic/config
 RUN chmod a+rwx /tmp/cic/config 
 
 COPY contract-migration/*.sh ./
-RUN chown grassroots:grassroots -R .
+# RUN chown grassroots:grassroots -R .
 RUN chmod gu+x *.sh
 
+# we copied these from the root build container.
+# this is dumb though...I guess the compile image should have the same user
+# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
 
-USER grassroots
+# USER grassroots
 
 ENTRYPOINT [ ]

apps/contract-migration/scripts/README.md

@@ -104,8 +104,8 @@ If importing using `cic_eth` or `cic_ussd` also run:
 * cic-eth-retrier
 
 If importing using `cic_ussd` also run:
-* cic-ussd-tasker
-* cic-ussd-server
+* cic-user-tasker
+* cic-user-ussd-server
 * cic-notify-tasker
 
 If metadata is to be imported, also run:
@@ -169,6 +169,24 @@ In second terminal:
 
 `python cic_ussd/import_users.py -v -c config out`
 
+Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
+
+In one terminal run:
+
+`python create_import_pins.py -c config -v --userdir <path to the users export dir tree> pinsdir <path to pin export dir tree>`
+
+This script will recursively walk through all the directories defining user data in the users export directory and generate a csv file containing phone numbers and password hashes generated using fernet in a manner reflecting the nature of said hashes in the current sempo system.
+This csv file will be stored in the pins export dir defined as the positional argument.
+
+Once the creation of the pins file is complete, proceed to import the pins and ussd data as follows:
+
+- To import the pins:
+
+`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
+
+- To import ussd data:
+`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
+
 The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
 
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.

apps/contract-migration/scripts/cic_meta/import_meta.js

@@ -3,10 +3,11 @@ const path = require('path');
 const http = require('http');
 
 const cic = require('cic-client-meta');
+const crdt = require('crdt-meta');
 
 //const conf = JSON.parse(fs.readFileSync('./cic.conf'));
 
-const config = new cic.Config('./config'); 
+const config = new crdt.Config('./config');
 config.process();
 console.log(config);
 
@@ -41,7 +42,7 @@ function sendit(uid, envelope) {
 }
 
 function doOne(keystore, filePath) {
-	const signer = new cic.PGPSigner(keystore);
+	const signer = new crdt.PGPSigner(keystore);
 	const parts = path.basename(filePath).split('.');
 	const ethereum_address = path.basename(parts[0]);
 
@@ -51,7 +52,7 @@ function doOne(keystore, filePath) {
 		//console.log(o);
 		fs.unlinkSync(filePath);
 
-		const s = new cic.Syncable(uid, o);
+		const s = new crdt.Syncable(uid, o);
 		s.setSigner(signer);
 		s.onwrap = (env) => {
 			sendit(uid, env);
@@ -65,7 +66,7 @@ const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_P
 pk = fs.readFileSync(privateKeyPath);
 pubk = fs.readFileSync(publicKeyPath);
 
-new cic.PGPKeyStore(
+new crdt.PGPKeyStore(
 	config.get('PGP_PASSPHRASE'),
 	pk,
 	pubk,

apps/contract-migration/scripts/cic_meta/import_meta_phone.js

@@ -4,10 +4,11 @@ const http = require('http');
 
 const cic = require('cic-client-meta');
 const vcfp = require('vcard-parser');
+const crdt = require('crdt-meta');
 
 //const conf = JSON.parse(fs.readFileSync('./cic.conf'));
 
-const config = new cic.Config('./config'); 
+const config = new crdt.Config('./config');
 config.process();
 console.log(config);
 
@@ -42,7 +43,7 @@ function sendit(uid, envelope) {
 }
 
 function doOne(keystore, filePath, address) {
-	const signer = new cic.PGPSigner(keystore);
+	const signer = new crdt.PGPSigner(keystore);
 
 	const j = JSON.parse(fs.readFileSync(filePath).toString());
 	const b = Buffer.from(j['vcard'], 'base64');
@@ -51,9 +52,8 @@ function doOne(keystore, filePath, address) {
 	const phone = o.tel[0].value;
 
 	cic.Phone.toKey(phone).then((uid) => {
-		const o = fs.readFileSync(filePath, 'utf-8');
 
-		const s = new cic.Syncable(uid, o);
+		const s = new crdt.Syncable(uid, address);
 		s.setSigner(signer);
 		s.onwrap = (env) => {
 			sendit(uid, env);
@@ -67,7 +67,7 @@ const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_P
 pk = fs.readFileSync(privateKeyPath);
 pubk = fs.readFileSync(publicKeyPath);
 
-new cic.PGPKeyStore(
+new crdt.PGPKeyStore(
 	config.get('PGP_PASSPHRASE'),
 	pk,
 	pubk,
@@ -123,7 +123,7 @@ function importMetaPhone(keystore) {
 		if (batchCount == batchSize) {
 			console.debug('reached batch size, breathing');
 			batchCount=0;
-			setTimeout(importMeta, batchDelay, keystore);
+			setTimeout(importMetaPhone, batchDelay, keystore);
 			return;
 		}
 	}