Merge branch 'master' into spencer/meta-multiple-hashes

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -16,7 +16,6 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -29,8 +28,10 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
+from chainsyncer.driver import (
+        HeadSyncer,
+        HistorySyncer,
+        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -112,10 +113,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, chain_interface))
+        syncers.append(HistorySyncer(syncer_backend))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, chain_interface))
+    syncers.append(HeadSyncer(syncer_backend))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/requirements.txt

@@ -1,13 +1,12 @@
-cic-base==0.1.3a3+build.984b5cff
+cic-base~=0.1.2b10
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.6a1
+cic-eth-registry~=0.5.5a4
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a1
+chainsyncer[sql]~=0.0.2a4

apps/cic-cache/scripts/migrate.py

@@ -2,7 +2,6 @@
 import os
 import argparse
 import logging
-import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -24,8 +23,6 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -56,10 +53,4 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,6 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-cic_base[full]==0.1.3a3+build.984b5cff
-sarafu-faucet~=0.0.4a1
+web3==5.12.2
+cic-eth-registry~=0.5.5a3
+cic-base[full]==0.1.2b8

apps/cic-eth/MANIFEST.in

@@ -1,2 +0,0 @@
-include *requirements.txt
-

apps/cic-eth/cic_eth/api/api_admin.py

@@ -562,13 +562,13 @@ class AdminApi:
             tx['source_token_symbol'] = source_token.symbol
             o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance(r)
+            tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
 
         if destination_token != None:
             tx['destination_token_symbol'] = destination_token.symbol
             o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
+            tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
             #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which

apps/cic-eth/cic_eth/api/api_task.py

@@ -204,82 +204,6 @@ class Api:
 #        return t
 
 
-    def transfer_from(self, from_address, to_address, value, token_symbol, spender_address):
-        """Executes a chain of celery tasks that performs a transfer of ERC20 tokens by one address on behalf of another address to a third party.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param to_address: Ethereum address of recipient
-        :type to_address: str, 0x-hex
-        :param value: Estimated return from conversion
-        :type  value: int
-        :param token_symbol: ERC20 token symbol of token to send
-        :type token_symbol: str
-        :param spender_address: Ethereum address of recipient
-        :type spender_address: str, 0x-hex
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [token_symbol],
-                    self.chain_spec.asdict(),
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_nonce = celery.signature(
-                'cic_eth.eth.nonce.reserve_nonce',
-                [
-                    self.chain_spec.asdict(),
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_tokens = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_allow = celery.signature(
-                'cic_eth.eth.erc20.check_allowance',
-                [
-                    from_address,
-                    value,
-                    self.chain_spec.asdict(),
-                    spender_address,
-                    ],
-                queue=self.queue,
-                )
-        s_transfer = celery.signature(
-                'cic_eth.eth.erc20.transfer_from',
-                [
-                    from_address,
-                    to_address,
-                    value,
-                    self.chain_spec.asdict(),
-                    spender_address,
-                    ],
-                queue=self.queue,
-                )
-        s_tokens.link(s_allow)
-        s_nonce.link(s_tokens)
-        s_check.link(s_nonce)
-        if self.callback_param != None:
-            s_transfer.link(self.callback_success)
-            s_allow.link(s_transfer).on_error(self.callback_error)
-        else:
-            s_allow.link(s_transfer)
-
-        t = s_check.apply_async(queue=self.queue)
-        return t
-
-
-
     def transfer(self, from_address, to_address, value, token_symbol):
         """Executes a chain of celery tasks that performs a transfer of ERC20 tokens from one address to another.
 

apps/cic-eth/cic_eth/error.py

@@ -80,8 +80,3 @@ class SignerError(SeppukuError):
 class RoleAgencyError(SeppukuError):
     """Exception raise when a role cannot perform its function. This is a critical exception
     """
-
-
-class YouAreBrokeError(Exception):
-    """Exception raised when a value transfer is attempted without access to sufficient funds
-    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -24,7 +24,6 @@ from cic_eth.error import (
         TokenCountError,
         PermanentTxError,
         OutOfGasError,
-        YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import (
@@ -72,117 +71,6 @@ def balance(tokens, holder_address, chain_spec_dict):
     return tokens
 
 
-@celery_app.task(bind=True)
-def check_allowance(self, tokens, holder_address, value, chain_spec_dict, spender_address):
-    """Best-effort verification that the allowance for a transfer from spend is sufficient.
-
-    :raises YouAreBrokeError: If allowance is insufficient
-    
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :param spender_address: Address of account spending on behalf of holder
-    :type spender_address: str, 0x-hex
-    :return: Token list as passed to task
-    :rtype: dict
-    """
-    logg.debug('tokens {}'.format(tokens))
-    if len(tokens) != 1:
-        raise TokenCountError
-    t = tokens[0]
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    caller_address = ERC20Token.caller_address 
-    c = ERC20(chain_spec)
-    o = c.allowance(t['address'], holder_address, spender_address, sender_address=caller_address)
-    r = rpc.do(o)
-    allowance = c.parse_allowance(r)
-    if allowance < value:
-        errstr = 'allowance {} insufficent to transfer {} {} by {} on behalf of {}'.format(allowance, value, t['symbol'], spender_address, holder_address)
-        logg.error(errstr)
-        raise YouAreBrokeError(errstr)
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
-def transfer_from(self, tokens, holder_address, receiver_address, value, chain_spec_dict, spender_address):
-    """Transfer ERC20 tokens between addresses
-
-    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
-
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param receiver_address: Token receiver address
-    :type receiver_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :param spender_address: Address of account spending on behalf of holder
-    :type spender_address: str, 0x-hex
-    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
-    :return: Transaction hash for tranfer operation
-    :rtype: str, 0x-hex
-    """
-    # we only allow one token, one transfer
-    logg.debug('tokens {}'.format(tokens))
-    if len(tokens) != 1:
-        raise TokenCountError
-    t = tokens[0]
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    queue = self.request.delivery_info.get('routing_key')
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
-
-    session = self.create_session()
-    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
-    c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
-    
-
-    rpc_signer.disconnect()
-    rpc.disconnect()
-
-    cache_task = 'cic_eth.eth.erc20.cache_transfer_from_data'
-
-    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
-    session.commit()
-    session.close()
-    
-    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
-    gas_budget = gas_pair[0] * gas_pair[1]
-    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
-
-    s = create_check_gas_task(
-             [tx_signed_raw_hex],
-             chain_spec,
-             holder_address,
-             gas_budget,
-             [tx_hash_hex],
-             queue,
-            )
-    s.apply_async()
-    return tx_hash_hex
-
-
-
 @celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
     """Transfer ERC20 tokens between addresses
@@ -344,7 +232,6 @@ def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
         logg.debug('token {}'.format(token_address))
         tokens.append({
             'address': token_address,
-            'symbol': token_symbol,
             'converters': [],
             })
     rpc.disconnect()
@@ -392,48 +279,6 @@ def cache_transfer_data(
     return (tx_hash_hex, cache_id)
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def cache_transfer_from_data(
-    tx_hash_hex,
-    tx_signed_raw_hex,
-    chain_spec_dict,
-        ):
-    """Helper function for otx_cache_transfer_from
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx: Signed raw transaction
-    :type tx: str, 0x-hex
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
-
-    tx_data = ERC20.parse_transfer_from_request(tx['data'])
-    spender_address = tx_data[0]
-    recipient_address = tx_data[1]
-    token_value = tx_data[2]
-
-    session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
-    session.close()
-    return (tx_hash_hex, cache_id)
-
-
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def cache_approve_data(
     tx_hash_hex,

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -0,0 +1,136 @@
+# standard imports
+import os
+import re
+import logging
+import argparse
+import json
+
+# third-party imports
+import web3
+import confini
+import celery
+from json.decoder import JSONDecodeError
+from cic_registry.chain import ChainSpec
+
+# local imports
+from cic_eth.db import dsn_from_config
+from cic_eth.db.models.base import SessionBase
+from cic_eth.eth.util import unpack_signed_raw_tx
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+dbdir = os.path.join(rootdir, 'cic_eth', 'db')
+migrationsdir = os.path.join(dbdir, 'migrations')
+
+config_dir = os.path.join('/usr/local/etc/cic-eth')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+args = argparser.parse_args()
+
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
+
+dsn = dsn_from_config(config)
+SessionBase.connect(dsn)
+
+celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+queue = args.q
+
+re_something = r'^/something/?'
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+
+
+def process_something(session, env):
+    r = re.match(re_something, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    #if env.get('CONTENT_TYPE') != 'application/json':
+    #    raise AttributeError('content type')
+
+    #if env.get('REQUEST_METHOD') != 'POST':
+    #    raise AttributeError('method')
+
+    #post_data = json.load(env.get('wsgi.input'))
+    
+    #return ('text/plain', 'foo'.encode('utf-8'),)
+
+
+# uwsgi application
+def application(env, start_response):
+    
+    for k in env.keys():
+        logg.debug('env {} {}'.format(k, env[k]))
+
+    headers = []
+    content = b''
+    err = None
+
+    session = SessionBase.create_session()
+    for handler in [
+            process_something,
+            ]:
+        try:
+            r = handler(session, env)
+        except AttributeError as e:
+            logg.error('handler fail attribute {}'.format(e))
+            err = '400 Impertinent request'
+            break
+        except JSONDecodeError as e:
+            logg.error('handler fail json {}'.format(e))
+            err = '400 Invalid data format'
+            break
+        except KeyError as e:
+            logg.error('handler fail key {}'.format(e))
+            err = '400 Invalid JSON'
+            break
+        except ValueError as e:
+            logg.error('handler fail value {}'.format(e))
+            err = '400 Invalid data'
+            break
+        except RuntimeError as e:
+            logg.error('task fail value {}'.format(e))
+            err = '500 Task failed, sorry I cannot tell you more'
+            break
+        if r != None:
+            (mime_type, content) = r
+            break
+    session.close()
+
+    if err != None:
+        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
+        start_response(err, headers)
+        session.close()
+        return [content]
+
+    headers.append(('Content-Length', str(len(content))),)
+    headers.append(('Access-Control-Allow-Origin', '*',));
+
+    if len(content) == 0:
+        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
+        start_response('404 Looked everywhere, sorry', headers)
+    else:
+        headers.append(('Content-Type', mime_type,))
+        start_response('200 OK', headers)
+
+    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,7 +15,6 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -27,8 +26,10 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
+from chainsyncer.driver import (
+        HeadSyncer,
+        HistorySyncer,
+        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -79,7 +80,6 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
-
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -121,11 +121,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend, chain_interface))
+            syncers.append(HistorySyncer(syncer_backend))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend, chain_interface))
+            syncers.append(HeadSyncer(syncer_backend))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         11,
-        1,
-        'alpha.3',
+        0,
+        'beta.16',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/requirements.txt

@@ -1,25 +1,25 @@
-cic-base==0.1.3a3+build.984b5cff
+cic-base~=0.1.2b15
 celery==4.4.7
-crypto-dev-signer~=0.4.14b6
+crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.6a1
+cic-eth-registry~=0.5.5a7
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.12a1
-erc20-transfer-authorization~=0.3.2a1
+eth_accounts_index~=0.0.11a12
+erc20-transfer-authorization~=0.3.1a7
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.2a1
-chainlib-eth~=0.0.5a1
+eth-address-index~=0.1.1a11
+chainlib~=0.0.3rc2
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.3a3
-chainqueue~=0.0.2b5
-sarafu-faucet~=0.0.4a1
-erc20-faucet~=0.2.2a1
+chainsyncer[sql]==0.0.2a5
+chainqueue~=0.0.2b3
+sarafu-faucet~=0.0.3a3
+erc20-faucet~=0.2.1a5
 coincurve==15.0.0
 potaahto~=0.0.1a2
 pycryptodome==3.10.1

apps/cic-eth/scripts/migrate.py

@@ -2,8 +2,6 @@
 import os
 import argparse
 import logging
-import re
-import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -25,8 +23,6 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -57,10 +53,4 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/tests/conftest.py

@@ -17,11 +17,11 @@ root_dir = os.path.dirname(script_dir)
 sys.path.insert(0, root_dir)
 
 # assemble fixtures
-from cic_eth.pytest.fixtures_config import *
-from cic_eth.pytest.fixtures_celery import *
-from cic_eth.pytest.fixtures_database import *
-from cic_eth.pytest.fixtures_role import *
-from cic_eth.pytest.fixtures_contract import *
+from tests.fixtures_config import *
+from tests.fixtures_database import *
+from tests.fixtures_celery import *
+from tests.fixtures_role import *
+from tests.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *

apps/cic-eth/tests/fixtures_config.py

@@ -2,13 +2,13 @@
 import os
 import logging
 
-# external imports
+# third-party imports
 import pytest
 import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(os.path.dirname(script_dir))
-logg = logging.getLogger(__name__)
+root_dir = os.path.dirname(script_dir)
+logg = logging.getLogger(__file__)
 
 
 @pytest.fixture(scope='session')

apps/cic-eth/tests/fixtures_database.py

@@ -37,8 +37,7 @@ def init_database(
         database_engine,
         ):
 
-    script_dir = os.path.dirname(os.path.realpath(__file__))
-    rootdir = os.path.dirname(os.path.dirname(script_dir))
+    rootdir = os.path.dirname(os.path.dirname(__file__))
     dbdir = os.path.join(rootdir, 'cic_eth', 'db')
     migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
     if not os.path.isdir(migrationsdir):

apps/cic-eth/tests/task/conftest.py

@@ -1 +1 @@
-from cic_eth.pytest.fixtures_celery import *
+from tests.fixtures_celery import *

apps/cic-eth/tests/task/test_task_erc20.py

@@ -13,7 +13,6 @@ from chainlib.eth.tx import (
 
 # local imports
 from cic_eth.queue.tx import register_tx
-from cic_eth.error import YouAreBrokeError
 
 logg = logging.getLogger()
 
@@ -168,101 +167,3 @@ def test_erc20_approve_task(
     r = t.get_leaf()
 
     logg.debug('result {}'.format(r))
-
-
-def test_erc20_transfer_from_task(
-        default_chain_spec,
-        foo_token,
-        agent_roles,
-        custodial_roles,
-        eth_signer,
-        eth_rpc,
-        init_database,
-        celery_session_worker,
-        token_roles,
-        ):
-
-    token_object = {
-        'address': foo_token,
-            }
-    transfer_value = 100 * (10 ** 6)
-
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
-    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
-    r = eth_rpc.do(o)
-    o = receipt(tx_hash)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    s_nonce = celery.signature(
-        'cic_eth.eth.nonce.reserve_nonce',
-        [
-            [token_object],
-            default_chain_spec.asdict(),
-            custodial_roles['FOO_TOKEN_GIFTER'],
-            ],
-        queue=None,
-            )
-    s_transfer = celery.signature(
-            'cic_eth.eth.erc20.transfer_from',
-            [
-                custodial_roles['FOO_TOKEN_GIFTER'],
-                agent_roles['BOB'],
-                transfer_value,
-                default_chain_spec.asdict(),
-                agent_roles['ALICE'],
-                ],
-            queue=None,
-            )
-    s_nonce.link(s_transfer)
-    t = s_nonce.apply_async()
-    r = t.get_leaf()
-
-    logg.debug('result {}'.format(r))
-
-
-def test_erc20_allowance_check_task(
-        default_chain_spec,
-        foo_token,
-        agent_roles,
-        custodial_roles,
-        eth_signer,
-        eth_rpc,
-        init_database,
-        celery_session_worker,
-        token_roles,
-        ):
-
-    token_object = {
-        'address': foo_token,
-        'symbol': 'FOO',
-            }
-    transfer_value = 100 * (10 ** 6)
-
-    s_check = celery.signature(
-        'cic_eth.eth.erc20.check_allowance',
-        [
-            [token_object],
-            custodial_roles['FOO_TOKEN_GIFTER'],
-            transfer_value,
-            default_chain_spec.asdict(),
-            agent_roles['ALICE']
-            ],
-        queue=None,
-            )
-    t = s_check.apply_async()
-    with pytest.raises(YouAreBrokeError):
-        t.get() 
-
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
-    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
-    r = eth_rpc.do(o)
-    o = receipt(tx_hash)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    t = s_check.apply_async()
-    t.get() 
-    assert t.successful()

apps/cic-meta/scripts/server/handlers.ts

@@ -147,7 +147,7 @@ function handleClientMergeGet(db, digest, keystore) {
 					doh(e);
 				});
 			}).catch((e) => {
-				console.error('mesage', e);
+				console.error('message', e);
 				doh(e);
 			});
 		});

apps/cic-meta/scripts/server/server.ts

@@ -87,7 +87,7 @@ async function startServer() {
 	http.createServer(processRequest).listen(config.get('SERVER_PORT'));
 }
 
-const re_digest = /^\/([a-fA-F0-9]{64})\/?$/;
+const re_digest = /^([a-fA-F0-9]{64})\/?$/;
 function parseDigest(url) {
 	const digest_test = url.match(re_digest);
 	if (digest_test === null) {
@@ -96,6 +96,42 @@ function parseDigest(url) {
 	return digest_test[1].toLowerCase();
 }
 
+function getIds(url: string): Array<string> {
+	const params: Array<string> = url.split('?')[1].split('&');
+	let ids: Array<string> = [];
+	for (let param of params) {
+		const splitParam: Array<string> = param.split('=');
+		if (splitParam[0] === 'id') {
+			ids.push(parseDigest(splitParam[1]));
+		}
+	}
+	return ids;
+}
+
+function generateResponseBody(digest: string, data: string | boolean): string {
+	let response = {
+		id: digest,
+		status: 0,
+		headers: {},
+		body: ''
+	}
+	if (typeof data === 'boolean' || data === undefined) {
+		response.body = `Metadata for identifier ${digest} not found!`;
+		response.status = 404;
+		response.headers = {"Content-Type": "text/plain"}
+	} else {
+		const responseContentLength = (new TextEncoder().encode(data)).length;
+		response.body = data;
+		response.status = 200;
+		response.headers = {
+			"Access-Control-Allow-Origin": "*",
+			"Content-Type": 'application/json',
+			"Content-Length": responseContentLength,
+		}
+	}
+	return JSON.stringify(response);
+}
+
 async function processRequest(req, res) {
 	let digest = undefined;
 	const headers = {
@@ -119,7 +155,16 @@ async function processRequest(req, res) {
 	}
 
 	try {
-		digest = parseDigest(req.url);
+		if (req.url.includes('id')) {
+			if (req.method !== 'GET') {
+				res.writeHead(405, {"Content-Type": "text/plain"});
+				res.end();
+				return;
+			}
+			digest = getIds(req.url);
+		} else {
+			digest = parseDigest(req.url.substring(1));
+		}
 	} catch(e) {
 		console.error('digest error: ' + e)
 		res.writeHead(400, {"Content-Type": "text/plain"});
@@ -162,7 +207,24 @@ async function processRequest(req, res) {
 					break;
 
 				case 'get:automerge:client':
-					content = await handlers.handleClientMergeGet(db, digest, keystore);	
+					if (digest instanceof Array) {
+						let response = [];
+						for (let dg of digest) {
+							const metadata = await handlers.handleClientMergeGet(db, dg, keystore);
+							response.push(generateResponseBody(dg, metadata));
+						}
+						const responseContentLength = (new TextEncoder().encode(response.toString())).length;
+						res.writeHead(207, {
+							"Access-Control-Allow-Origin": "*",
+							"Content-Type": contentType,
+							"Content-Length": responseContentLength,
+						});
+						res.write(response.toString());
+						res.end();
+						return;
+					} else {
+						content = await handlers.handleClientMergeGet(db, digest, keystore);
+					}
 					break;
 
 				case 'post:automerge:server':
@@ -182,13 +244,30 @@ async function processRequest(req, res) {
 				//	break;
 
 				case 'get:automerge:none':
-					r = await handlers.handleNoMergeGet(db, digest, keystore);	
-					if (r == false) {
-						res.writeHead(404, {"Content-Type": "text/plain"});
+					if (digest instanceof Array) {
+						let response = [];
+						for (let dg of digest) {
+							const metadata = await handlers.handleNoMergeGet(db, dg, keystore);
+							response.push(generateResponseBody(dg, metadata));
+						}
+						const responseContentLength = (new TextEncoder().encode(response.toString())).length;
+						res.writeHead(207, {
+							"Access-Control-Allow-Origin": "*",
+							"Content-Type": contentType,
+							"Content-Length": responseContentLength,
+						});
+						res.write(response.toString());
 						res.end();
 						return;
+					} else {
+						r = await handlers.handleNoMergeGet(db, digest, keystore);
+						if (r == false) {
+							res.writeHead(404, {"Content-Type": "text/plain"});
+							res.end();
+							return;
+						}
+						content = r;
 					}
-					content = r;
 					break;
 
 				default:
@@ -205,7 +284,7 @@ async function processRequest(req, res) {
 
 		if (content === undefined) {
 			console.error('empty content', data);
-			res.writeHead(400, {"Content-Type": "text/plain"});
+			res.writeHead(404, {"Content-Type": "text/plain"});
 			res.end();
 			return;
 		}

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -34,8 +34,6 @@ elif args.v:
 config = confini.Config(args.c, args.env_prefix)
 config.process()
 config.add(args.q, '_CELERY_QUEUE', True)
-config.censor('API_KEY', 'AFRICASTALKING')
-config.censor('API_USERNAME', 'AFRICASTALKING')
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.7')
+version = (0, 4, 0, 'alpha.5')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/requirements.txt

@@ -1 +1 @@
-cic_base[full_graph]==0.1.3a3+build.984b5cff
+cic_base[full_graph]~=0.1.2a61

apps/cic-notify/test_requirements.txt

@@ -2,3 +2,4 @@ pytest~=6.0.1
 pytest-celery~=0.0.0a1
 pytest-mock~=3.3.1
 pysqlite3~=0.4.3
+

apps/cic-ussd/cic_ussd/processor.py

@@ -294,7 +294,6 @@ def process_display_user_metadata(user: Account, display_key: str):
             preferred_language=user.preferred_language,
             full_name=absent,
             gender=absent,
-            age=absent,
             location=absent,
             products=absent
         )

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]==0.1.3a3+build.984b5cff
-cic-eth~=0.11.1a3
-cic-notify~=0.4.0a7
+cic_base[full_graph]~=0.1.2b21
+cic-eth~=0.11.0b16
+cic-notify~=0.4.0a5
 cic-types~=0.1.0a11

apps/cic-ussd/test_requirements.txt

@@ -8,4 +8,4 @@ pytest-mock==3.3.1
 pytest-ordering==0.6
 pytest-redis==2.0.0
 requests-mock==1.8.0
-tavern==1.14.2
+tavern==1.14.2

apps/cic-ussd/tests/fixtures/integration.py

@@ -124,6 +124,46 @@ def second_profile_management_session_id() -> str:
     return session_id()
 
 
+@pytest.fixture(scope='session')
+def first_account_change_given_name() -> str:
+    return fake.first_name()
+
+
+@pytest.fixture(scope='session')
+def second_account_change_given_name() -> str:
+    return fake.first_name()
+
+
+@pytest.fixture(scope='session')
+def first_account_change_family_name() -> str:
+    return fake.last_name()
+
+
+@pytest.fixture(scope='session')
+def second_account_change_family_name() -> str:
+    return fake.last_name()
+
+
+@pytest.fixture(scope='session')
+def first_account_change_location() -> str:
+    return fake.city()
+
+
+@pytest.fixture(scope='session')
+def second_account_change_location() -> str:
+    return fake.city()
+
+
+@pytest.fixture(scope='session')
+def first_account_change_product() -> str:
+    return fake.color_name()
+
+
+@pytest.fixture(scope='session')
+def second_account_change_product() -> str:
+    return fake.color_name()
+
+
 @pytest.fixture(scope='session')
 def first_profile_management_session_id_1() -> str:
     return session_id()

apps/cic-ussd/tests/integration/README.md

@@ -1,25 +0,0 @@
-# INTEGRATION TESTING
-
-This folder contains integration tests.
-
-## OVERVIEW
-
-There are four files defining the integration tests.
-
-* **test_account_creation**: Tests account sign up process.
-* **test_transactions**: Tests transactions between two accounts.
-* **test_profile_management**: Tests that account metadata can be edited.
-* **test_account_management**: Tests that account management functionalities are intact.
-
-## REQUIREMENTS
-
-In order to run the transaction tests, please ensure that the faucet amount is set to a non-zero value, ideally `50000000`
-which is the value set in the config file `.config/test/integration.ini`.
-
-This implies setting the `DEV_FAUCET_AMOUNT` to a non-zero value before bringing up the contract-migration image:
-
-```shell
-export DEV_FAUCET_AMOUNT=50000000
-RUN_MASK=1 docker-compose up contract-migration
-RUN_MASK=2 docker-compose up contract-migration
-```

apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml

@@ -214,13 +214,12 @@ stages:
       status_code:
         - 200
       headers:
-        Content-Length: '51'
+        Content-Length: '28'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
-    delay_before: 10
+          expected_response: "CON Enter first name\n0. Back"
 
   - name: Pin number confirmation [{second_account_pin_number} - second account]
     request:
@@ -233,6 +232,227 @@ stages:
       headers:
         content-type: "application/x-www-form-urlencoded"
       method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '37'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka jina lako la kwanza\n0. Nyuma"
+
+  - name: Enter first name [first_account_given_name - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '29'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter family name\n0. Back"
+
+  - name: Enter first name [second_account_given_name - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '37'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka jina lako la mwisho\n0. Nyuma"
+
+  - name: Enter last name [first_account_family_name - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '51'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter gender\n1. Male\n2. Female\n3. Other\n0. Back"
+
+  - name: Enter last name [second_account_family_name - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '64'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n3. Nyngine\n0. Nyuma"
+
+  - name: Select gender [Male - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '31'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter your location\n0. Back"
+
+  - name: Select gender [Female - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '27'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka eneo lako\n0. Nyuma"
+
+  - name: Enter location [first_account_location - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '55'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Please enter a product or service you offer\n0. Back"
+
+  - name: Enter location [second_account_location - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '42'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka bidhaa ama huduma unauza\n0. Nyuma"
+
+  - name: Enter product [first_account_product - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}*{first_account_product}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '51'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
+    delay_before: 10
+
+  - name: Enter product [second_account_product - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}*{second_account_product}"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
     response:
       status_code:
         - 200

apps/cic-ussd/tests/integration/test_account_management.tavern.yaml

@@ -31,6 +31,7 @@ stages:
       status_code:
         - 200
       headers:
+        Content-Length: '51'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response

apps/cic-ussd/tests/integration/test_transactions.tavern.yaml

@@ -170,7 +170,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
+          expected_response: "CON {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
 
   - name: Enter transcation amount [second account]
     request:
@@ -191,7 +191,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
+          expected_response: "CON {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
 
   - name: Pin to authorize transaction [first account]
     request:
@@ -212,7 +212,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Your request has been sent. {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\n00. Back\n99. Exit"
+          expected_response: "CON Your request has been sent. {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\n00. Back\n99. Exit"
 
   - name: Pin to authorize transaction [second account]
     request:
@@ -233,7 +233,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Ombi lako limetumwa. {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
+          expected_response: "CON Ombi lako limetumwa. {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
 
   - name: Verify balance changes [first account]
     delay_before: 10

apps/cic-ussd/transitions/age_setting_transitions.json

@@ -12,7 +12,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_date_of_birth",
-    "dest": "enter_location",
+    "dest": "enter_gender",
     "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"

apps/cic-ussd/transitions/gender_setting_transitions.json

@@ -2,7 +2,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_gender",
-    "dest": "enter_date_of_birth",
+    "dest": "enter_location",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "conditions": "cic_ussd.state_machine.logic.validator.is_valid_gender_selection",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"

apps/cic-ussd/transitions/name_setting_transitions.json

@@ -15,7 +15,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_family_name",
-    "dest": "enter_gender",
+    "dest": "enter_date_of_birth",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
   },

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -55,7 +55,7 @@ en:
       CON My profile
       1. Edit name
       2. Edit gender
-      3. Edit age
+      3. Edit Age
       4. Edit location
       5. Edit products
       6. View my profile

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -55,7 +55,7 @@ sw:
       CON Wasifu wangu
       1. Weka jina
       2. Weka jinsia
-      3. Weka umri
+      3  Weka umri
       4. Weka eneo
       5. Weka bidhaa
       6. Angalia wasifu wako

apps/data-seeding/README.md

@@ -171,11 +171,7 @@ Then, in sequence, run in first terminal:
 
 In second terminal:
 
-`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> -c config out`
-
-In the event that you are running the command in a local environment you may want to consider passing the `--ussd-no-ssl` flag i.e:
-
-`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> --ussd-no-ssl -c config out`
+`python cic_ussd/import_users.py -v -c config out`
 
 
 
@@ -203,13 +199,6 @@ If _number of users_ is omitted the script will run until manually interrupted.
 If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
 
 
-### Importing preferences metadata
-
-`node cic_meta/import_meta_preferences.js <datadir> <number_of_users>`
-
-If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported.
-
-
 ##### Importing pins and ussd data (optional)
 
 Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:

apps/data-seeding/cic_eth/import_balance.py

@@ -18,17 +18,19 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
+        block_by_number,
+        Block,
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import JSONRPCRequest
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -36,7 +38,6 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
-from cic_base.eth.syncer import chain_interface
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -69,14 +70,13 @@ elif args.vv == True:
 config_dir = os.path.join(args.c)
 os.makedirs(config_dir, 0o777, True)
 config = confini.Config(config_dir, args.env_prefix)
-# override args
 config.process()
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+# override args
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'y'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -185,6 +185,27 @@ class Handler:
         #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
 
 
+#class BlockGetter:
+#
+#    def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
+#        self.conn = conn
+#        self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
+#
+#
+#    def get(self, n):
+#        o = block_by_number(n)
+#        r = self.conn.do(o)
+#        b = None
+#        try:
+#            b = Block(r)
+#        except TypeError as e:
+#            if r == None:
+#                logg.debug('block not found {}'.format(n))
+#            else:
+#                logg.error('block retrieve error {}'.format(e))
+#        return b
+
+
 def progress_callback(block_number, tx_index):
     sys.stdout.write(str(block_number).ljust(200) + "\n")
 
@@ -205,13 +226,11 @@ def main():
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
-   
-    j = JSONRPCRequest()
-    o = j.template()
+    
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -225,11 +244,10 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = j.template()
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     try:
         sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
@@ -287,7 +305,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/config/cic.ini

@@ -7,5 +7,4 @@ approval_escrow_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-user_ussd_svc_service_port =
-
+user_ussd_svc_service_port=

apps/data-seeding/config/eth.ini

@@ -1,2 +1,8 @@
 [eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
+gas_provider_address = 
+#chain_id = 
+abi_dir = /usr/local/share/cic/solidity/abi
+account_accounts_index_writer = 

apps/data-seeding/create_import_users.py

@@ -204,9 +204,9 @@ def gen():
          ]))
     if random.randint(0, 1):
         # fake.local_latitude()
-        p.location['latitude'] = (random.random() * 180) - 90
+        p.location['latitude'] = (random.random() + 180) - 90
         # fake.local_latitude()
-        p.location['longitude'] = (random.random() * 360) - 179
+        p.location['longitude'] = (random.random() + 360) - 180
 
     return (old_blockchain_checksum_address, phone, p)
 

apps/data-seeding/docker/Dockerfile

@@ -1,21 +1,19 @@
 # syntax = docker/dockerfile:1.2
-#FROM python:3.8.6-slim-buster as compile-image
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
+FROM python:3.8.6-slim-buster as compile-image
+
+WORKDIR /root
+
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
 
 RUN mkdir -vp /usr/local/etc/cic
 
-COPY data-seeding/package.json \
-     data-seeding/package-lock.json \
-     .
-
-RUN npm install
-
 COPY data-seeding/requirements.txt . 
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
-
 COPY data-seeding/ . 
 
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
+
+
 ENTRYPOINT [ ]

apps/data-seeding/eth/import_balance.py

@@ -18,24 +18,25 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
+        block_by_number,
+        Block,
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import JSONRPCRequest
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
-from cic_base.eth.syncer import chain_interface
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -74,7 +75,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'y')
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -183,6 +184,27 @@ class Handler:
         #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
 
 
+#class BlockGetter:
+#
+#    def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
+#        self.conn = conn
+#        self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
+#
+#
+#    def get(self, n):
+#        o = block_by_number(n)
+#        r = self.conn.do(o)
+#        b = None
+#        try:
+#            b = Block(r)
+#        except TypeError as e:
+#            if r == None:
+#                logg.debug('block not found {}'.format(n))
+#            else:
+#                logg.error('block retrieve error {}'.format(e))
+#        return b
+
+
 def progress_callback(block_number, tx_index):
     sys.stdout.write(str(block_number).ljust(200) + "\n")
 
@@ -203,13 +225,11 @@ def main():
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
-   
-    j = JSONRPCRequest()
-    o = j.template()
+    
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -223,11 +243,10 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = j.template()
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     try:
         sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
@@ -281,7 +300,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/eth/import_users.py

@@ -59,7 +59,7 @@ config.process()
 args_override = {
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'y')
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli')
 config.add(args.user_dir, '_USERDIR', True)

apps/data-seeding/requirements.txt

@@ -1,5 +1,5 @@
-cic_base[full_graph]==0.1.3a3+build.984b5cff
-sarafu-faucet==0.0.4a1
-cic-eth==0.11.1a1
-cic-types==0.1.0a13
-crypto-dev-signer==0.4.14b6
+cic-base[full_graph]==0.1.2b15
+sarafu-faucet==0.0.3a3
+cic-eth==0.11.0b16
+cic-types==0.1.0a11
+crypto-dev-signer==0.4.14b3

apps/data-seeding/verify.py

@@ -25,7 +25,7 @@ from chainlib.eth.gas import (
 )
 from chainlib.eth.tx import TxFactory
 from chainlib.hash import keccak256_string_to_hex
-from chainlib.jsonrpc import JSONRPCRequest
+from chainlib.jsonrpc import jsonrpc_template
 from cic_types.models.person import (
     Person,
     generate_metadata_pointer,
@@ -254,10 +254,6 @@ class Verifier:
             if len(k) > 7 and k[:7] == 'verify_':
                 logg.debug('verifier has verify method {}'.format(k))
                 verifymethods.append(k[7:])
-        o = self.faucet_tx_factory.token_amount(self.faucet_address, sender_address=ZERO_ADDRESS)
-        r = self.conn.do(o)
-        self.faucet_amount = self.faucet_tx_factory.parse_token_amount(r)
-        logg.info('faucet amount set to {} at verify initialization time'.format(self.faucet_amount))
 
         self.state = VerifierState(verifymethods, active_tests=active_tests)
 
@@ -268,11 +264,9 @@ class Verifier:
         data += eth_abi.encode_single('address', address).hex()
         tx = self.tx_factory.set_code(tx, data)
         tx = self.tx_factory.normalize(tx)
-        j = JSONRPCRequest()
-        o = j.template()
+        o = jsonrpc_template()
         o['method'] = 'eth_call'
         o['params'].append(tx)
-        o = j.finalize(o)
         r = self.conn.do(o)
         logg.debug('index check for {}: {}'.format(address, r))
         n = eth_abi.decode_single('uint256', bytes.fromhex(strip_0x(r)))
@@ -288,7 +282,6 @@ class Verifier:
         except ValueError:
             actual_balance = int(r)
         balance = int(balance / 1000000) * 1000000
-        balance += self.faucet_amount
         logg.debug('balance for {}: {}'.format(address, balance))
         if balance != actual_balance:
             raise VerifierError((actual_balance, balance), 'balance')
@@ -436,12 +429,10 @@ def main():
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
     
-    j = JSONRPCRequest()
-    o = j.template()
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -450,11 +441,10 @@ def main():
     data += eth_abi.encode_single('bytes32', b'AccountRegistry').hex()
     txf.set_code(tx, data)
     
-    o = j.template()
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     account_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found account index address {}'.format(account_index_address))
@@ -463,11 +453,10 @@ def main():
     data += eth_abi.encode_single('bytes32', b'Faucet').hex()
     txf.set_code(tx, data)
     
-    o = j.template()
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     faucet_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found faucet {}'.format(faucet_address))
@@ -482,11 +471,10 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = j.template()
+    o = jsonrpc_template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
     sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token address {}'.format(sarafu_token_address))

apps/util/requirements/base_requirement.txt

@@ -1 +0,0 @@
-cic-base==0.1.3a3+build.984b5cff

apps/util/requirements/requirements.txt

@@ -1 +0,0 @@
-requirements-magic~=0.0.2

apps/util/requirements/update_base.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-which pyreq-merge &> /dev/null
-if [ $? -gt 0 ]; then
-	>&2 echo pyreq-merge missing, please install requirements
-	exit 1
-fi
-
-t=$(mktemp)
->&2 echo using tmp $t
-
-repos=(../../cic-cache ../../cic-eth ../../cic-ussd ../../data-seeding ../../cic-notify)
-
-for r in ${repos[@]}; do
-	f="$r/requirements.txt"
-	>&2 echo updating $f
-	pyreq-update $f base_requirement.txt -vv > $t 
-	cp $t $f
-
-	f="$r/test_requirements.txt"
-	>&2 echo updating $f
-	pyreq-update $f base_requirement.txt -vv > $t 
-	cp $t $f
-done

docker-compose.yml

@@ -387,6 +387,44 @@ services:
     # command: "/root/start_retry.sh -q cic-eth -vv"
 
 
+#  cic-eth-server:
+#    build:
+#      context: apps/
+#      dockerfile: cic-eth/docker/Dockerfile
+#    environment:
+#      CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC
+#      CELERY_BROKER_URL: $CELERY_BROKER_URL
+#      CELERY_RESULT_URL: $CELERY_RESULT_URL
+#      SERVER_PORT: 8000
+#    depends_on:
+#      - eth
+#      - postgres
+#      - redis
+#    ports:
+#      - ${HTTP_PORT_CIC_ETH:-63314}:8000
+#    deploy:
+#      restart_policy:
+#        condition: on-failure
+#    volumes:
+#        - contract-config:/tmp/cic/config/:ro
+#    command:
+#      - /bin/bash
+#      - -c
+#      - |
+#        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
+#        "/usr/local/bin/uwsgi" \
+#        --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \
+#        --http :80 \
+#        --pyargv -vv
+##     entrypoint:
+##       - "/usr/local/bin/uwsgi"
+##       - "--wsgi-file"
+##       - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py"
+##       - "--http"
+##       - ":80"
+#    # command: "--pyargv -vv"
+
+
 
   cic-notify-tasker:
     build: