Remove dev dockerfile

Add allowance check and transferFrom task

See merge request grassrootseconomics/cic-internal-integration!203

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -16,6 +16,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -28,10 +29,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -113,10 +112,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend))
+        syncers.append(HistorySyncer(syncer_backend, chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend))
+    syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/requirements.txt

@@ -1,12 +1,13 @@
-cic-base~=0.1.2b10
+cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.6a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a4
+chainsyncer[sql]~=0.0.3a3
+erc20-faucet~=0.2.2a1

apps/cic-cache/scripts/migrate.py

@@ -2,6 +2,7 @@
 import os
 import argparse
 import logging
+import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/test_requirements.txt

@@ -6,6 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-web3==5.12.2
-cic-eth-registry~=0.5.5a3
-cic-base[full]==0.1.2b8
+cic_base[full]==0.1.3a3+build.984b5cff
+sarafu-faucet~=0.0.4a1

apps/cic-eth/MANIFEST.in

@@ -0,0 +1,2 @@
+include *requirements.txt
+

apps/cic-eth/cic_eth/api/api_admin.py

@@ -562,13 +562,13 @@ class AdminApi:
             tx['source_token_symbol'] = source_token.symbol
             o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
+            tx['sender_token_balance'] = erc20_c.parse_balance(r)
 
         if destination_token != None:
             tx['destination_token_symbol'] = destination_token.symbol
             o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
+            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
             #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which

apps/cic-eth/cic_eth/api/api_task.py

@@ -204,6 +204,82 @@ class Api:
 #        return t
 
 
+    def transfer_from(self, from_address, to_address, value, token_symbol, spender_address):
+        """Executes a chain of celery tasks that performs a transfer of ERC20 tokens by one address on behalf of another address to a third party.
+
+        :param from_address: Ethereum address of sender
+        :type from_address: str, 0x-hex
+        :param to_address: Ethereum address of recipient
+        :type to_address: str, 0x-hex
+        :param value: Estimated return from conversion
+        :type  value: int
+        :param token_symbol: ERC20 token symbol of token to send
+        :type token_symbol: str
+        :param spender_address: Ethereum address of recipient
+        :type spender_address: str, 0x-hex
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        s_check = celery.signature(
+                'cic_eth.admin.ctrl.check_lock',
+                [
+                    [token_symbol],
+                    self.chain_spec.asdict(),
+                    LockEnum.QUEUE,
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_nonce = celery.signature(
+                'cic_eth.eth.nonce.reserve_nonce',
+                [
+                    self.chain_spec.asdict(),
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_tokens = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_allow = celery.signature(
+                'cic_eth.eth.erc20.check_allowance',
+                [
+                    from_address,
+                    value,
+                    self.chain_spec.asdict(),
+                    spender_address,
+                    ],
+                queue=self.queue,
+                )
+        s_transfer = celery.signature(
+                'cic_eth.eth.erc20.transfer_from',
+                [
+                    from_address,
+                    to_address,
+                    value,
+                    self.chain_spec.asdict(),
+                    spender_address,
+                    ],
+                queue=self.queue,
+                )
+        s_tokens.link(s_allow)
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
+        if self.callback_param != None:
+            s_transfer.link(self.callback_success)
+            s_allow.link(s_transfer).on_error(self.callback_error)
+        else:
+            s_allow.link(s_transfer)
+
+        t = s_check.apply_async(queue=self.queue)
+        return t
+
+
+
     def transfer(self, from_address, to_address, value, token_symbol):
         """Executes a chain of celery tasks that performs a transfer of ERC20 tokens from one address to another.
 

apps/cic-eth/cic_eth/error.py

@@ -80,3 +80,8 @@ class SignerError(SeppukuError):
 class RoleAgencyError(SeppukuError):
     """Exception raise when a role cannot perform its function. This is a critical exception
     """
+
+
+class YouAreBrokeError(Exception):
+    """Exception raised when a value transfer is attempted without access to sufficient funds
+    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -24,6 +24,7 @@ from cic_eth.error import (
         TokenCountError,
         PermanentTxError,
         OutOfGasError,
+        YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import (
@@ -71,6 +72,117 @@ def balance(tokens, holder_address, chain_spec_dict):
     return tokens
 
 
+@celery_app.task(bind=True)
+def check_allowance(self, tokens, holder_address, value, chain_spec_dict, spender_address):
+    """Best-effort verification that the allowance for a transfer from spend is sufficient.
+
+    :raises YouAreBrokeError: If allowance is insufficient
+    
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :return: Token list as passed to task
+    :rtype: dict
+    """
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    caller_address = ERC20Token.caller_address 
+    c = ERC20(chain_spec)
+    o = c.allowance(t['address'], holder_address, spender_address, sender_address=caller_address)
+    r = rpc.do(o)
+    allowance = c.parse_allowance(r)
+    if allowance < value:
+        errstr = 'allowance {} insufficent to transfer {} {} by {} on behalf of {}'.format(allowance, value, t['symbol'], spender_address, holder_address)
+        logg.error(errstr)
+        raise YouAreBrokeError(errstr)
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def transfer_from(self, tokens, holder_address, receiver_address, value, chain_spec_dict, spender_address):
+    """Transfer ERC20 tokens between addresses
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param receiver_address: Token receiver address
+    :type receiver_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    :return: Transaction hash for tranfer operation
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    queue = self.request.delivery_info.get('routing_key')
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+
+    session = self.create_session()
+    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+    try:
+        (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
+    except FileNotFoundError as e:
+        raise SignerError(e)
+    except ConnectionError as e:
+        raise SignerError(e)
+    
+
+    rpc_signer.disconnect()
+    rpc.disconnect()
+
+    cache_task = 'cic_eth.eth.erc20.cache_transfer_from_data'
+
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
+    session.commit()
+    session.close()
+    
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
+
+    s = create_check_gas_task(
+             [tx_signed_raw_hex],
+             chain_spec,
+             holder_address,
+             gas_budget,
+             [tx_hash_hex],
+             queue,
+            )
+    s.apply_async()
+    return tx_hash_hex
+
+
+
 @celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
     """Transfer ERC20 tokens between addresses
@@ -232,6 +344,7 @@ def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
         logg.debug('token {}'.format(token_address))
         tokens.append({
             'address': token_address,
+            'symbol': token_symbol,
             'converters': [],
             })
     rpc.disconnect()
@@ -279,6 +392,48 @@ def cache_transfer_data(
     return (tx_hash_hex, cache_id)
 
 
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def cache_transfer_from_data(
+    tx_hash_hex,
+    tx_signed_raw_hex,
+    chain_spec_dict,
+        ):
+    """Helper function for otx_cache_transfer_from
+
+    :param tx_hash_hex: Transaction hash
+    :type tx_hash_hex: str, 0x-hex
+    :param tx: Signed raw transaction
+    :type tx: str, 0x-hex
+    :returns: Transaction hash and id of cache element in storage backend, respectively
+    :rtype: tuple
+    """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
+
+    tx_data = ERC20.parse_transfer_from_request(tx['data'])
+    spender_address = tx_data[0]
+    recipient_address = tx_data[1]
+    token_value = tx_data[2]
+
+    session = SessionBase.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
+    session.close()
+    return (tx_hash_hex, cache_id)
+
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def cache_approve_data(
     tx_hash_hex,

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -2,13 +2,13 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
-logg = logging.getLogger(__file__)
+root_dir = os.path.dirname(os.path.dirname(script_dir))
+logg = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -37,7 +37,8 @@ def init_database(
         database_engine,
         ):
 
-    rootdir = os.path.dirname(os.path.dirname(__file__))
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    rootdir = os.path.dirname(os.path.dirname(script_dir))
     dbdir = os.path.join(rootdir, 'cic_eth', 'db')
     migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
     if not os.path.isdir(migrationsdir):

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -1,136 +0,0 @@
-# standard imports
-import os
-import re
-import logging
-import argparse
-import json
-
-# third-party imports
-import web3
-import confini
-import celery
-from json.decoder import JSONDecodeError
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db import dsn_from_config
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth.util import unpack_signed_raw_tx
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_eth', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-eth')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-queue = args.q
-
-re_something = r'^/something/?'
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-
-def process_something(session, env):
-    r = re.match(re_something, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    #if env.get('CONTENT_TYPE') != 'application/json':
-    #    raise AttributeError('content type')
-
-    #if env.get('REQUEST_METHOD') != 'POST':
-    #    raise AttributeError('method')
-
-    #post_data = json.load(env.get('wsgi.input'))
-    
-    #return ('text/plain', 'foo'.encode('utf-8'),)
-
-
-# uwsgi application
-def application(env, start_response):
-    
-    for k in env.keys():
-        logg.debug('env {} {}'.format(k, env[k]))
-
-    headers = []
-    content = b''
-    err = None
-
-    session = SessionBase.create_session()
-    for handler in [
-            process_something,
-            ]:
-        try:
-            r = handler(session, env)
-        except AttributeError as e:
-            logg.error('handler fail attribute {}'.format(e))
-            err = '400 Impertinent request'
-            break
-        except JSONDecodeError as e:
-            logg.error('handler fail json {}'.format(e))
-            err = '400 Invalid data format'
-            break
-        except KeyError as e:
-            logg.error('handler fail key {}'.format(e))
-            err = '400 Invalid JSON'
-            break
-        except ValueError as e:
-            logg.error('handler fail value {}'.format(e))
-            err = '400 Invalid data'
-            break
-        except RuntimeError as e:
-            logg.error('task fail value {}'.format(e))
-            err = '500 Task failed, sorry I cannot tell you more'
-            break
-        if r != None:
-            (mime_type, content) = r
-            break
-    session.close()
-
-    if err != None:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response(err, headers)
-        session.close()
-        return [content]
-
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',));
-
-    if len(content) == 0:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response('404 Looked everywhere, sorry', headers)
-    else:
-        headers.append(('Content-Type', mime_type,))
-        start_response('200 OK', headers)
-
-    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,6 +15,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -26,10 +27,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -80,6 +79,7 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -121,11 +121,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend))
+            syncers.append(HistorySyncer(syncer_backend, chain_interface))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend))
+            syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         11,
-        0,
-        'beta.16',
+        1,
+        'alpha.3',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/requirements.txt

@@ -1,25 +1,25 @@
-cic-base~=0.1.2b15
+cic-base==0.1.3a3+build.984b5cff
 celery==4.4.7
-crypto-dev-signer~=0.4.14b3
+crypto-dev-signer~=0.4.14b6
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a7
+cic-eth-registry~=0.5.6a1
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a7
+eth_accounts_index~=0.0.12a1
+erc20-transfer-authorization~=0.3.2a1
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a11
-chainlib~=0.0.3rc2
+eth-address-index~=0.1.2a1
+chainlib-eth~=0.0.5a1
 hexathon~=0.0.1a7
-chainsyncer[sql]==0.0.2a5
-chainqueue~=0.0.2b3
-sarafu-faucet~=0.0.3a3
-erc20-faucet~=0.2.1a5
+chainsyncer[sql]~=0.0.3a3
+chainqueue~=0.0.2b5
+sarafu-faucet~=0.0.4a1
+erc20-faucet~=0.2.2a1
 coincurve==15.0.0
 potaahto~=0.0.1a2
 pycryptodome==3.10.1

apps/cic-eth/scripts/migrate.py

@@ -2,6 +2,8 @@
 import os
 import argparse
 import logging
+import re
+import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/tests/conftest.py

@@ -17,11 +17,11 @@ root_dir = os.path.dirname(script_dir)
 sys.path.insert(0, root_dir)
 
 # assemble fixtures
-from tests.fixtures_config import *
-from tests.fixtures_database import *
-from tests.fixtures_celery import *
-from tests.fixtures_role import *
-from tests.fixtures_contract import *
+from cic_eth.pytest.fixtures_config import *
+from cic_eth.pytest.fixtures_celery import *
+from cic_eth.pytest.fixtures_database import *
+from cic_eth.pytest.fixtures_role import *
+from cic_eth.pytest.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *

apps/cic-eth/tests/task/conftest.py

@@ -1 +1 @@
-from tests.fixtures_celery import *
+from cic_eth.pytest.fixtures_celery import *

apps/cic-eth/tests/task/test_task_erc20.py

@@ -13,6 +13,7 @@ from chainlib.eth.tx import (
 
 # local imports
 from cic_eth.queue.tx import register_tx
+from cic_eth.error import YouAreBrokeError
 
 logg = logging.getLogger()
 
@@ -167,3 +168,101 @@ def test_erc20_approve_task(
     r = t.get_leaf()
 
     logg.debug('result {}'.format(r))
+
+
+def test_erc20_transfer_from_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    s_nonce = celery.signature(
+        'cic_eth.eth.nonce.reserve_nonce',
+        [
+            [token_object],
+            default_chain_spec.asdict(),
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            ],
+        queue=None,
+            )
+    s_transfer = celery.signature(
+            'cic_eth.eth.erc20.transfer_from',
+            [
+                custodial_roles['FOO_TOKEN_GIFTER'],
+                agent_roles['BOB'],
+                transfer_value,
+                default_chain_spec.asdict(),
+                agent_roles['ALICE'],
+                ],
+            queue=None,
+            )
+    s_nonce.link(s_transfer)
+    t = s_nonce.apply_async()
+    r = t.get_leaf()
+
+    logg.debug('result {}'.format(r))
+
+
+def test_erc20_allowance_check_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+        'symbol': 'FOO',
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    s_check = celery.signature(
+        'cic_eth.eth.erc20.check_allowance',
+        [
+            [token_object],
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            transfer_value,
+            default_chain_spec.asdict(),
+            agent_roles['ALICE']
+            ],
+        queue=None,
+            )
+    t = s_check.apply_async()
+    with pytest.raises(YouAreBrokeError):
+        t.get() 
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    t = s_check.apply_async()
+    t.get() 
+    assert t.successful()

apps/cic-meta/scripts/server/handlers.ts

@@ -147,7 +147,7 @@ function handleClientMergeGet(db, digest, keystore) {
 					doh(e);
 				});
 			}).catch((e) => {
-				console.error('message', e);
+				console.error('mesage', e);
 				doh(e);
 			});
 		});

apps/cic-meta/scripts/server/server.ts

@@ -87,7 +87,7 @@ async function startServer() {
 	http.createServer(processRequest).listen(config.get('SERVER_PORT'));
 }
 
-const re_digest = /^([a-fA-F0-9]{64})\/?$/;
+const re_digest = /^\/([a-fA-F0-9]{64})\/?$/;
 function parseDigest(url) {
 	const digest_test = url.match(re_digest);
 	if (digest_test === null) {
@@ -96,42 +96,6 @@ function parseDigest(url) {
 	return digest_test[1].toLowerCase();
 }
 
-function getIds(url: string): Array<string> {
-	const params: Array<string> = url.split('?')[1].split('&');
-	let ids: Array<string> = [];
-	for (let param of params) {
-		const splitParam: Array<string> = param.split('=');
-		if (splitParam[0] === 'id') {
-			ids.push(parseDigest(splitParam[1]));
-		}
-	}
-	return ids;
-}
-
-function generateResponseBody(digest: string, data: string | boolean): string {
-	let response = {
-		id: digest,
-		status: 0,
-		headers: {},
-		body: ''
-	}
-	if (typeof data === 'boolean' || data === undefined) {
-		response.body = `Metadata for identifier ${digest} not found!`;
-		response.status = 404;
-		response.headers = {"Content-Type": "text/plain"}
-	} else {
-		const responseContentLength = (new TextEncoder().encode(data)).length;
-		response.body = data;
-		response.status = 200;
-		response.headers = {
-			"Access-Control-Allow-Origin": "*",
-			"Content-Type": 'application/json',
-			"Content-Length": responseContentLength,
-		}
-	}
-	return JSON.stringify(response);
-}
-
 async function processRequest(req, res) {
 	let digest = undefined;
 	const headers = {
@@ -155,16 +119,7 @@ async function processRequest(req, res) {
 	}
 
 	try {
-		if (req.url.includes('id')) {
-			if (req.method !== 'GET') {
-				res.writeHead(405, {"Content-Type": "text/plain"});
-				res.end();
-				return;
-			}
-			digest = getIds(req.url);
-		} else {
-			digest = parseDigest(req.url.substring(1));
-		}
+		digest = parseDigest(req.url);
 	} catch(e) {
 		console.error('digest error: ' + e)
 		res.writeHead(400, {"Content-Type": "text/plain"});
@@ -207,24 +162,7 @@ async function processRequest(req, res) {
 					break;
 
 				case 'get:automerge:client':
-					if (digest instanceof Array) {
-						let response = [];
-						for (let dg of digest) {
-							const metadata = await handlers.handleClientMergeGet(db, dg, keystore);
-							response.push(generateResponseBody(dg, metadata));
-						}
-						const responseContentLength = (new TextEncoder().encode(response.toString())).length;
-						res.writeHead(207, {
-							"Access-Control-Allow-Origin": "*",
-							"Content-Type": contentType,
-							"Content-Length": responseContentLength,
-						});
-						res.write(response.toString());
-						res.end();
-						return;
-					} else {
-						content = await handlers.handleClientMergeGet(db, digest, keystore);
-					}
+					content = await handlers.handleClientMergeGet(db, digest, keystore);	
 					break;
 
 				case 'post:automerge:server':
@@ -244,30 +182,13 @@ async function processRequest(req, res) {
 				//	break;
 
 				case 'get:automerge:none':
-					if (digest instanceof Array) {
-						let response = [];
-						for (let dg of digest) {
-							const metadata = await handlers.handleNoMergeGet(db, dg, keystore);
-							response.push(generateResponseBody(dg, metadata));
-						}
-						const responseContentLength = (new TextEncoder().encode(response.toString())).length;
-						res.writeHead(207, {
-							"Access-Control-Allow-Origin": "*",
-							"Content-Type": contentType,
-							"Content-Length": responseContentLength,
-						});
-						res.write(response.toString());
+					r = await handlers.handleNoMergeGet(db, digest, keystore);	
+					if (r == false) {
+						res.writeHead(404, {"Content-Type": "text/plain"});
 						res.end();
 						return;
-					} else {
-						r = await handlers.handleNoMergeGet(db, digest, keystore);
-						if (r == false) {
-							res.writeHead(404, {"Content-Type": "text/plain"});
-							res.end();
-							return;
-						}
-						content = r;
 					}
+					content = r;
 					break;
 
 				default:
@@ -284,7 +205,7 @@ async function processRequest(req, res) {
 
 		if (content === undefined) {
 			console.error('empty content', data);
-			res.writeHead(404, {"Content-Type": "text/plain"});
+			res.writeHead(400, {"Content-Type": "text/plain"});
 			res.end();
 			return;
 		}

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.5')
+version = (0, 4, 0, 'alpha.7')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/requirements.txt

@@ -1 +1 @@
-cic_base[full_graph]~=0.1.2a61
+cic_base[full_graph]==0.1.3a3+build.984b5cff

apps/cic-notify/test_requirements.txt

@@ -2,4 +2,3 @@ pytest~=6.0.1
 pytest-celery~=0.0.0a1
 pytest-mock~=3.3.1
 pysqlite3~=0.4.3
-

apps/cic-ussd/cic_ussd/processor.py

@@ -294,6 +294,7 @@ def process_display_user_metadata(user: Account, display_key: str):
             preferred_language=user.preferred_language,
             full_name=absent,
             gender=absent,
+            age=absent,
             location=absent,
             products=absent
         )

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b21
-cic-eth~=0.11.0b16
-cic-notify~=0.4.0a5
+cic_base[full_graph]==0.1.3a3+build.984b5cff
+cic-eth~=0.11.1a3
+cic-notify~=0.4.0a7
 cic-types~=0.1.0a11

apps/cic-ussd/test_requirements.txt

@@ -8,4 +8,4 @@ pytest-mock==3.3.1
 pytest-ordering==0.6
 pytest-redis==2.0.0
 requests-mock==1.8.0
-tavern==1.14.2
+tavern==1.14.2

apps/cic-ussd/tests/fixtures/integration.py

@@ -124,46 +124,6 @@ def second_profile_management_session_id() -> str:
     return session_id()
 
 
-@pytest.fixture(scope='session')
-def first_account_change_given_name() -> str:
-    return fake.first_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_given_name() -> str:
-    return fake.first_name()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_family_name() -> str:
-    return fake.last_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_family_name() -> str:
-    return fake.last_name()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_location() -> str:
-    return fake.city()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_location() -> str:
-    return fake.city()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_product() -> str:
-    return fake.color_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_product() -> str:
-    return fake.color_name()
-
-
 @pytest.fixture(scope='session')
 def first_profile_management_session_id_1() -> str:
     return session_id()

apps/cic-ussd/tests/integration/README.md

@@ -0,0 +1,25 @@
+# INTEGRATION TESTING
+
+This folder contains integration tests.
+
+## OVERVIEW
+
+There are four files defining the integration tests.
+
+* **test_account_creation**: Tests account sign up process.
+* **test_transactions**: Tests transactions between two accounts.
+* **test_profile_management**: Tests that account metadata can be edited.
+* **test_account_management**: Tests that account management functionalities are intact.
+
+## REQUIREMENTS
+
+In order to run the transaction tests, please ensure that the faucet amount is set to a non-zero value, ideally `50000000`
+which is the value set in the config file `.config/test/integration.ini`.
+
+This implies setting the `DEV_FAUCET_AMOUNT` to a non-zero value before bringing up the contract-migration image:
+
+```shell
+export DEV_FAUCET_AMOUNT=50000000
+RUN_MASK=1 docker-compose up contract-migration
+RUN_MASK=2 docker-compose up contract-migration
+```

apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml

@@ -214,12 +214,13 @@ stages:
       status_code:
         - 200
       headers:
-        Content-Length: '28'
+        Content-Length: '51'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Enter first name\n0. Back"
+          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
+    delay_before: 10
 
   - name: Pin number confirmation [{second_account_pin_number} - second account]
     request:
@@ -232,227 +233,6 @@ stages:
       headers:
         content-type: "application/x-www-form-urlencoded"
       method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '37'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jina lako la kwanza\n0. Nyuma"
-
-  - name: Enter first name [first_account_given_name - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '29'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter family name\n0. Back"
-
-  - name: Enter first name [second_account_given_name - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '37'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jina lako la mwisho\n0. Nyuma"
-
-  - name: Enter last name [first_account_family_name - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '51'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter gender\n1. Male\n2. Female\n3. Other\n0. Back"
-
-  - name: Enter last name [second_account_family_name - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '64'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n3. Nyngine\n0. Nyuma"
-
-  - name: Select gender [Male - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '31'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter your location\n0. Back"
-
-  - name: Select gender [Female - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '27'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka eneo lako\n0. Nyuma"
-
-  - name: Enter location [first_account_location - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '55'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Please enter a product or service you offer\n0. Back"
-
-  - name: Enter location [second_account_location - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '42'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka bidhaa ama huduma unauza\n0. Nyuma"
-
-  - name: Enter product [first_account_product - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}*{first_account_product}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '51'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
-    delay_before: 10
-
-  - name: Enter product [second_account_product - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}*{second_account_product}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
     response:
       status_code:
         - 200

apps/cic-ussd/tests/integration/test_account_management.tavern.yaml

@@ -31,7 +31,6 @@ stages:
       status_code:
         - 200
       headers:
-        Content-Length: '51'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response

apps/cic-ussd/tests/integration/test_transactions.tavern.yaml

@@ -170,7 +170,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
+          expected_response: "CON {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
 
   - name: Enter transcation amount [second account]
     request:
@@ -191,7 +191,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
+          expected_response: "CON {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
 
   - name: Pin to authorize transaction [first account]
     request:
@@ -212,7 +212,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Your request has been sent. {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\n00. Back\n99. Exit"
+          expected_response: "CON Your request has been sent. {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\n00. Back\n99. Exit"
 
   - name: Pin to authorize transaction [second account]
     request:
@@ -233,7 +233,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Ombi lako limetumwa. {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
+          expected_response: "CON Ombi lako limetumwa. {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
 
   - name: Verify balance changes [first account]
     delay_before: 10

apps/cic-ussd/transitions/age_setting_transitions.json

@@ -12,7 +12,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_date_of_birth",
-    "dest": "enter_gender",
+    "dest": "enter_location",
     "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"

apps/cic-ussd/transitions/gender_setting_transitions.json

@@ -2,7 +2,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_gender",
-    "dest": "enter_location",
+    "dest": "enter_date_of_birth",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "conditions": "cic_ussd.state_machine.logic.validator.is_valid_gender_selection",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"

apps/cic-ussd/transitions/name_setting_transitions.json

@@ -15,7 +15,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_family_name",
-    "dest": "enter_date_of_birth",
+    "dest": "enter_gender",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
   },

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -55,7 +55,7 @@ en:
       CON My profile
       1. Edit name
       2. Edit gender
-      3. Edit Age
+      3. Edit age
       4. Edit location
       5. Edit products
       6. View my profile

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -55,7 +55,7 @@ sw:
       CON Wasifu wangu
       1. Weka jina
       2. Weka jinsia
-      3  Weka umri
+      3. Weka umri
       4. Weka eneo
       5. Weka bidhaa
       6. Angalia wasifu wako

apps/contract-migration/docker/Dockerfile

@@ -22,7 +22,7 @@ COPY contract-migration/nvm.sh .
 ENV CONFINI_DIR /usr/local/etc/cic/
 RUN mkdir -vp $CONFINI_DIR
 
-ARG cic_config_commit=35c69ba75f00c8147150acf325565d5391cf25bf
+ARG cic_config_commit=0abe0867f18077907c7023bf0ef5e466a3984dd8
 ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
 RUN echo Install confini schema files && \
 	git clone --depth 1 $cic_config_url cic-config && \
@@ -56,7 +56,9 @@ COPY contract-migration/requirements.txt .
 ARG pip_extra_args=""
 ARG pip_index_url=https://pypi.org/simple
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-RUN pip install --index-url https://pypi.org/simple \
+ARG pip_trusted_host=pypi.org
+RUN pip install --index-url $pip_index_url \
+    --trusted-host $pip_trusted_host \
     --extra-index-url $pip_extra_index_url -r requirements.txt
 
 # -------------- begin runtime container ---------------- 
@@ -97,4 +99,13 @@ RUN chmod gu+x *.sh
 
 # USER grassroots
 
+ARG pip_index_url=https://pypi.org/simple
+ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
+ARG pip_trusted_host=pypi.org
+COPY contract-migration/override_requirements.txt . 
+RUN pip install --index-url $pip_index_url \
+    --force-reinstall \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $pip_extra_index_url -r override_requirements.txt
+
 ENTRYPOINT [ ]

apps/contract-migration/override_requirements.txt

@@ -0,0 +1,3 @@
+#eth-contract-registry==0.5.5a3
+#erc20-demurrage-token==0.0.1b4
+#eth-address-index==0.1.1a12

apps/contract-migration/requirements.txt

@@ -1,4 +1,4 @@
-	cic-base[full_graph]==0.1.2b15
-	sarafu-faucet==0.0.3a3
-	sarafu-token==0.0.1a8
-	cic-eth==0.11.0b16 
+cic_base[full]==0.1.3a3+build.984b5cff
+sarafu-faucet~=0.0.4a1
+cic-eth~=0.11.1a2
+erc20-demurrage-token==0.0.2a1

apps/contract-migration/reset.sh

@@ -2,8 +2,20 @@
 
 set -a
 
-CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
 CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL}
+cat <<EOF
+external token settings:
+token_symbol: $TOKEN_SYMBOL
+token_name: $TOKEN_NAME
+token_decimals: $TOKEN_DECIMALS
+token_demurrage: $TOKEN_DEMURRAGE_LEVEL
+token_redistribution_period: $TOKEN_REDISTRIBUTION_PERIOD
+token_supply_limit: $TOKEN_SUPPLY_LIMIT
+EOF
+
+CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
+TOKEN_SYMBOL=${TOKEN_SYMBOL:-GFT}
 DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
@@ -19,8 +31,8 @@ if [ ! -z $DEV_ETH_GAS_PRICE ]; then
 	>&2 echo using static gas price $DEV_ETH_GAS_PRICE
 fi
 
-if [[ $CIC_DEFAULT_TOKEN_SYMBOL != 'GFT' && $CIC_DEFAULT_TOKEN_SYMBOL != 'SRF' ]]; then
-	>&2 echo CIC_DEFAULT_TOKEN_SYMBOL must be one of [GFT,SRF], but was $CIC_DEFAULT_TOKEN_SYMBOL
+if [[ $TOKEN_SYMBOL != 'GFT' && $TOKEN_SYMBOL != 'SRF' ]]; then
+	>&2 echo TOKEN_SYMBOL must be one of [GFT,SRF], but was $TOKEN_SYMBOL
 	exit 1
 fi
 
@@ -60,17 +72,18 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 	fi
 
-	if [ $CIC_DEFAULT_TOKEN_SYMBOL == 'GFT' ]; then
+	if [ $TOKEN_SYMBOL == 'GFT' ]; then
 		>&2 echo "deploying 'giftable token'"
-		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w --name "Giftable Token" --symbol "GFT" --decimals 6 -vv`
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "Giftable Token" --symbol "GFT" --decimals 6 -vv`
 	else
-		>&2 echo "deploying 'sarafu' token'"
-		DEV_RESERVE_ADDRESS=`sarafu-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w --name "Sarafu" --decimals 6 -vv SRF $DEV_SARAFU_DEMURRAGE_LEVEL`
+		>&2 echo "deploying 'redistributed demurrage token'"
+		if [ -z $TOKEN_SINK_ADDRESS && ! -z $TOKEN_REDISTRIBUTION_PERIOD ]; then
+			>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
+		fi
+		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww`
 	fi
 	giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
 
-	#BANCOR_REGISTRY_ADDRESS=`cic-bancor-deploy $gas_price_arg --bancor-dir /usr/local/share/cic/bancor -z $DEV_ETH_RESERVE_ADDRESS -p $ETH_PROVIDER -o $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
-
 	>&2 echo "deploy account index contract"
 	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -vv -w`
 	>&2 echo "add deployer address as account index writer"

apps/contract-migration/run_job.sh

@@ -2,13 +2,22 @@
 
 if [[ $((RUN_MASK & 1)) -eq 1 ]]
 then
+	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 1 - contract deployment"
 	./reset.sh
 	if [ $? -ne "0" ]; then
-	  exit 1;
+		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 1 - contract deployment"
+		exit 1;
 	fi
+	>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 1 - contract deployment"
 fi
 
 if [[ $((RUN_MASK & 2)) -eq 2 ]]
 then
+	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 2 - custodial service initialization"
 	./seed_cic_eth.sh
+	if [ $? -ne "0" ]; then
+		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 2 - custodial service initialization"
+		exit 1;
+	fi
+	>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 2 - custodial service initialization"
 fi

apps/contract-migration/seed_cic_eth.sh

@@ -14,6 +14,7 @@ DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"}
 CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic} 
 ETH_PASSPHRASE=''
 CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
 if [[ $CIC_DEFAULT_TOKEN_SYMBOL != 'GFT' && $CIC_DEFAULT_TOKEN_SYMBOL != 'SRF' ]]; then
 	>&2 echo CIC_DEFAULT_TOKEN_SYMBOL must be one of [GFT,SRF], but was $CIC_DEFAULT_TOKEN_SYMBOL
 	exit 1
@@ -98,8 +99,8 @@ echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >> $env_out_file
 export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 # Transfer tokens to gifter address
->&2 echo "transfer sarafu tokens to token gifter address"
->&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
+>&2 echo "transfer tokens to token gifter address"
+>&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
 
 #echo -n 0 > $init_level_file
 

apps/data-seeding/README.md

@@ -171,7 +171,11 @@ Then, in sequence, run in first terminal:
 
 In second terminal:
 
-`python cic_ussd/import_users.py -v -c config out`
+`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> -c config out`
+
+In the event that you are running the command in a local environment you may want to consider passing the `--ussd-no-ssl` flag i.e:
+
+`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> --ussd-no-ssl -c config out`
 
 
 
@@ -199,6 +203,13 @@ If _number of users_ is omitted the script will run until manually interrupted.
 If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
 
 
+### Importing preferences metadata
+
+`node cic_meta/import_meta_preferences.js <datadir> <number_of_users>`
+
+If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported.
+
+
 ##### Importing pins and ussd data (optional)
 
 Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:

apps/data-seeding/cic_eth/import_balance.py

@@ -18,19 +18,17 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
-        block_by_number,
-        Block,
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import jsonrpc_template
+from chainlib.jsonrpc import JSONRPCRequest
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -38,6 +36,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
+from cic_base.eth.syncer import chain_interface
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -70,13 +69,14 @@ elif args.vv == True:
 config_dir = os.path.join(args.c)
 os.makedirs(config_dir, 0o777, True)
 config = confini.Config(config_dir, args.env_prefix)
-config.process()
 # override args
+config.process()
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y'),
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -185,27 +185,6 @@ class Handler:
         #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
 
 
-#class BlockGetter:
-#
-#    def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
-#        self.conn = conn
-#        self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
-#
-#
-#    def get(self, n):
-#        o = block_by_number(n)
-#        r = self.conn.do(o)
-#        b = None
-#        try:
-#            b = Block(r)
-#        except TypeError as e:
-#            if r == None:
-#                logg.debug('block not found {}'.format(n))
-#            else:
-#                logg.error('block retrieve error {}'.format(e))
-#        return b
-
-
 def progress_callback(block_number, tx_index):
     sys.stdout.write(str(block_number).ljust(200) + "\n")
 
@@ -226,11 +205,13 @@ def main():
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
-    
-    o = jsonrpc_template()
+   
+    j = JSONRPCRequest()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -244,10 +225,11 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     try:
         sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
@@ -305,7 +287,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/config/cic.ini

@@ -7,4 +7,5 @@ approval_escrow_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-user_ussd_svc_service_port=
+user_ussd_svc_service_port =
+

apps/data-seeding/config/eth.ini

@@ -1,8 +1,2 @@
 [eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
-gas_provider_address = 
-#chain_id = 
-abi_dir = /usr/local/share/cic/solidity/abi
-account_accounts_index_writer = 

apps/data-seeding/create_import_users.py

@@ -204,9 +204,9 @@ def gen():
          ]))
     if random.randint(0, 1):
         # fake.local_latitude()
-        p.location['latitude'] = (random.random() + 180) - 90
+        p.location['latitude'] = (random.random() * 180) - 90
         # fake.local_latitude()
-        p.location['longitude'] = (random.random() + 360) - 180
+        p.location['longitude'] = (random.random() * 360) - 179
 
     return (old_blockchain_checksum_address, phone, p)
 

apps/data-seeding/docker/Dockerfile

@@ -1,19 +1,21 @@
 # syntax = docker/dockerfile:1.2
-FROM python:3.8.6-slim-buster as compile-image
-
-WORKDIR /root
-
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
+#FROM python:3.8.6-slim-buster as compile-image
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
 
 RUN mkdir -vp /usr/local/etc/cic
 
+COPY data-seeding/package.json \
+     data-seeding/package-lock.json \
+     .
+
+RUN npm install
+
 COPY data-seeding/requirements.txt . 
 
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
+
 COPY data-seeding/ . 
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
-
-
 ENTRYPOINT [ ]

apps/data-seeding/eth/import_balance.py

@@ -18,25 +18,24 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
-        block_by_number,
-        Block,
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import jsonrpc_template
+from chainlib.jsonrpc import JSONRPCRequest
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
+from cic_base.eth.syncer import chain_interface
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -75,7 +74,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -184,27 +183,6 @@ class Handler:
         #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
 
 
-#class BlockGetter:
-#
-#    def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
-#        self.conn = conn
-#        self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
-#
-#
-#    def get(self, n):
-#        o = block_by_number(n)
-#        r = self.conn.do(o)
-#        b = None
-#        try:
-#            b = Block(r)
-#        except TypeError as e:
-#            if r == None:
-#                logg.debug('block not found {}'.format(n))
-#            else:
-#                logg.error('block retrieve error {}'.format(e))
-#        return b
-
-
 def progress_callback(block_number, tx_index):
     sys.stdout.write(str(block_number).ljust(200) + "\n")
 
@@ -225,11 +203,13 @@ def main():
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
-    
-    o = jsonrpc_template()
+   
+    j = JSONRPCRequest()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -243,10 +223,11 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     try:
         sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
@@ -300,7 +281,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/eth/import_users.py

@@ -59,7 +59,7 @@ config.process()
 args_override = {
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y')
         }
 config.dict_override(args_override, 'cli')
 config.add(args.user_dir, '_USERDIR', True)

apps/data-seeding/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b15
-sarafu-faucet==0.0.3a3
-cic-eth==0.11.0b16
-cic-types==0.1.0a11
-crypto-dev-signer==0.4.14b3
+cic_base[full_graph]==0.1.3a3+build.984b5cff
+sarafu-faucet==0.0.4a1
+cic-eth==0.11.1a1
+cic-types==0.1.0a13
+crypto-dev-signer==0.4.14b6

apps/data-seeding/verify.py

@@ -25,7 +25,7 @@ from chainlib.eth.gas import (
 )
 from chainlib.eth.tx import TxFactory
 from chainlib.hash import keccak256_string_to_hex
-from chainlib.jsonrpc import jsonrpc_template
+from chainlib.jsonrpc import JSONRPCRequest
 from cic_types.models.person import (
     Person,
     generate_metadata_pointer,
@@ -254,6 +254,10 @@ class Verifier:
             if len(k) > 7 and k[:7] == 'verify_':
                 logg.debug('verifier has verify method {}'.format(k))
                 verifymethods.append(k[7:])
+        o = self.faucet_tx_factory.token_amount(self.faucet_address, sender_address=ZERO_ADDRESS)
+        r = self.conn.do(o)
+        self.faucet_amount = self.faucet_tx_factory.parse_token_amount(r)
+        logg.info('faucet amount set to {} at verify initialization time'.format(self.faucet_amount))
 
         self.state = VerifierState(verifymethods, active_tests=active_tests)
 
@@ -264,9 +268,11 @@ class Verifier:
         data += eth_abi.encode_single('address', address).hex()
         tx = self.tx_factory.set_code(tx, data)
         tx = self.tx_factory.normalize(tx)
-        o = jsonrpc_template()
+        j = JSONRPCRequest()
+        o = j.template()
         o['method'] = 'eth_call'
         o['params'].append(tx)
+        o = j.finalize(o)
         r = self.conn.do(o)
         logg.debug('index check for {}: {}'.format(address, r))
         n = eth_abi.decode_single('uint256', bytes.fromhex(strip_0x(r)))
@@ -282,6 +288,7 @@ class Verifier:
         except ValueError:
             actual_balance = int(r)
         balance = int(balance / 1000000) * 1000000
+        balance += self.faucet_amount
         logg.debug('balance for {}: {}'.format(address, balance))
         if balance != actual_balance:
             raise VerifierError((actual_balance, balance), 'balance')
@@ -429,10 +436,12 @@ def main():
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
     
-    o = jsonrpc_template()
+    j = JSONRPCRequest()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -441,10 +450,11 @@ def main():
     data += eth_abi.encode_single('bytes32', b'AccountRegistry').hex()
     txf.set_code(tx, data)
     
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     account_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found account index address {}'.format(account_index_address))
@@ -453,10 +463,11 @@ def main():
     data += eth_abi.encode_single('bytes32', b'Faucet').hex()
     txf.set_code(tx, data)
     
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     faucet_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found faucet {}'.format(faucet_address))
@@ -471,10 +482,11 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token address {}'.format(sarafu_token_address))

apps/util/requirements/base_requirement.txt

@@ -0,0 +1 @@
+cic-base==0.1.3a3+build.984b5cff

apps/util/requirements/requirements.txt

@@ -0,0 +1 @@
+requirements-magic~=0.0.2

apps/util/requirements/update_base.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+which pyreq-merge &> /dev/null
+if [ $? -gt 0 ]; then
+	>&2 echo pyreq-merge missing, please install requirements
+	exit 1
+fi
+
+t=$(mktemp)
+>&2 echo using tmp $t
+
+repos=(../../cic-cache ../../cic-eth ../../cic-ussd ../../data-seeding ../../cic-notify ../../contract-migration)
+
+for r in ${repos[@]}; do
+	f="$r/requirements.txt"
+	>&2 echo updating $f
+	pyreq-update $f base_requirement.txt -v > $t 
+	cp $t $f
+
+	f="$r/test_requirements.txt"
+	if [ -f $f ]; then
+		>&2 echo updating $f
+		pyreq-update $f base_requirement.txt -v > $t
+		cp $t $f
+	fi
+done

docker-compose.yml

@@ -72,11 +72,11 @@ services:
 
   contract-migration:
     build:
+      dockerfile: contract-migration/docker/Dockerfile
       args:
         pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
         pip_extra_args: $PIP_EXTRA_ARGS
       context: apps/
-      dockerfile: contract-migration/docker/Dockerfile
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
       # ETH_PROVIDER should be broken out into host/port but cic-eth expects this
@@ -101,9 +101,16 @@ services:
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
       RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
       DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
-      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
-      DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
+      #DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
       DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
+      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+      TOKEN_NAME: $TOKEN_NAME
+      TOKEN_DECIMALS: $TOKEN_DECIMALS
+      TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD
+      TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT
+      TOKEN_DEMURRAGE_LEVEL: ${TOKEN_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
+      TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
+        #CONFINI_DIR: ${CONFINI_DIR:-/tmp/cic/config}
     command: ["./run_job.sh"]
         #command: ["./reset.sh"]
     depends_on:
@@ -387,44 +394,6 @@ services:
     # command: "/root/start_retry.sh -q cic-eth -vv"
 
 
-#  cic-eth-server:
-#    build:
-#      context: apps/
-#      dockerfile: cic-eth/docker/Dockerfile
-#    environment:
-#      CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC
-#      CELERY_BROKER_URL: $CELERY_BROKER_URL
-#      CELERY_RESULT_URL: $CELERY_RESULT_URL
-#      SERVER_PORT: 8000
-#    depends_on:
-#      - eth
-#      - postgres
-#      - redis
-#    ports:
-#      - ${HTTP_PORT_CIC_ETH:-63314}:8000
-#    deploy:
-#      restart_policy:
-#        condition: on-failure
-#    volumes:
-#        - contract-config:/tmp/cic/config/:ro
-#    command:
-#      - /bin/bash
-#      - -c
-#      - |
-#        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-#        "/usr/local/bin/uwsgi" \
-#        --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \
-#        --http :80 \
-#        --pyargv -vv
-##     entrypoint:
-##       - "/usr/local/bin/uwsgi"
-##       - "--wsgi-file"
-##       - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py"
-##       - "--http"
-##       - ":80"
-#    # command: "--pyargv -vv"
-
-
 
   cic-notify-tasker:
     build: