Refactors to use fixtures for test configs.

cic-cache-tracker, cic-eth-tracker: Add optional and customizable history start for trackers

Closes #50

See merge request grassrootseconomics/cic-internal-integration!143

apps/cic-cache/cic_cache/runnable/daemons/filters/__init__.py

@@ -1 +1,2 @@
 from .erc20 import *
+from .faucet import *

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -2,7 +2,6 @@
 import logging
 
 # external imports
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.address import (
         to_checksum_address,
         )
@@ -13,6 +12,7 @@ from cic_eth_registry.error import (
         NotAContractError,
         ContractMismatchError,
         )
+from eth_erc20 import ERC20
 
 # local imports
 from .base import TagSyncFilter

apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py

@@ -0,0 +1,73 @@
+# standard imports
+import logging
+
+# external imports
+from erc20_faucet import Faucet
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.status import Status
+from hexathon import strip_0x
+
+# local imports
+import cic_cache.db as cic_cache_db
+from .base import TagSyncFilter
+
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
+
+
+class FaucetFilter(TagSyncFilter):
+
+    def __init__(self, chain_spec, sender_address=ZERO_ADDRESS):
+        super(FaucetFilter, self).__init__('give_to', domain='faucet')
+        self.chain_spec = chain_spec
+        self.sender_address = sender_address
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        try:
+            data = strip_0x(tx.payload)
+        except ValueError:
+            return False
+        logg.debug('data {}'.format(data))
+        if Faucet.method_for(data[:8]) == None:
+            return False
+
+        token_sender = tx.inputs[0]
+        token_recipient = data[64+8-40:]
+        logg.debug('token recipient {}'.format(token_recipient))
+ 
+        f = Faucet(self.chain_spec)
+        o = f.token(token_sender, sender_address=self.sender_address)
+        r = conn.do(o)
+        token = f.parse_token(r)
+
+        f = Faucet(self.chain_spec)
+        o = f.token_amount(token_sender, sender_address=self.sender_address)
+        r = conn.do(o)
+        token_value = f.parse_token_amount(r)
+
+        cic_cache_db.add_transaction(
+                db_session,
+                tx.hash,
+                block.number,
+                tx.index,
+                to_checksum_address(token_sender),
+                to_checksum_address(token_recipient),
+                token,
+                token,
+                token_value,
+                token_value,
+                tx.status == Status.SUCCESS,
+                block.timestamp,
+                )
+        db_session.flush()
+        cic_cache_db.tag_transaction(
+                db_session,
+                tx.hash,
+                self.tag_name,
+                domain=self.tag_domain,
+                )
+        db_session.commit()
+
+        return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -41,16 +41,26 @@ from cic_cache.db import (
         )
 from cic_cache.runnable.daemons.filters import (
         ERC20TransferFilter,
+        FaucetFilter,
         )
 
 script_dir = os.path.realpath(os.path.dirname(__file__))
 
+def add_block_args(argparser):
+    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+    return argparser
+
+
 logg = cic_base.log.create()
 argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-#argparser = cic_base.argparse.add(argparser, add_traffic_args, 'traffic')
+argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
 args = cic_base.argparse.parse(argparser, logg)
 config = cic_base.config.create(args.c, args, args.env_prefix)
 
+config.add(args.history_start, 'SYNCER_HISTORY_START', True)
+config.add(args.no_history, '_NO_HISTORY', True)
+
 cic_base.config.log(config)
 
 dsn = dsn_from_config(config)
@@ -59,7 +69,6 @@ SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
@@ -71,6 +80,7 @@ def register_filter_tags(filters, session):
             session.commit()
             logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
         except sqlalchemy.exc.IntegrityError:
+            session.rollback()
             logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
 
 
@@ -82,7 +92,7 @@ def main():
     r = rpc.do(o)
     block_offset = int(strip_0x(r), 16) + 1
 
-    logg.debug('starting at block {}'.format(block_offset))
+    logg.debug('current block height {}'.format(block_offset))
 
     syncers = []
 
@@ -91,8 +101,13 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        logg.info('found no backends to resume')
+        initial_block_start = config.get('SYNCER_HISTORY_START')
-        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
+        initial_block_offset = block_offset
+        if config.get('_NO_HISTORY'):
+            initial_block_start = block_offset
+            initial_block_offset += 1
+        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
+        logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
@@ -112,9 +127,11 @@ def main():
         logg.info('using trusted address {}'.format(address))
 
     erc20_transfer_filter = ERC20TransferFilter(chain_spec)
+    faucet_filter = FaucetFilter(chain_spec)
 
     filters = [
         erc20_transfer_filter,
+        faucet_filter,
             ]
 
     session = SessionBase.create_session()

apps/cic-cache/config/docker/eth.ini

@@ -1,2 +1,2 @@
 [eth]
-provider = ws://localhost:63546
+provider = http://localhost:63545

apps/cic-cache/config/docker/syncer.ini

@@ -1,2 +1,3 @@
 [syncer]
 loop_interval = 1
+history_start = 0

apps/cic-cache/config/syncer.ini

@@ -1,2 +1,3 @@
 [syncer]
 loop_interval = 5
+history_start = 0

apps/cic-cache/docker/Dockerfile

@@ -17,7 +17,7 @@ RUN apt-get update && \
 
 # Copy shared requirements from top of mono-repo
 RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
 COPY cic-cache/requirements.txt ./
 COPY cic-cache/setup.cfg \

apps/cic-cache/requirements.txt

@@ -1,12 +1,12 @@
-cic-base~=0.1.2b6
+cic-base~=0.1.2b10
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a1
+cic-eth-registry~=0.5.5a4
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a2
+chainsyncer[sql]~=0.0.2a4

apps/cic-cache/test_requirements.txt

@@ -8,6 +8,4 @@ eth_tester==0.5.0b3
 py-evm==0.3.0a20
 web3==5.12.2
 cic-eth-registry~=0.5.5a3
-giftable-erc20-token~=0.0.8a10
+cic-base[full]==0.1.2b8
-eth-address-index~=0.1.1a10
-sarafu-faucet~=0.0.3a1

apps/cic-cache/tests/filters/test_erc20.py

@@ -22,7 +22,7 @@ from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
 logg = logging.getLogger()
 
 
-def test_cache(
+def test_erc20_filter(
         eth_rpc,
         foo_token,
         init_database,

apps/cic-cache/tests/filters/test_faucet.py

@@ -0,0 +1,71 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.chain import ChainSpec
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.block import (
+        block_by_hash,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        unpack,
+        transaction,
+        Tx,
+        )
+from hexathon import strip_0x
+from erc20_faucet.faucet import SingleShotFaucet
+from sqlalchemy import text
+
+# local imports
+from cic_cache.db import add_tag
+from cic_cache.runnable.daemons.filters.faucet import FaucetFilter
+
+logg = logging.getLogger()
+
+
+def test_filter_faucet(
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        faucet_noregistry,
+        init_database,
+        list_defaults,
+        contract_roles,
+        agent_roles,
+        tags,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = FaucetFilter(chain_spec, contract_roles['CONTRACT_DEPLOYER'])
+
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
+    c = SingleShotFaucet(chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex, o) = c.give_to(faucet_noregistry, agent_roles['ALICE'], agent_roles['ALICE'])
+    r = eth_rpc.do(o)
+
+    tx_src = unpack(bytes.fromhex(strip_0x(o['params'][0])), chain_spec)
+
+    o = receipt(r)
+    r = eth_rpc.do(o)
+    rcpt = Tx.src_normalize(r)
+
+    assert r['status'] == 1
+
+    o = block_by_hash(r['block_hash'])
+    r = eth_rpc.do(o)
+    block_object = Block(r)
+
+    tx = Tx(tx_src, block_object)
+    tx.apply_receipt(rcpt)
+
+    r = fltr.filter(eth_rpc, block_object, tx, init_database)
+    assert r
+
+    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
+    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
+    assert r[0] == tx.hash

apps/cic-eth/cic_eth/eth/account.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from erc20_faucet import Faucet
 from hexathon import (
         strip_0x,
         )
@@ -20,8 +20,9 @@ from chainlib.eth.tx import (
         )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
-from eth_accounts_index import AccountRegistry
+from eth_accounts_index.registry import AccountRegistry
-from sarafu_faucet import MinterFaucet as Faucet
+from eth_accounts_index import AccountsIndex 
+from sarafu_faucet import MinterFaucet
 from chainqueue.db.models.tx import TxCache
 
 # local import
@@ -133,7 +134,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
     gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
-    account_registry = AccountRegistry(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
 
@@ -185,7 +186,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -338,7 +339,7 @@ def cache_account_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
     tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = AccountRegistry.parse_add_request(tx['data'])
+    tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_cache = TxCache(

apps/cic-eth/cic_eth/eth/erc20.py

@@ -6,7 +6,6 @@ import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -16,6 +15,7 @@ from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
 from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase

apps/cic-eth/cic_eth/ext/address.py

@@ -7,7 +7,7 @@ from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth_registry import CICRegistry
-from eth_address_declarator import AddressDeclarator
+from eth_address_declarator import Declarator
 
 # local imports
 from cic_eth.task import BaseTask
@@ -23,12 +23,12 @@ def translate_address(address, trusted_addresses, chain_spec, sender_address=ZER
     registry = CICRegistry(chain_spec, rpc)
     
     declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    c = AddressDeclarator(chain_spec)
+    c = Declarator(chain_spec)
 
     for trusted_address in trusted_addresses:
         o = c.declaration(declarator_address, trusted_address, address, sender_address=sender_address)
         r = rpc.do(o)
-        declaration_hex = AddressDeclarator.parse_declaration(r)
+        declaration_hex = Declarator.parse_declaration(r)
         declaration_hex = declaration_hex[0].rstrip('0')
         declaration_bytes = bytes.fromhex(declaration_hex)
         declaration = None

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,13 +14,13 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
-from chainlib.eth.erc20 import ERC20
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.query import get_tx_cache
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.queue.time import tx_times

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -3,19 +3,20 @@ import logging
 
 # external imports
 import celery
-from cic_eth_registry.error import UnknownContractError
+from cic_eth_registry.error import (
+    UnknownContractError,
+    NotAContractError,
+    )
 from chainlib.status import Status as TxStatus
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.erc20 import ERC20
 from hexathon import (
         strip_0x,
         add_0x,
         )
-# TODO: use sarafu_Faucet for both when inheritance has been implemented
+from eth_erc20 import ERC20
-from erc20_single_shot_faucet import SingleShotFaucet
+from erc20_faucet import Faucet
-from sarafu_faucet import MinterFaucet as Faucet
 
 # local imports
 from .base import SyncFilter
@@ -71,14 +72,13 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        c = SingleShotFaucet(self.chain_spec)
+        o = Faucet.token(faucet_contract, sender_address=self.caller_address)
-        o = c.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 
-        o = c.amount(faucet_contract, sender_address=self.caller_address)
+        o = c.token_amount(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
-        transfer_data['value'] = c.parse_amount(r)
+        transfer_data['value'] = c.parse_token_amount(r)
 
         return ('tokengift', transfer_data)
 
@@ -127,8 +127,7 @@ class CallbackFilter(SyncFilter):
                     (transfer_type, transfer_data) = parser(tx, conn)
                     if transfer_type == None:
                         continue
-                else:
+                break
-                    pass
             except RequestMismatchException:
                 continue
 
@@ -171,7 +170,9 @@ class CallbackFilter(SyncFilter):
                 t = self.call_back(transfer_type, result)
                 logg.info('callback success task id {} tx {} queue {}'.format(t, tx.hash, t.queue))
             except UnknownContractError:
-                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tx.queue, tx.method, transfer_data['to'], tx.hash))
+                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(self.queue, self.method, transfer_data['to'], tx.hash))
+            except NotAContractError:
+                logg.debug('callback filter {}:{} skipping "transfer" on non-contract address {} tx {}'.format(self.queue, self.method, transfer_data['to'], tx.hash))
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -14,7 +14,7 @@ from .base import SyncFilter
 
 logg = logging.getLogger().getChild(__name__)
 
-account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd'
+account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 
 
 class RegistrationFilter(SyncFilter):

apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py

@@ -30,7 +30,7 @@ class TxFilter(SyncFilter):
         if otx == None:
             logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
             return None
-        logg.info('tx filter match on {}'.format(otx.tx_hash))
+        logg.debug('otx filter match on {}'.format(otx.tx_hash))
         db_session.flush()
         SessionBase.release_session(db_session)
         s_final_state = celery.signature(

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -51,15 +51,23 @@ from cic_eth.registry import (
 
 script_dir = os.path.realpath(os.path.dirname(__file__))
 
+def add_block_args(argparser):
+    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+    return argparser
+
+
 logg = cic_base.log.create()
 argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-#argparser = cic_base.argparse.add(argparser, add_traffic_args, 'traffic')
+argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
 args = cic_base.argparse.parse(argparser, logg)
+
 config = cic_base.config.create(args.c, args, args.env_prefix)
 
 config.add(args.y, '_KEYSTORE_FILE', True)
-
 config.add(args.q, '_CELERY_QUEUE', True)
+config.add(args.history_start, 'SYNCER_HISTORY_START', True)
+config.add(args.no_history, '_NO_HISTORY', True)
 
 cic_base.config.log(config)
 
@@ -69,9 +77,9 @@ SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -89,7 +97,7 @@ def main():
         stat = init_chain_stat(rpc, block_start=block_current)
         loop_interval = stat.block_average()
 
-    logg.debug('starting at block {}'.format(block_offset))
+    logg.debug('current block height {}'.format(block_offset))
 
     syncers = []
 
@@ -98,8 +106,13 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        logg.info('found no backends to resume')
+        initial_block_start = config.get('SYNCER_HISTORY_START')
-        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
+        initial_block_offset = block_offset
+        if config.get('_NO_HISTORY'):
+            initial_block_start = block_offset
+            initial_block_offset += 1
+        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
+        logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
@@ -155,7 +168,6 @@ def main():
         for cf in callback_filters:
             syncer.add_filter(cf)
 
-        #r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
         r = syncer.loop(int(loop_interval), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 

apps/cic-eth/cic_eth/runnable/info.py

@@ -12,7 +12,10 @@ import confini
 import celery
 
 # local imports
-from cic_eth.api import Api
+from cic_eth.api import (
+        Api,
+        AdminApi,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -53,8 +56,13 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
 queue = args.q
 
 api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
+admin_api = AdminApi(None)
 
 def main():
+    t = admin_api.registry()
+    registry = t.get()
+    print('Registry address: {}'.format(registry))
+
     t = api.default_token()
     token_info = t.get()
     print('Default token symbol: {}'.format(token_info['symbol']))

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.11',
+        'beta.13',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/docker/syncer.ini

@@ -1,2 +1,3 @@
 [SYNCER]
 loop_interval = 
+history_start = 0

apps/cic-eth/config/syncer.ini

@@ -1,2 +1,3 @@
 [SYNCER]
 loop_interval = 
+history_start = 0

apps/cic-eth/docker/Dockerfile

@@ -19,7 +19,7 @@ RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
 # Copy shared requirements from top of mono-repo
-RUN echo "copying root req file ${root_requirement_file}"
+RUN echo "copying root req file:  ${root_requirement_file}"
 #COPY $root_requirement_file .
 #RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
 RUN /usr/local/bin/python -m pip install --upgrade pip
@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
+RUN  pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./

apps/cic-eth/requirements.txt

@@ -1,25 +1,24 @@
-cic-base==0.1.2b5
+cic-base~=0.1.2b11
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.4a16
+cic-eth-registry~=0.5.5a4
-#cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a9
+eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a5
+erc20-transfer-authorization~=0.3.1a6
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a9
+eth-address-index~=0.1.1a11
-chainlib~=0.0.2a20
+chainlib~=0.0.3a2
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a2
+chainsyncer[sql]~=0.0.2a4
 chainqueue~=0.0.2a2
-pysha3==1.0.2
+sarafu-faucet==0.0.3a3
+erc20-faucet==0.2.1a4
 coincurve==15.0.0
-sarafu-faucet==0.0.2a28
+potaahto~=0.0.1a2
-potaahto~=0.0.1a1

apps/cic-ussd/.config/app.ini

@@ -4,7 +4,7 @@ LOCALE_FALLBACK=en
 LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
-SERVICE_CODE=*483*46#
+SERVICE_CODE=*483*46#,*483*061#,*384*96#
 
 [phone_number]
 REGION=KE

apps/cic-ussd/.config/test/integration.ini

@@ -0,0 +1,4 @@
+[test]
+gift_value = 50.00
+server_url = http://localhost:63315/
+token_symbol = GFT

apps/cic-ussd/cic_ussd/conversions.py

@@ -24,7 +24,7 @@ def from_wei(value: int) -> float:
     """This function converts values in Wei to a token in the cic network.
     :param value: Value in Wei
     :type value: int
-    :return: SRF equivalent of value in Wei
+    :return: platform's default token equivalent of value in Wei
     :rtype: float
     """
     value = float(value) / 1e+6
@@ -33,9 +33,9 @@ def from_wei(value: int) -> float:
 
 def to_wei(value: int) -> int:
     """This functions converts values from a token in the cic network to Wei.
-    :param value: Value in SRF
+    :param value: Value in platform's default token
     :type value: int
-    :return: Wei equivalent of value in SRF
+    :return: Wei equivalent of value in platform's default token
     :rtype: int
     """
-    return int(value * 1e+6)
+    return int(value * 1e+6)

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -13,7 +13,7 @@ import argparse
 import logging
 import urllib
 from xdg.BaseDirectory import xdg_config_home
-from urllib import request
+from urllib import parse, request
 
 # third-party imports
 from confini import Config
@@ -92,9 +92,9 @@ def main():
             data['text'] = user_input
 
         req = urllib.request.Request(url)
-        data_str = json.dumps(data)
+        urlencoded_data = parse.urlencode(data)
-        data_bytes = data_str.encode('utf-8')
+        data_bytes = urlencoded_data.encode('utf-8')
-        req.add_header('Content-Type', 'application/json')
+        req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         req.data = data_bytes
         response = urllib.request.urlopen(req)
         response_data = response.read().decode('utf-8')

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -4,6 +4,7 @@
 # standard imports
 import json
 import logging
+from urllib.parse import parse_qs
 
 # third-party imports
 import celery
@@ -33,8 +34,7 @@ from cic_ussd.requests import (get_request_endpoint,
 from cic_ussd.runnable.server_base import exportable_parser, logg
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.state_machine import UssdStateMachine
-from cic_ussd.validator import check_ip, check_request_content_length, check_service_code, validate_phone_number, \
+from cic_ussd.validator import check_ip, check_request_content_length, validate_phone_number, validate_presence
-    validate_presence
 
 args = exportable_parser.parse_args()
 
@@ -124,6 +124,9 @@ else:
     raise InitializationError(f'Default token data for: {chain_str} not found.')
 
 
+valid_service_codes = config.get('APP_SERVICE_CODE').split(",")
+
+
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
     :param env: Object containing server and request information
@@ -139,13 +142,27 @@ def application(env, start_response):
 
     if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/':
 
-        # get post data
+        if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded':
-        post_data = json.load(env.get('wsgi.input'))
+            start_response('405 Play by the rules', errors_headers)
+            return []
 
-        service_code = post_data.get('serviceCode')
+        post_data = env.get('wsgi.input').read()
-        phone_number = post_data.get('phoneNumber')
+        post_data = post_data.decode('utf-8')
-        external_session_id = post_data.get('sessionId')
+
-        user_input = post_data.get('text')
+        try:
+            post_data = parse_qs(post_data)
+        except TypeError:
+            start_response('400 Size matters', errors_headers)
+            return []
+
+        service_code = post_data.get('serviceCode')[0]
+        phone_number = post_data.get('phoneNumber')[0]
+        external_session_id = post_data.get('sessionId')[0]
+
+        try:
+            user_input = post_data.get('text')[0]
+        except TypeError:
+            user_input = ""
 
         # add validation for phone number
         if phone_number:
@@ -162,14 +179,14 @@ def application(env, start_response):
             return []
 
         # validate service code
-        if not check_service_code(code=service_code, config=config):
+        if service_code not in valid_service_codes:
             response = define_multilingual_responses(
                 key='ussd.kenya.invalid_service_code',
                 locales=['en', 'sw'],
                 prefix='END',
-                valid_service_code=config.get('APP_SERVICE_CODE'))
+                valid_service_code=valid_service_codes[0])
-            response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
+            response_bytes, headers = define_response_with_content(headers=headers, response=response)
-            start_response('400 Invalid service code', headers)
+            start_response('200 OK', headers)
             return [response_bytes]
 
         # validate phone number
@@ -192,3 +209,8 @@ def application(env, start_response):
         start_response('200 OK,', headers)
         SessionBase.session.close()
         return [response_bytes]
+
+    else:
+        start_response('405 Play by the rules', errors_headers)
+        return []
+

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -12,6 +12,7 @@ from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
 from cic_ussd.phone_number import get_user_by_phone_number
+from cic_ussd.processor import retrieve_token_symbol
 from cic_ussd.redis import create_cached_data_key, get_cached_data
 from cic_ussd.transactions import OutgoingTransactionProcessor
 
@@ -124,14 +125,18 @@ def process_transaction_request(state_machine_data: Tuple[str, dict, Account]):
     """
     user_input, ussd_session, user = state_machine_data
 
+    # retrieve token symbol
+    chain_str = Chain.spec.__str__()
+
     # get user from phone number
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
     to_address = recipient.blockchain_address
     from_address = user.blockchain_address
     amount = int(ussd_session.get('session_data').get('transaction_amount'))
-    chain_str = Chain.spec.__str__()
+    token_symbol = retrieve_token_symbol(chain_str=chain_str)
+
     outgoing_tx_processor = OutgoingTransactionProcessor(chain_str=chain_str,
                                                          from_address=from_address,
                                                          to_address=to_address)
-    outgoing_tx_processor.process_outgoing_transfer_transaction(amount=amount)
+    outgoing_tx_processor.process_outgoing_transfer_transaction(amount=amount, token_symbol=token_symbol)

apps/cic-ussd/cic_ussd/transactions.py

@@ -120,7 +120,7 @@ class OutgoingTransactionProcessor:
         self.from_address = from_address
         self.to_address = to_address
 
-    def process_outgoing_transfer_transaction(self, amount: int, token_symbol='SRF'):
+    def process_outgoing_transfer_transaction(self, amount: int, token_symbol: str):
         """This function initiates standard transfers between one account to another
         :param amount: The amount of tokens to be sent
         :type amount: int

apps/cic-ussd/cic_ussd/validator.py

@@ -45,19 +45,6 @@ def check_request_content_length(config: Config, env: dict):
         config.get('APP_MAX_BODY_LENGTH'))
 
 
-def check_service_code(code: str, config: Config):
-    """Checks whether provided code matches expected service code
-    :param config: A dictionary object containing configuration values
-    :type config: Config
-    :param code: Service code passed over request
-    :type code: str
-
-    :return: Service code validity
-    :rtype: boolean
-    """
-    return code == config.get('APP_SERVICE_CODE')
-
-
 def check_known_user(phone: str):
     """
     This method attempts to ascertain whether the user already exists and is known to the system.

apps/cic-ussd/test_requirements.txt

@@ -1,7 +1,10 @@
-pytest==6.0.1
+Faker==8.1.2
+faker-e164==0.1.0
+pytest==6.2.4
 pytest-alembic==0.2.5
 pytest-celery==0.0.0a1
 pytest-cov==2.10.1
 pytest-mock==3.3.1
 pytest-redis==2.0.0
-requests-mock==1.8.0
+requests-mock==1.8.0
+tavern==1.14.2

apps/cic-ussd/tests/conftest.py

@@ -3,6 +3,7 @@ from cic_types.pytest import *
 
 
 # local imports
+from tests.fixtures.accounts import *
 from tests.fixtures.config import *
 from tests.fixtures.db import *
 from tests.fixtures.celery import *

apps/cic-ussd/tests/fixtures/accounts.py

@@ -0,0 +1,54 @@
+# standard imports
+
+# external imports
+import pytest
+
+# local imports
+
+# test imports
+from tests.helpers.accounts import phone_number, session_id
+
+
+@pytest.fixture(scope='function')
+def generate_phone_number() -> str:
+    return phone_number()
+
+
+@pytest.fixture(scope='function')
+def generate_session_id() -> str:
+    return session_id()
+
+
+@pytest.fixture(scope='session')
+def first_account_phone_number() -> str:
+    return phone_number()
+
+
+@pytest.fixture(scope='session')
+def second_account_phone_number() -> str:
+    return phone_number()
+
+
+@pytest.fixture(scope='session')
+def first_metadata_entry_session_id() -> str:
+    return session_id()
+
+
+@pytest.fixture(scope='session')
+def second_metadata_entry_session_id() -> str:
+    return session_id()
+
+
+@pytest.fixture(scope='session')
+def gift_value(load_config):
+    return load_config.get('TEST_GIFT_VALUE')
+
+
+@pytest.fixture(scope='session')
+def server_url(load_config):
+    return load_config.get('TEST_SERVER_URL')
+
+
+@pytest.fixture(scope='session')
+def token_symbol(load_config):
+    return load_config.get('TEST_TOKEN_SYMBOL')

apps/cic-ussd/tests/helpers/accounts.py

@@ -0,0 +1,21 @@
+# standard imports
+import uuid
+
+# external imports
+from faker import Faker
+from faker_e164.providers import E164Provider
+
+# local imports
+
+# test imports
+
+fake = Faker()
+fake.add_provider(E164Provider)
+
+
+def phone_number() -> str:
+    return fake.e164('KE')
+
+
+def session_id() -> str:
+    return uuid.uuid4().hex

apps/cic-ussd/tests/integration/ext/validator.py

@@ -0,0 +1,11 @@
+import logging
+
+
+logg = logging.getLogger()
+logg.setLevel(logging.DEBUG)
+
+
+def validate_response(response, expected_response):
+    """Makes sure that the response received matches the expected response"""
+    logg.debug(f'RESPONSE: {response.content.decode("utf-8")}')
+    assert response.content.decode('utf-8') == expected_response

apps/cic-ussd/tests/integration/run.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+PYTHONPATH=. tavern-ci test_accounts.tavern.yaml --debug -vv --log-level debug -s --log-cli-level debug

apps/cic-ussd/tests/integration/test_accounts.tavern.yaml

@@ -0,0 +1,454 @@
+test_name: Create an account through the cic_user_ussd_server entrypoint.
+marks:
+  - usefixtures:
+    - gift_value
+    - server_url
+    - token_symbol
+    - generate_session_id
+    - first_account_phone_number
+    - second_account_phone_number
+    - first_metadata_entry_session_id
+    - second_metadata_entry_session_id
+stages:
+  - name: Initiate account creation process [first account].
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{generate_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: ""
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '175'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "END Your account is being created. You will receive an SMS when your account is ready.\nAkaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.\n"
+
+  - name: Initiate account creation process [second account].
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{generate_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: ""
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '175'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "END Your account is being created. You will receive an SMS when your account is ready.\nAkaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.\n"
+    delay_after: 5
+
+  - name: Initaite account metadata entry [first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: ""
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '53'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Welcome to Sarafu\n1. English\n2. Kiswahili\n3. Help"
+
+  - name: Initaite account metadata entry [second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: ""
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '53'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Welcome to Sarafu\n1. English\n2. Kiswahili\n3. Help"
+
+  - name: Select preferred language [English]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '54'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Please enter a PIN to manage your account.\n0. Back"
+
+  - name: Select preferred language [Kiswahili]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '59'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Tafadhali weka PIN ili kudhibiti akaunti yako.\n0. Nyuma"
+
+  - name: Enter pin number [0000 - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*0000"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '32'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter your PIN again\n0. Back"
+
+  - name: Enter pin number [1212 - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*1212"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '31'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka PIN yako tena\n0. Nyuma"
+
+  - name: Pin number confirmation [0000 - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*0000*0000"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '28'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter first name\n0. Back"
+
+  - name: Pin number confirmation [1212 - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*1212*1212"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '37'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka jina lako la kwanza\n0. Nyuma"
+
+  - name: Enter first name [Kimani - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*0000*0000*Kimani"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '27'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter last name\n0. Back"
+
+  - name: Enter first name [Chebet - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*1212*1212*Chebet"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '37'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka jina lako la mwisho\n0. Nyuma"
+
+  - name: Enter last name [Omollo - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*0000*0000*Kimani*Omollo"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '42'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter gender\n1. Male\n2. Female\n0. Back"
+
+  - name: Enter last name [Musau - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*1212*1212*Chebet*Musau"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '53'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n0. Nyuma"
+
+  - name: Select gender [Male - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*0000*0000*Kimani*Omollo*1"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '26'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Enter location\n0. Back"
+
+  - name: Select gender [Female - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*1212*1212*Chebet*Musau*2"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '27'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Weka eneo lako\n0. Nyuma"
+
+  - name: Enter location [Kangemi - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*0000*0000*Kimani*Omollo*1*Kangemi"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '55'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Please enter a product or service you offer\n0. Back"
+
+  - name: Enter location [Gachororo - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*1212*1212*Chebet*Musau*2*Gachororo"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '52'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Tafadhali weka bidhaa ama huduma unauza\n0. Nyuma"
+
+  - name: Enter product [Potatoes - first account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{first_metadata_entry_session_id}"
+        phoneNumber: "{first_account_phone_number}"
+        text: "1*0000*0000*Kimani*Omollo*1*Kangemi*Potatoes"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '51'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
+    delay_before: 5
+
+  - name: Enter product [Mkalimani - second account]
+    request:
+      url: "{server_url}"
+      data:
+        serviceCode: "*483*46#"
+        sessionId: "{second_metadata_entry_session_id}"
+        phoneNumber: "{second_account_phone_number}"
+        text: "2*1212*1212*Chebet*Musau*2*Gachororo*Mkalimani"
+      headers:
+        content-type: "application/x-www-form-urlencoded"
+      method: POST
+    response:
+      status_code:
+        - 200
+      headers:
+        Content-Length: '56'
+        Content-Type: "text/plain"
+      verify_response_with:
+        function: ext.validator:validate_response
+        extra_kwargs:
+          expected_response: "CON Salio {gift_value} {token_symbol}\n1. Tuma\n2. Akaunti yangu\n3. Usaidizi"
+    delay_before: 5

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -158,6 +158,8 @@ en:
       Your Sarafu-Network balances is: %{token_balance}
       00. Back
       99. Exit
+    invalid_service_code: |-
+      Please dial %{valid_service_code} to access Sarafu Network
     help: |-
       CON For assistance call %{support_phone}
       00. Back

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -158,6 +158,8 @@ sw:
       Akaunti yako ya Sarafu-Network ina salio ifuatayo: %{token_balance}
       00. Nyuma
       99. Ondoka
+    invalid_service_code: |-
+      Bonyeza %{valid_service_code} kutumia mtandao wa Sarafu
     help: |-
       CON Kwa usaidizi piga simu %{support_phone}
       0. Nyuma

apps/contract-migration/docker/Dockerfile

@@ -47,54 +47,61 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh |
 ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
 ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
 
-RUN useradd --create-home grassroots
+#RUN useradd --create-home grassroots
-WORKDIR /home/grassroots
+# WORKDIR /home/grassroots
-USER grassroots
+# USER grassroots
 
+ARG pip_extra_args=""
+ARG pip_index_url=https://pypi.org/simple
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a79
+ARG cic_base_version=0.1.2b11
-ARG cic_eth_version=0.11.0b8+build.c2286e5c
+ARG cic_eth_version=0.11.0b14
-ARG sarafu_faucet_version=0.0.2a28
+ARG sarafu_token_version=0.0.1a8
-ARG sarafu_token_version=0.0.1a6
+ARG sarafu_faucet_version=0.0.3a3
-ARG cic_contracts_version=0.0.2a2
+RUN pip install --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url \
-RUN pip install --user --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
+	cic-base[full_graph]==$cic_base_version \
 	cic-eth==$cic_eth_version \
-	cic-contracts==$cic_contracts_version \
 	sarafu-faucet==$sarafu_faucet_version \
-	sarafu-token==$sarafu_token_version
+	sarafu-token==$sarafu_token_version \
+	cic-eth==$cic_eth_version 
 
+# -------------- begin runtime container ---------------- 
 FROM python:3.8.6-slim-buster as runtime-image
 
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
-RUN apt-get install -y --no-install-recommends jq
+RUN apt-get install -y jq bash iputils-ping socat
 
 COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
 COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
+COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
 
-RUN useradd --create-home grassroots
+ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
-WORKDIR /home/grassroots
+# RUN useradd -u 1001 --create-home grassroots
-# COPY python dependencies to user dir
+# RUN adduser grassroots sudo && \
-COPY --from=compile-image /home/grassroots/.local .local
+#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-ENV PATH=/home/grassroots/.local/bin:$PATH
+# WORKDIR /home/grassroots
 
 COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
 COPY contract-migration/envlist .
+COPY contract-migration/scripts scripts/ 
 
-# RUN chown grassroots:grassroots .local/
-
-RUN mkdir -p /tmp/cic/config
-RUN chown grassroots:grassroots /tmp/cic/config
 # A shared output dir for environment configs
+RUN mkdir -p /tmp/cic/config
+# RUN chown grassroots:grassroots /tmp/cic/config
 RUN chmod a+rwx /tmp/cic/config 
 
 COPY contract-migration/*.sh ./
-RUN chown grassroots:grassroots -R .
+# RUN chown grassroots:grassroots -R .
 RUN chmod gu+x *.sh
 
+# we copied these from the root build container.
+# this is dumb though...I guess the compile image should have the same user
+# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
 
-USER grassroots
+# USER grassroots
 
 ENTRYPOINT [ ]

apps/contract-migration/scripts/cic_eth/import_balance.py

@@ -27,16 +27,17 @@ from chainlib.eth.block import (
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
 from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
+from chainlib.eth.constant import ZERO_ADDRESS
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
+from eth_erc20 import ERC20
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -51,7 +52,7 @@ argparser.add_argument('-c', type=str, default=config_dir, help='config root to
 argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
-argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
+argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('--head', action='store_true', help='start at current block height (overrides --offset)')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@@ -252,6 +253,10 @@ def main():
     except ValueError as e:
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
+
+    if sarafu_token_address == ZERO_ADDRESS:
+        raise KeyError('token address for symbol {} is zero'.format(token_symbol))
+
     logg.info('found token address {}'.format(sarafu_token_address))
 
     syncer_backend = MemBackend(chain_str, 0)

apps/contract-migration/scripts/eth/import_balance.py

@@ -27,7 +27,6 @@ from chainlib.eth.block import (
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
@@ -37,6 +36,7 @@ from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
+from eth_erc20 import ERC20
 
 
 logging.basicConfig(level=logging.WARNING)

apps/contract-migration/scripts/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b2
+cic-base[full_graph]==0.1.2b9
-sarafu-faucet==0.0.2a28
+sarafu-faucet==0.0.3a3
-cic-eth==0.11.0b10
+cic-eth==0.11.0b13
-cic-types==0.1.0a10
+cic-types==0.1.0a11
 crypto-dev-signer==0.4.14b3

apps/contract-migration/scripts/verify.py

@@ -34,7 +34,6 @@ from chainlib.eth.block import (
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import (
         OverrideGasOracle,
         balance,
@@ -46,7 +45,8 @@ from cic_types.models.person import (
         Person,
         generate_metadata_pointer,
         )
-from erc20_single_shot_faucet import SingleShotFaucet
+from erc20_faucet import Faucet
+from eth_erc20 import ERC20
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -224,7 +224,7 @@ class Verifier:
         self.api = cic_eth_api
         self.data_dir = data_dir
         self.exit_on_error = exit_on_error
-        self.faucet_tx_factory = SingleShotFaucet(chain_spec, gas_oracle=gas_oracle)
+        self.faucet_tx_factory = Faucet(chain_spec, gas_oracle=gas_oracle)
 
         verifymethods = []
         for k in dir(self):

apps/contract-migration/seed_cic_eth.sh

@@ -99,7 +99,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 # Transfer tokens to gifter address
 >&2 echo "transfer sarafu tokens to token gifter address"
->&2 eth-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
+>&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
 
 #echo -n 0 > $init_level_file
 

docker-compose.yml

@@ -76,6 +76,9 @@ services:
 
   contract-migration:
     build:
+      args:
+        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
+        pip_extra_args: $PIP_EXTRA_ARGS
       context: apps/
       dockerfile: contract-migration/docker/Dockerfile
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest