Add missing package file encode

Upgrade outer tools

See merge request grassrootseconomics/cic-internal-integration!257

apps/cic-cache/cic_cache/version.py

@@ -5,7 +5,7 @@ version = (
         0,
         2,
         1,
-        'alpha.1',
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-cache/docker/Dockerfile

@@ -10,9 +10,10 @@ COPY requirements.txt .
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_ARGS=""
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt
 
 COPY . . 

apps/cic-cache/docker/Dockerfile_ci

@@ -10,8 +10,9 @@ COPY requirements.txt .
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_ARGS=""
 RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt
 
 COPY . . 

apps/cic-cache/requirements.txt

@@ -2,14 +2,14 @@ alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
 moolb~=0.1.1b2
-cic-eth-registry~=0.5.8a1
+cic-eth-registry~=0.6.1a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]>=0.0.6a1,<0.1.0
+chainsyncer[sql]>=0.0.6a3,<0.1.0
-erc20-faucet>=0.2.4a2, <0.3.0
+erc20-faucet>=0.3.2a1, <0.4.0
-chainlib-eth>=0.0.7a3,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
-chainlib>=0.0.7a3,<0.1.0
+chainlib>=0.0.9a3,<0.1.0
-eth-address-index>=0.1.4a1,<0.2.0
+eth-address-index>=0.2.3a1,<0.3.0

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-sarafu-faucet~=0.0.5a2
+sarafu-faucet~=0.0.7a1
-erc20-transfer-authorization>=0.3.4a1,<0.4.0
+erc20-transfer-authorization>=0.3.5a1,<0.4.0

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.5.6a2,<0.6.0
+cic-eth-registry>=0.6.1a2,<0.7.0
-hexathon~=0.0.1a7
+hexathon~=0.0.1a8
-chainqueue>=0.0.3a1,<0.1.0
+chainqueue>=0.0.4a6,<0.1.0
-eth-erc20>=0.0.10a3,<0.1.0
+eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -4,7 +4,6 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from hexathon import (
         add_0x,
@@ -20,18 +19,17 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         )
 from cic_eth.error import LockedError
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-def normalize_address(a):
-    if a == None:
-        return None
-    return add_0x(hex_uniform(strip_0x(a)))
-
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -43,7 +41,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -53,7 +51,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -65,7 +63,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -75,7 +73,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -85,7 +83,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -93,7 +91,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -103,7 +101,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -111,7 +109,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -121,7 +119,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -129,7 +127,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -139,7 +137,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -148,12 +146,13 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    address = normalize_address(address)
+    if address != None:
+        address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
     if address != None:
         r |= Lock.check(chain_str, lock_flags, address=address, session=session)
     if r > 0:

apps/cic-eth/cic_eth/admin/nonce.py

@@ -33,6 +33,7 @@ from cic_eth.admin.ctrl import (
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
 from cic_eth.task import BaseTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -73,7 +74,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
+    query_address = tx_normalize.wallet_address(address)
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==query_address)

apps/cic-eth/cic_eth/api/admin.py

@@ -32,7 +32,6 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
 from eth_erc20 import ERC20
 
 # local imports
@@ -40,6 +39,7 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
+from cic_eth.queue.query import get_tx_local
 
 app = celery.current_app
 
@@ -284,7 +284,7 @@ class AdminApi:
         tx_hash_hex = None
         session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
+            tx_dict = get_tx_local(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
         session.close()

apps/cic-eth/cic_eth/db/models/lock.py

@@ -4,12 +4,12 @@ import logging
 
 # third-party imports
 from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode  import ZERO_ADDRESS_NORMAL
 
 logg = logging.getLogger()
 
@@ -37,7 +37,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
+    def set(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None, tx_hash=None):
         """Sets flags associated with the given address and chain.
 
         If a flags entry does not exist it is created.
@@ -90,7 +90,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def reset(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Resets flags associated with the given address and chain.
 
         If the resulting flags entry value is 0, the entry will be deleted.
@@ -134,7 +134,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def check(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Checks whether all given flags are set for given address and chain. 
 
         Does not validate the address against any other tables or components.

apps/cic-eth/cic_eth/encode.py

@@ -0,0 +1,16 @@
+# external imports
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainqueue.encode import TxHexNormalizer
+from chainlib.eth.tx import unpack
+
+tx_normalize = TxHexNormalizer()
+
+ZERO_ADDRESS_NORMAL = tx_normalize.wallet_address(ZERO_ADDRESS)
+
+
+def unpack_normal(signed_tx_bytes, chain_spec):
+    tx = unpack(signed_tx_bytes, chain_spec)
+    tx['hash'] = tx_normalize.tx_hash(tx['hash'])
+    tx['from'] = tx_normalize.wallet_address(tx['from'])
+    tx['to'] = tx_normalize.wallet_address(tx['to'])
+    return tx

apps/cic-eth/cic_eth/eth/account.py

@@ -14,10 +14,7 @@ from chainlib.eth.sign import (
         sign_message,
         )
 from chainlib.eth.address import to_checksum_address
-from chainlib.eth.tx import (
+from chainlib.eth.tx import TxFormat
-        TxFormat,
-        unpack,
-        )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
@@ -49,6 +46,10 @@ from cic_eth.eth.nonce import (
 from cic_eth.queue.tx import (
         register_tx,
         )
+from cic_eth.encode import (
+        unpack_normal,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 logg = logging.getLogger()
 celery_app = celery.current_app 
@@ -295,17 +296,17 @@ def cache_gift_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = Faucet.parse_give_to_request(tx['data'])
 
     session = self.create_session()
 
     tx_dict = {
-            'hash': tx_hash_hex,
+            'hash': tx['hash'],
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
+            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
             'to_value': 0,
             }
@@ -334,17 +335,17 @@ def cache_account_data(
     :rtype: tuple
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_dict = {
-            'hash': tx_hash_hex,
+            'hash': tx['hash'],
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
+            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
             'to_value': 0,
             }

apps/cic-eth/cic_eth/eth/gas.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import celery
 from hexathon import strip_0x
-from chainlib.eth.constant import ZERO_ADDRESS
+#from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
@@ -21,7 +21,6 @@ from chainlib.eth.error import (
 from chainlib.eth.tx import (
         TxFactory,
         TxFormat,
-        unpack,
         )
 from chainlib.eth.contract import (
         abi_decode_single,
@@ -45,6 +44,7 @@ from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.queue.tx import (
         queue_create,
         register_tx,
+        unpack,
         )
 from cic_eth.queue.query import get_tx
 from cic_eth.task import (
@@ -53,6 +53,11 @@ from cic_eth.task import (
         CriticalSQLAlchemyAndSignerTask,
         CriticalWeb3AndSignerTask,
         )
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        unpack_normal,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -66,6 +71,7 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
+#def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 
@@ -130,16 +136,16 @@ def cache_gas_data(
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
 
     session = SessionBase.create_session()
 
     tx_dict = {
-            'hash': tx_hash_hex,
+            'hash': tx['hash'],
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
+            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': tx['value'],
             'to_value': tx['value'],
             }
@@ -150,7 +156,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -170,6 +176,20 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
+    if address != None:
+        if not is_checksum_address(address):
+            raise ValueError('invalid address {}'.format(address))
+        address = tx_normalize.wallet_address(address)
+
+    tx_hashes = []
+    txs = []
+    for tx_hash in tx_hashes_hex:
+        tx_hash = tx_normalize.tx_hash(tx_hash)
+        tx_hashes.append(tx_hash)
+    for tx in txs_hex:
+        tx = tx_normalize.tx_wire(tx)
+        txs.append(tx)
+
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
 
@@ -187,9 +207,6 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
                 raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
             addresspass.append(address)
 
-    if not is_checksum_address(address):
-        raise ValueError('invalid address {}'.format(address))
-
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
@@ -304,6 +321,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # Determine value of gas tokens to send
     # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
     # however, we will perform a 0-value transaction instead
+    recipient_address = tx_normalize.wallet_address(recipient_address)
     zero_amount = False
     session = SessionBase.create_session()
     status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
@@ -378,6 +396,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     :returns: Transaction hash
     :rtype: str, 0x-hex
     """
+    txold_hash_hex = tx_normalize.tx_hash(txold_hash_hex)
     session = SessionBase.create_session()
 
     otx = Otx.load(txold_hash_hex, session)

apps/cic-eth/cic_eth/queue/balance.py

@@ -15,6 +15,7 @@ from chainqueue.db.enum import (
 # local imports
 from cic_eth.db import SessionBase
 from cic_eth.task import CriticalSQLAlchemyTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -22,6 +23,9 @@ logg = logging.getLogger()
 
 
 def __balance_outgoing_compatible(token_address, holder_address):
+    token_address = tx_normalize.executable_address(token_address)
+    holder_address = tx_normalize.wallet_address(holder_address)
+
     session = SessionBase.create_session()
     q = session.query(TxCache.from_value)
     q = q.join(Otx)
@@ -58,6 +62,9 @@ def balance_outgoing(tokens, holder_address, chain_spec_dict):
 
 
 def __balance_incoming_compatible(token_address, receiver_address):
+    token_address = tx_normalize.executable_address(token_address)
+    receiver_address = tx_normalize.wallet_address(receiver_address)
+
     session = SessionBase.create_session()
     q = session.query(TxCache.to_value)
     q = q.join(Otx)
@@ -110,7 +117,7 @@ def assemble_balances(balances_collection):
     logg.debug('received collection {}'.format(balances_collection))
     for c in balances_collection:
         for b in c:
-            address = b['address']
+            address = tx_normalize.executable_address(b['address'])
             if tokens.get(address) == None:
                 tokens[address] = {
                     'address': address,

apps/cic-eth/cic_eth/queue/lock.py

@@ -6,6 +6,7 @@ import celery
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db import SessionBase
 from cic_eth.db.models.lock import Lock
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -21,6 +22,9 @@ def get_lock(address=None):
     :returns: List of locks
     :rtype: list of dicts
     """
+    if address != None:
+        address = tx_normalize.wallet_address(address)
+
     session = SessionBase.create_session()
     q = session.query(
             Lock.date_created,

apps/cic-eth/cic_eth/queue/query.py

@@ -4,8 +4,8 @@ import datetime
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.eth.tx import unpack
 import chainqueue.sql.query
+from chainlib.eth.tx import unpack
 from chainqueue.db.enum import (
         StatusEnum,
         is_alive,
@@ -20,6 +20,10 @@ from cic_eth.db.enum import LockEnum
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.lock import Lock
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import (
+        tx_normalize,
+        unpack_normal,
+        )
 
 celery_app = celery.current_app
 
@@ -27,49 +31,76 @@ celery_app = celery.current_app
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx_cache(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    session = SessionBase.create_session()
+    return get_tx_cache_local(chain_spec, tx_hash)
+
+
+def get_tx_cache_local(chain_spec, tx_hash, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    session = SessionBase.bind_session(session)
     r = chainqueue.sql.query.get_tx_cache(chain_spec, tx_hash, session=session)
-    session.close()
+    SessionBase.release_session(session)
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    session = SessionBase.create_session()
+    return get_tx_local(chain_spec, tx_hash)
+
+
+def get_tx_local(chain_spec, tx_hash, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    session = SessionBase.bind_session(session)
     r =  chainqueue.sql.query.get_tx(chain_spec, tx_hash, session=session)
-    session.close()
+    SessionBase.release_session(session)
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    session = SessionBase.create_session()
+    return get_account_tx_local(chain_spec, address, as_sender=as_sender, as_recipient=as_recipient, counterpart=counterpart)
+
+
+def get_account_tx_local(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=None):
+    address = tx_normalize.wallet_address(address)
+    session = SessionBase.bind_session(session)
     r = chainqueue.sql.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
-    session.close()
+    SessionBase.release_session(session)
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
+def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    return get_upcoming_tx_nolock_local(chain_spec, status=status, not_status=not_status, recipient=recipient, before=before, limit=limit)
+
+
+def get_upcoming_tx_nolock_local(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
+    recipient = tx_normalize.wallet_address(recipient)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
+    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack_normal)
     session.close()
     return r
 
 
 def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
-    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
+    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack_normal)
 
 
 def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
-    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
+    sender = tx_normalize.wallet_address(sender)
+    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack_normal)
 
 
 def get_nonce_tx(chain_spec, nonce, sender):
-    return get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack)
+    sender = tx_normalize.wallet_address(sender)
+    return get_nonce_tx_local(chain_spec, nonce, sender)
+
+
+def get_nonce_tx_local(chain_spec, nonce, sender, session=None):
+    sender = tx_normalize.wallet_address(sender)
+    return chainqueue.sql.query.get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack_normal, session=session)
 
 
 def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
@@ -91,6 +122,8 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     :returns: Transactions
     :rtype: dict, with transaction hash as key, signed raw transaction as value
     """
+    if recipient != None:
+        recipient = tx_normalize.wallet_address(recipient)
     session = SessionBase.bind_session(session)
     q_outer = session.query(
             TxCache.sender,

apps/cic-eth/cic_eth/queue/state.py

@@ -6,12 +6,14 @@ import chainqueue.sql.state
 import celery
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_sent(chain_spec_dict, tx_hash, fail=False):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_sent(chain_spec, tx_hash, fail, session=session)
@@ -21,6 +23,7 @@ def set_sent(chain_spec_dict, tx_hash, fail=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
@@ -30,6 +33,7 @@ def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_cancel(chain_spec_dict, tx_hash, manual=False):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_cancel(chain_spec, tx_hash, manual, session=session)
@@ -39,6 +43,7 @@ def set_cancel(chain_spec_dict, tx_hash, manual=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_rejected(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_rejected(chain_spec, tx_hash, session=session)
@@ -48,6 +53,7 @@ def set_rejected(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_fubar(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_fubar(chain_spec, tx_hash, session=session)
@@ -57,6 +63,7 @@ def set_fubar(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_manual(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_manual(chain_spec, tx_hash, session=session)
@@ -66,6 +73,7 @@ def set_manual(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_ready(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_ready(chain_spec, tx_hash, session=session)
@@ -75,6 +83,7 @@ def set_ready(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_reserved(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_reserved(chain_spec, tx_hash, session=session)
@@ -84,6 +93,7 @@ def set_reserved(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_waitforgas(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_waitforgas(chain_spec, tx_hash, session=session)
@@ -93,6 +103,7 @@ def set_waitforgas(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_state_log(chain_spec_dict, tx_hash):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.get_state_log(chain_spec, tx_hash, session=session)
@@ -102,6 +113,7 @@ def get_state_log(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def obsolete(chain_spec_dict, tx_hash, final):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)

apps/cic-eth/cic_eth/queue/time.py

@@ -13,6 +13,7 @@ from chainqueue.error import NotLocalTxError
 # local imports
 from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -20,6 +21,7 @@ logg = logging.getLogger()
 
 
 def tx_times(tx_hash, chain_spec, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
 
     session = SessionBase.bind_session(session)
 

apps/cic-eth/cic_eth/queue/tx.py

@@ -32,12 +32,16 @@ from cic_eth.db import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.error import LockedError
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
 def queue_create(chain_spec, nonce, holder_address, tx_hash, signed_tx, session=None):
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    signed_tx = tx_normalize.tx_hash(signed_tx)
+    holder_address = tx_normalize.wallet_address(holder_address)
     session = SessionBase.bind_session(session)
 
     lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session) 
@@ -67,6 +71,8 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No
     :returns: Tuple; Transaction hash, signed raw transaction data
     :rtype: tuple
     """
+    tx_hash_hex = tx_normalize.tx_hash(tx_hash_hex)
+    tx_signed_raw_hex = tx_normalize.tx_hash(tx_signed_raw_hex)
     logg.debug('adding queue tx {}:{} -> {}'.format(chain_spec, tx_hash_hex, tx_signed_raw_hex))
     tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex))
     tx = unpack(tx_signed_raw, chain_spec)

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -101,14 +101,14 @@ class DispatchSyncer:
                     LockEnum.QUEUE,
                     tx['from'],
                     ],
-                queue=queue,
+                queue=config.get('CELERY_QUEUE'),
                 )
             s_send = celery.signature(
                     'cic_eth.eth.tx.send',
                     [
                         self.chain_spec.asdict(),
                         ], 
-                    queue=queue,
+                    queue=config.get('CELERY_QUEUE'),
                     )
             s_check.link(s_send)
             t = s_check.apply_async()

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -10,15 +10,14 @@ from chainlib.eth.tx import unpack
 from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
-from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
 from chainlib.eth.address import to_checksum_address
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.queue.query import get_paused_tx
 from .base import SyncFilter
 
-#logg = logging.getLogger().getChild(__name__)
 logg = logging.getLogger()
 
 

apps/cic-eth/cic_eth/runnable/tag.py

@@ -18,7 +18,7 @@ from cic_eth.db.models.base import SessionBase
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_base
+arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
 local_arg_flags = cic_eth.cli.argflag_local_taskcallback
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.add_positional('tag', type=str, help='address tag')

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         12,
-        2,
+        4,
-        'alpha.4',
+        'alpha.7',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -8,6 +8,7 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_ARGS=""
 #RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
 #    pip install --index-url https://pypi.org/simple \
 #    --force-reinstall \
@@ -18,6 +19,7 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
+    $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt
@@ -31,6 +33,7 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
+    $EXTRA_PIP_ARGS \
     cic-eth-aux-erc20-demurrage-token~=0.0.2a6
 
 COPY docker/entrypoints/* ./ 

apps/cic-eth/docker/Dockerfile_ci

@@ -9,6 +9,7 @@ WORKDIR /usr/src/cic-eth
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_ARGS=""
 #RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
 #    pip install --index-url https://pypi.org/simple \
 #    --force-reinstall \
@@ -18,6 +19,7 @@ COPY *requirements.txt .
 RUN pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
+    $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.7a5,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
 semver==2.13.0

apps/cic-eth/scripts/migrate.py

@@ -10,6 +10,7 @@ from alembic.config import Config as AlembicConfig
 import confini
 
 from cic_eth.db import dsn_from_config
+import cic_eth.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -19,25 +20,20 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_eth', 'db')
 migrationsdir = os.path.join(dbdir, 'migrations')
 
-config_dir = os.path.join('/usr/local/etc/cic-eth')
+arg_flags = cic_eth.cli.argflag_std_base
 
-argparser = argparse.ArgumentParser()
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
 argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-if args.vv:
+extra_args = {
-    logging.getLogger().setLevel(logging.DEBUG)
+    'migrations_dir': None,
-elif args.v:
+    'reset': None,
-    logging.getLogger().setLevel(logging.INFO)
+    'f': '_FORCE_ACTION',
-
+        }
-config = confini.Config(args.c, args.env_prefix)
+config = cic_eth.cli.Config.from_args(args, arg_flags, 0, extra_args=extra_args)
-config.process()
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config:\n{}'.format(config))

apps/cic-eth/services_requirements.txt

@@ -1,15 +1,15 @@
-chainqueue>=0.0.3a2,<0.1.0
+chainqueue>=0.0.5a1,<0.1.0
-chainsyncer[sql]>=0.0.6a1,<0.1.0
+chainsyncer[sql]>=0.0.6a3,<0.1.0
 alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 redis==3.5.3
-hexathon~=0.0.1a7
+hexathon~=0.0.1a8
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.1.4a1,<0.2.0
+eth-address-index>=0.2.3a4,<0.3.0
-eth-accounts-index>=0.0.14a1,<0.1.0
+eth-accounts-index>=0.1.2a3,<0.2.0
-cic-eth-registry>=0.5.8a1,<0.6.0
+cic-eth-registry>=0.6.1a2,<0.7.0
-erc20-faucet>=0.2.4a2,<0.3.0
+erc20-faucet>=0.3.2a2,<0.4.0
-erc20-transfer-authorization>=0.3.4a1,<0.4.0
+erc20-transfer-authorization>=0.3.5a2,<0.4.0
-sarafu-faucet>=0.0.5a2,<0.1.0
+sarafu-faucet>=0.0.7a2,<0.1.0
 moolb~=0.1.1b2

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.0.12a1
+eth-erc20~=0.1.2a2

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -200,6 +200,7 @@ def test_callback_filter(
     assert r['status'] == 1
 
     rcpt = snake_and_camel(r)
+    tx.block.hash = rcpt['block_hash']
     tx.apply_receipt(rcpt)
 
     fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -1,7 +1,6 @@
 # external imports
 from chainlib.connection import RPCConnection
 from chainlib.eth.nonce import OverrideNonceOracle
-from chainqueue.sql.tx import create as queue_create
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -26,6 +25,8 @@ from chainqueue.db.enum import StatusBits
 # local imports
 from cic_eth.runnable.daemons.filters.gas import GasFilter
 from cic_eth.eth.gas import cache_gas_data
+from cic_eth.encode import tx_normalize
+from cic_eth.queue.tx import queue_create
 
 
 def test_filter_gas(

apps/cic-eth/tests/filters/test_register_filter.py

@@ -22,10 +22,11 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+from cic_eth.encode import tx_normalize
+from cic_eth.queue.query import get_account_tx_local
 
 logg = logging.getLogger()
 
@@ -79,7 +80,7 @@ def test_register_filter(
     t.get_leaf()
     assert t.successful()
 
-    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    gift_txs = get_account_tx_local(default_chain_spec, agent_roles['ALICE'], as_sender=True, session=init_database)
     ks = list(gift_txs.keys())
     assert len(ks) == 1
 

apps/cic-eth/tests/task/api/test_admin.py

@@ -34,10 +34,6 @@ from chainqueue.sql.state import (
         set_ready,
         set_reserved,
         )
-from chainqueue.sql.query import (
-        get_tx,
-        get_nonce_tx_cache,
-        )
 
 # local imports
 from cic_eth.api.admin import AdminApi
@@ -46,6 +42,11 @@ from cic_eth.db.enum import LockEnum
 from cic_eth.error import InitializationError
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.queue.tx import queue_create
+from cic_eth.queue.query import (
+        get_tx,
+        get_nonce_tx_local,
+        )
+from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger()
 
@@ -286,13 +287,15 @@ def test_fix_nonce(
     assert t.successful()
 
     init_database.commit()
-    
+   
-    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
+    logg.debug('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+    txs = get_nonce_tx_local(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
     ks = txs.keys()
     assert len(ks) == 2
 
     for k in ks:
-        hsh = add_0x(k)
+        #hsh = add_0x(k)
+        hsh = tx_normalize.tx_hash(k)
         otx = Otx.load(hsh, session=init_database)
         init_database.refresh(otx)
         logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -30,7 +30,6 @@ from chainqueue.sql.state import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.query import get_nonce_tx_cache
 from eth_erc20 import ERC20
 from cic_eth_registry import CICRegistry
 
@@ -38,6 +37,7 @@ from cic_eth_registry import CICRegistry
 from cic_eth.api.admin import AdminApi
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.eth.erc20 import cache_transfer_data
+from cic_eth.queue.query import get_nonce_tx_local
 
 logg = logging.getLogger()
 
@@ -312,7 +312,7 @@ def test_resend_inplace(
     otx = Otx.load(tx_hash_hex, session=init_database)
     assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
 
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_local(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 2
 
 
@@ -363,10 +363,10 @@ def test_resend_clone(
     assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
     assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
 
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_local(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 1
 
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_local(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 1
 
     otx = Otx.load(txs[0], session=init_database)

apps/cic-eth/tests/task/test_task_gas.py

@@ -21,7 +21,6 @@ from chainlib.eth.constant import (
         MINIMUM_FEE_UNITS,
         MINIMUM_FEE_PRICE,
         )
-from chainqueue.sql.tx import create as queue_create
 from chainqueue.sql.query import get_tx
 from chainqueue.db.enum import StatusBits
 from chainqueue.sql.state import (
@@ -35,6 +34,7 @@ from hexathon import strip_0x
 # local imports
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.error import OutOfGasError
+from cic_eth.queue.tx import queue_create
 
 logg = logging.getLogger()
 

apps/cic-eth/tests/task/test_task_list.py

@@ -51,6 +51,7 @@ def test_ext_tx_collate(
         tx_hash_hex,
         tx_signed_raw_hex,
         )
+    otx.block = 666
     init_database.add(otx)
     init_database.commit()
 

apps/cic-eth/tests/unit/db/test_tx.py

@@ -46,6 +46,7 @@ def test_set(
         tx_hash_hex,
         tx_signed_raw_hex,
         )
+    otx.block = 666
     init_database.add(otx)
     init_database.commit()
 
@@ -74,7 +75,6 @@ def test_set(
     assert (tx_stored.destination_token_address == ZERO_ADDRESS)
     assert (tx_stored.from_value == tx['value'])
     assert (tx_stored.to_value == to_value)
-    assert (tx_stored.block_number == 666)
     assert (tx_stored.tx_index == 13)
 
 

apps/cic-eth/tests/unit/queue/test_balances.py

@@ -13,6 +13,7 @@ from cic_eth.queue.balance import (
         balance_incoming,
         assemble_balances,
         )
+from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger()
 
@@ -51,8 +52,8 @@ def test_assemble():
     r = assemble_balances(b)
     logg.debug('r {}'.format(r))
 
-    assert r[0]['address'] == token_foo
+    assert r[0]['address'] == tx_normalize.executable_address(token_foo)
-    assert r[1]['address'] == token_bar
+    assert r[1]['address'] == tx_normalize.executable_address(token_bar)
     assert r[0].get('balance_foo') != None
     assert r[0].get('balance_bar') != None
     assert r[1].get('balance_baz') != None
@@ -74,11 +75,11 @@ def test_outgoing_balance(
     token_address = '0x' + os.urandom(20).hex()
     sender = '0x' + os.urandom(20).hex()
     txc = TxCache(
-            tx_hash,
+            tx_normalize.tx_hash(tx_hash),
-            sender,
+            tx_normalize.wallet_address(sender),
-            recipient,
+            tx_normalize.wallet_address(recipient),
-            token_address,
+            tx_normalize.executable_address(token_address),
-            token_address,
+            tx_normalize.executable_address(token_address),
             1000,
             1000,
             session=init_database,
@@ -125,11 +126,11 @@ def test_incoming_balance(
     token_address = '0x' + os.urandom(20).hex()
     sender = '0x' + os.urandom(20).hex()
     txc = TxCache(
-            tx_hash,
+            tx_normalize.tx_hash(tx_hash),
-            sender,
+            tx_normalize.wallet_address(sender),
-            recipient,
+            tx_normalize.wallet_address(recipient),
-            token_address,
+            tx_normalize.executable_address(token_address),
-            token_address,
+            tx_normalize.executable_address(token_address),
             1000,
             1000,
             session=init_database,

apps/cic-eth/tests/unit/queue/test_query.py

@@ -21,6 +21,7 @@ from cic_eth.db.models.lock import Lock
 from cic_eth.queue.query import get_upcoming_tx
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import cache_gas_data
+from cic_eth.encode import tx_normalize
 
 # test imports
 from tests.util.nonce import StaticNonceOracle
@@ -39,8 +40,8 @@ def test_upcoming_with_lock(
     gas_oracle = RPCGasOracle(eth_rpc)
     c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
 
-    alice_normal = add_0x(hex_uniform(strip_0x(agent_roles['ALICE'])))
+    alice_normal = tx_normalize.wallet_address(agent_roles['ALICE'])
-    bob_normal = add_0x(hex_uniform(strip_0x(agent_roles['BOB'])))
+    bob_normal = tx_normalize.wallet_address(agent_roles['BOB'])
 
     (tx_hash_hex, tx_rpc) = c.create(alice_normal, bob_normal, 100 * (10 ** 6))
     tx_signed_raw_hex = tx_rpc['params'][0]

apps/cic-eth/tests/unit/queue/test_queue_lock.py

@@ -9,7 +9,7 @@ from cic_eth.db.models.lock import Lock
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import LockedError
 from cic_eth.queue.tx import queue_create
-
+from cic_eth.encode import tx_normalize
 
 def test_queue_lock(
     init_database,
@@ -21,6 +21,8 @@ def test_queue_lock(
     address = '0x' + os.urandom(20).hex()
     tx_hash = '0x' + os.urandom(32).hex()
     tx_raw = '0x' + os.urandom(128).hex()
+    address_normal = tx_normalize.wallet_address(address)
+    tx_hash_normal = tx_normalize.tx_hash(tx_hash)
 
     Lock.set(chain_str, LockEnum.QUEUE)
     with pytest.raises(LockedError):
@@ -32,7 +34,7 @@ def test_queue_lock(
                 tx_raw,
                 )
 
-    Lock.set(chain_str, LockEnum.QUEUE, address=address)
+    Lock.set(chain_str, LockEnum.QUEUE, address=address_normal)
     with pytest.raises(LockedError):
         queue_create(
                 default_chain_spec,
@@ -52,7 +54,7 @@ def test_queue_lock(
                 tx_raw,
                 )
 
-    Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
+    Lock.set(chain_str, LockEnum.QUEUE, address=address_normal, tx_hash=tx_hash_normal)
     with pytest.raises(LockedError):
         queue_create(
                 default_chain_spec,
@@ -61,5 +63,3 @@ def test_queue_lock(
                 tx_hash,
                 tx_raw,
                 )
-
-

apps/cic-eth/tools_requirements.txt

@@ -1,8 +1,7 @@
-crypto-dev-signer>=0.4.14b7,<=0.4.14
+crypto-dev-signer>=0.4.15a1,<=0.4.15
-chainqueue>=0.0.3a1,<0.1.0
+chainqueue>=0.0.5a1,<0.1.0
-confini>=0.3.6rc4,<0.5.0
+cic-eth-registry>=0.6.1a2,<0.7.0
-cic-eth-registry>=0.5.8a1,<0.6.0
 redis==3.5.3
-hexathon~=0.0.1a7
+hexathon~=0.0.1a8
 pycryptodome==3.10.1
 pyxdg==0.27

apps/cic-ussd/cic_ussd/account/balance.py

@@ -20,7 +20,7 @@ def get_balances(address: str,
                  asynchronous: bool = False,
                  callback_param: any = None,
                  callback_queue='cic-ussd',
-                 callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Optional[list]:
+                 callback_task='cic_ussd.tasks.callback_handler.balances_callback') -> Optional[list]:
     """This function queries cic-eth for an account's balances, It provides a means to receive the balance either
     asynchronously or synchronously.. It returns a dictionary containing the network, outgoing and incoming balances.
     :param address: Ethereum address of an account.

apps/cic-ussd/cic_ussd/account/transaction.py

@@ -117,18 +117,18 @@ def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
     return recipient_transaction_data, sender_transaction_data
 
 
-def validate_transaction_account(session: Session, transaction: dict) -> Account:
+def validate_transaction_account(blockchain_address: str, role: str, session: Session) -> Account:
     """This function checks whether the blockchain address specified in a parsed transaction object resolves to an
     account object in the ussd system.
-    :param session: Database session object.
+    :param blockchain_address:
-    :type session: Session
+    :type blockchain_address:
-    :param transaction: Parsed transaction data object.
+    :param role:
-    :type transaction: dict
+    :type role:
+    :param session:
+    :type session:
     :return:
     :rtype:
     """
-    blockchain_address = transaction.get('blockchain_address')
-    role = transaction.get('role')
     session = SessionBase.bind_session(session)
     account = session.query(Account).filter_by(blockchain_address=blockchain_address).first()
     if not account:

apps/cic-ussd/cic_ussd/processor/util.py

@@ -67,6 +67,7 @@ def resume_last_ussd_session(last_state: str) -> Document:
         'exit',
         'exit_invalid_pin',
         'exit_invalid_new_pin',
+        'exit_invalid_recipient',
         'exit_invalid_request',
         'exit_pin_blocked',
         'exit_pin_mismatch',

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -4,6 +4,7 @@ from typing import Tuple
 
 # third party imports
 import celery
+from phonenumbers.phonenumberutil import NumberParseException
 
 # local imports
 from cic_ussd.account.balance import get_cached_available_balance
@@ -21,23 +22,21 @@ logg = logging.getLogger(__file__)
 
 
 def is_valid_recipient(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
-    """This function checks that a user exists, is not the initiator of the transaction, has an active account status
+    """This function checks that a phone number provided as the recipient of a transaction does not match the sending
-    and is authorized to perform  standard transactions.
+    party's own phone number.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
-    :return: A user's validity
+    :return: A recipient account's validity for a transaction
     :rtype: bool
     """
     user_input, ussd_session, account, session = state_machine_data
-    phone_number = process_phone_number(user_input, E164Format.region)
+    try:
-    session = SessionBase.bind_session(session=session)
+        phone_number = process_phone_number(user_input, E164Format.region)
-    recipient = Account.get_by_phone_number(phone_number=phone_number, session=session)
+    except NumberParseException:
-    SessionBase.release_session(session=session)
+        phone_number = None
     is_not_initiator = phone_number != account.phone_number
-    has_active_account_status = False
+    is_present = Account.get_by_phone_number(phone_number, session) is not None
-    if recipient:
+    return phone_number is not None and phone_number.startswith('+') and is_present and is_not_initiator
-        has_active_account_status = recipient.get_status(session) == AccountStatus.ACTIVE.name
-    return is_not_initiator and has_active_account_status and recipient is not None
 
 
 def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -138,7 +138,7 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     balances_data = result[0]
     available_balance = calculate_available_balance(balances_data)
     transaction = param
-    blockchain_address = param.get('blockchain_address')
+    blockchain_address = transaction.get('blockchain_address')
     transaction['available_balance'] = available_balance
     queue = self.request.delivery_info.get('routing_key')
 
@@ -150,10 +150,10 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     )
     s_notify_account = celery.signature('cic_ussd.tasks.notifications.transaction', queue=queue)
 
-    if param.get('transaction_type') == 'transfer':
+    if transaction.get('transaction_type') == 'transfer':
         celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async()
 
-    if param.get('transaction_type') == 'tokengift':
+    if transaction.get('transaction_type') == 'tokengift':
         s_process_account_metadata = celery.signature(
             'cic_ussd.tasks.processor.parse_transaction', [{}, transaction], queue=queue
         )
@@ -184,10 +184,11 @@ def transaction_callback(result: dict, param: str, status_code: int):
     source_token_value = result.get('source_token_value')
 
     recipient_metadata = {
-        "token_symbol": destination_token_symbol,
+        "alt_blockchain_address": sender_blockchain_address,
-        "token_value": destination_token_value,
         "blockchain_address": recipient_blockchain_address,
         "role": "recipient",
+        "token_symbol": destination_token_symbol,
+        "token_value": destination_token_value,
         "transaction_type": param
     }
 
@@ -201,10 +202,11 @@ def transaction_callback(result: dict, param: str, status_code: int):
 
     if param == 'transfer':
         sender_metadata = {
+            "alt_blockchain_address": recipient_blockchain_address,
             "blockchain_address": sender_blockchain_address,
+            "role": "sender",
             "token_symbol": source_token_symbol,
             "token_value": source_token_value,
-            "role": "sender",
             "transaction_type": param
         }
 

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -29,7 +29,8 @@ def transaction(notification_data: dict):
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')
     token_symbol = notification_data.get('token_symbol')
-    transaction_account_metadata = notification_data.get('metadata_id')
+    alt_metadata_id = notification_data.get('alt_metadata_id')
+    metadata_id = notification_data.get('metadata_id')
     transaction_type = notification_data.get('transaction_type')
     timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
 
@@ -47,7 +48,8 @@ def transaction(notification_data: dict):
                                            preferred_language=preferred_language,
                                            amount=amount,
                                            token_symbol=token_symbol,
-                                           tx_sender_information=transaction_account_metadata,
+                                           tx_recipient_information=metadata_id,
+                                           tx_sender_information=alt_metadata_id,
                                            timestamp=timestamp,
                                            balance=balance)
         if role == 'sender':
@@ -56,6 +58,7 @@ def transaction(notification_data: dict):
                                            preferred_language=preferred_language,
                                            amount=amount,
                                            token_symbol=token_symbol,
-                                           tx_recipient_information=transaction_account_metadata,
+                                           tx_recipient_information=alt_metadata_id,
+                                           tx_sender_information=metadata_id,
                                            timestamp=timestamp,
                                            balance=balance)

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -11,6 +11,7 @@ from chainlib.hash import strip_0x
 from cic_ussd.account.statement import get_cached_statement
 from cic_ussd.account.transaction import aux_transaction_data, validate_transaction_account
 from cic_ussd.cache import cache_data, cache_data_key
+from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 
 
@@ -72,13 +73,17 @@ def parse_transaction(preferences: dict, transaction: dict) -> dict:
     preferred_language = preferences.get('preferred_language')
     if not preferred_language:
         preferred_language = i18n.config.get('fallback')
-
+    transaction['preferred_language'] = preferred_language
     transaction = aux_transaction_data(preferred_language, transaction)
     session = SessionBase.create_session()
-    account = validate_transaction_account(session, transaction)
+    role = transaction.get('role')
-    metadata_id = account.standard_metadata_id()
+    alt_blockchain_address = transaction.get('alt_blockchain_address')
-    transaction['metadata_id'] = metadata_id
+    blockchain_address = transaction.get('blockchain_address')
+    account = validate_transaction_account(blockchain_address, role, session)
+    alt_account = session.query(Account).filter_by(blockchain_address=alt_blockchain_address).first()
+    if alt_account:
+        transaction['alt_metadata_id'] = alt_account.standard_metadata_id()
+    transaction['metadata_id'] = account.standard_metadata_id()
     transaction['phone_number'] = account.phone_number
-    session.commit()
     session.close()
     return transaction

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.10')
+version = (0, 3, 1, 'alpha.4')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/requirements.txt

@@ -1,10 +1,10 @@
 alembic==1.4.2
 bcrypt==3.2.0
 celery==4.4.7
-cic-eth[services]==0.12.2a3
+cic-eth[services]~=0.12.4a7
 cic-notify~=0.4.0a10
 cic-types~=0.1.0a14
-confini~=0.4.1a1
+confini>=0.4.1a1,<0.5.0
 phonenumbers==8.12.12
 psycopg2==2.8.6
 python-i18n[YAML]==0.3.9
@@ -13,11 +13,5 @@ redis==3.5.3
 semver==2.13.0
 SQLAlchemy==1.3.20
 tinydb==4.2.0
-phonenumbers==8.12.12
-redis==3.5.3
-celery==4.4.7
-python-i18n[YAML]==0.3.9
-pyxdg==0.27
-bcrypt==3.2.0
-uWSGI==2.0.19.1
 transitions==0.8.4
+uWSGI==2.0.19.1

apps/cic-ussd/tests/cic_ussd/account/test_transaction.py

@@ -75,17 +75,21 @@ def test_transaction_actors(activated_account, transaction_result, valid_recipie
 def test_validate_transaction_account(activated_account, init_database, transactions_list):
     sample_transaction = transactions_list[0]
     recipient_transaction, sender_transaction = transaction_actors(sample_transaction)
-    recipient_account = validate_transaction_account(init_database, recipient_transaction)
+    recipient_account = validate_transaction_account(
-    sender_account = validate_transaction_account(init_database, sender_transaction)
+        recipient_transaction.get('blockchain_address'), recipient_transaction.get('role'), init_database)
+    sender_account = validate_transaction_account(
+        sender_transaction.get('blockchain_address'), sender_transaction.get('role'),  init_database)
     assert isinstance(recipient_account, Account)
     assert isinstance(sender_account, Account)
     sample_transaction = transactions_list[1]
     recipient_transaction, sender_transaction = transaction_actors(sample_transaction)
     with pytest.raises(UnknownUssdRecipient) as error:
-        validate_transaction_account(init_database, recipient_transaction)
+        validate_transaction_account(
+            recipient_transaction.get('blockchain_address'), recipient_transaction.get('role'), init_database)
     assert str(
         error.value) == f'Tx for recipient: {recipient_transaction.get("blockchain_address")} has no matching account in the system.'
-    validate_transaction_account(init_database, sender_transaction)
+    validate_transaction_account(
+        sender_transaction.get('blockchain_address'), sender_transaction.get('role'), init_database)
     assert f'Tx from sender: {sender_transaction.get("blockchain_address")} has no matching account in system.'
 
 

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py

@@ -30,8 +30,6 @@ def test_is_valid_recipient(activated_account,
                             valid_recipient):
     state_machine = ('0112365478', generic_ussd_session, valid_recipient, init_database)
     assert is_valid_recipient(state_machine) is False
-    state_machine = (pending_account.phone_number, generic_ussd_session, valid_recipient, init_database)
-    assert is_valid_recipient(state_machine) is False
     state_machine = (valid_recipient.phone_number, generic_ussd_session, activated_account, init_database)
     assert is_valid_recipient(state_machine) is True
 

apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py

@@ -24,7 +24,8 @@ def test_transaction(celery_session_worker,
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')
     token_symbol = notification_data.get('token_symbol')
-    transaction_account_metadata = notification_data.get('metadata_id')
+    alt_metadata_id = notification_data.get('alt_metadata_id')
+    metadata_id = notification_data.get('metadata_id')
     timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
     s_transaction = celery.signature(
         'cic_ussd.tasks.notifications.transaction', [notification_data]
@@ -36,7 +37,8 @@ def test_transaction(celery_session_worker,
                               preferred_language=preferred_language,
                               amount=amount,
                               token_symbol=token_symbol,
-                              tx_recipient_information=transaction_account_metadata,
+                              tx_recipient_information=alt_metadata_id,
+                              tx_sender_information=metadata_id,
                               timestamp=timestamp,
                               balance=balance)
     assert mock_notifier_api.get('message') == message
@@ -52,7 +54,8 @@ def test_transaction(celery_session_worker,
                               preferred_language=preferred_language,
                               amount=amount,
                               token_symbol=token_symbol,
-                              tx_sender_information=transaction_account_metadata,
+                              tx_recipient_information=metadata_id,
+                              tx_sender_information=alt_metadata_id,
                               timestamp=timestamp,
                               balance=balance)
     assert mock_notifier_api.get('message') == message

apps/cic-ussd/tests/fixtures/transaction.py

@@ -5,12 +5,18 @@ import random
 import pytest
 
 # local import
+from cic_ussd.account.balance import get_cached_available_balance
 
 # tests imports
 
 
 @pytest.fixture(scope='function')
-def notification_data(activated_account, cache_person_metadata, cache_preferences, preferences):
+def notification_data(activated_account,
+                      cache_person_metadata,
+                      cache_preferences,
+                      cache_balances,
+                      preferences,
+                      valid_recipient):
     return {
         'blockchain_address': activated_account.blockchain_address,
         'token_symbol': 'GFT',
@@ -18,6 +24,7 @@ def notification_data(activated_account, cache_person_metadata, cache_preference
         'role': 'sender',
         'action_tag': 'Sent',
         'direction_tag': 'To',
+        'alt_metadata_id': valid_recipient.standard_metadata_id(),
         'metadata_id': activated_account.standard_metadata_id(),
         'phone_number': activated_account.phone_number,
         'available_balance': 50.0,

apps/cic-ussd/var/lib/locale/sms.en.yml

@@ -2,9 +2,9 @@ en:
   account_successfully_created: |-
     You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
   received_tokens: |-
-    Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
+    Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp} to %{tx_recipient_information}. New balance is %{balance} %{token_symbol}.
   sent_tokens: |-
-    Successfully sent %{amount} %{token_symbol} to %{tx_recipient_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
+    Successfully sent %{amount} %{token_symbol} to %{tx_recipient_information} %{timestamp} from %{tx_sender_information}. New balance is %{balance} %{token_symbol}.
   terms: |-
     By using the service, you agree to the terms and conditions at http://grassecon.org/tos
   upsell_unregistered_recipient: |-

apps/cic-ussd/var/lib/locale/sms.sw.yml

@@ -2,9 +2,9 @@ sw:
   account_successfully_created: |-
     Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
   received_tokens: |-
-    Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
+    Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp} ikapokewa na %{tx_recipient_information}. Salio lako ni %{balance} %{token_symbol}.
   sent_tokens: |-
-    Umetuma %{amount} %{token_symbol} kwa %{tx_recipient_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
+    Umetuma %{amount} %{token_symbol} kwa %{tx_recipient_information} %{timestamp} kutoka kwa %{tx_sender_information}. Salio lako ni %{balance} %{token_symbol}.
   terms: |-
     Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos
   upsell_unregistered_recipient: |-

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -7,10 +7,8 @@ en:
       3. Help
     initial_pin_entry: |-
       CON Please enter a new four number PIN for your account.
-      0. Back
     initial_pin_confirmation: |-
       CON Enter your four number PIN again
-      0. Back
     enter_given_name: |-
       CON Enter first name
       0. Back
@@ -183,7 +181,7 @@ en:
     exit_pin_mismatch: |-
       END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}.
     exit_invalid_recipient: |-
-      CON Recipient phone number is incorrect.
+      CON Recipient's phone number is not registered or is invalid:
         00. Retry
         99. Exit
     exit_successful_transaction: |-

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -7,13 +7,10 @@ sw:
       3. Help
     initial_pin_entry: |-
       CON Tafadhali weka pin mpya yenye nambari nne kwa akaunti yako
-      0. Nyuma
     initial_pin_confirmation: |-
       CON Weka PIN yako tena
-      0. Nyuma
     enter_given_name: |-
       CON Weka jina lako la kwanza
-      0. Nyuma
     enter_family_name: |-
       CON Weka jina lako la mwisho
       0. Nyuma

apps/contract-migration/docker/Dockerfile

@@ -14,30 +14,33 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C20
 RUN mkdir -vp /usr/local/etc/cic
 
 ENV CONFINI_DIR /usr/local/etc/cic/
-#RUN mkdir -vp $CONFINI_DIR
 
-#ARG cic_config_commit=24287fb253196820f23ff8a7177b122f2cd99a11
-#ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
-#RUN echo Install confini schema files && \
-#	git clone --depth 1 $cic_config_url cic-config && \
-#	cd cic-config && \
-#	git fetch --depth 1 origin $cic_config_commit && \
-#	git checkout $cic_config_commit && \	
-#	cp -v *.ini $CONFINI_DIR
 
 COPY config_template/ /usr/local/etc/cic/
 COPY requirements.txt . 
+COPY override_requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_ARGS=""
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 ARG pip_trusted_host=pypi.org
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple \
+    --pre \
     --force-reinstall \
     --trusted-host $pip_trusted_host \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt 
 
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple \
+    --force-reinstall \
+    --pre \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r override_requirements.txt 
+
+
 COPY . .
 RUN chmod +x *.sh

apps/contract-migration/docker/Dockerfile_ci

@@ -14,28 +14,31 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C20
 RUN mkdir -vp /usr/local/etc/cic
 
 ENV CONFINI_DIR /usr/local/etc/cic/
-RUN mkdir -vp $CONFINI_DIR
 
-ARG cic_config_commit=0abe0867f18077907c7023bf0ef5e466a3984dd8
-ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
-RUN echo Install confini schema files && \
-	git clone --depth 1 $cic_config_url cic-config && \
-	cd cic-config && \
-	git fetch --depth 1 origin $cic_config_commit && \
-	git checkout $cic_config_commit && \	
-	cp -v *.ini $CONFINI_DIR
 
+COPY config_template/ /usr/local/etc/cic/
 COPY requirements.txt . 
+COPY override_requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_ARGS=""
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 ARG pip_trusted_host=pypi.org
 RUN pip install --index-url https://pypi.org/simple \
+    pip install --index-url https://pypi.org/simple \
+    --pre \
     --force-reinstall \
     --trusted-host $pip_trusted_host \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt 
 
+RUN pip install --index-url https://pypi.org/simple \
+    --force-reinstall \
+    --pre \
+    --trusted-host $pip_trusted_host \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r override_requirements.txt 
+
 COPY . .
 RUN chmod +x *.sh

apps/contract-migration/override_requirements.txt

@@ -1,3 +1,4 @@
-#eth-contract-registry==0.5.5a3
+#eth-contract-registry==0.6.3a2
 #erc20-demurrage-token==0.0.2a3
 #eth-address-index==0.1.1a12
+

apps/contract-migration/requirements.txt

@@ -1,10 +1,10 @@
-cic-eth[tools]==0.12.2a4
+cic-eth[tools]==0.12.4a4
-eth-erc20>=0.0.12a1,<0.1.0
+chainlib-eth>=0.0.9a7,<0.1.0
-erc20-demurrage-token>=0.0.3a1,<0.1.0
+eth-erc20>=0.1.2a2,<0.2.0
-eth-address-index>=0.1.4a1,<0.2.0
+erc20-demurrage-token>=0.0.5a2,<0.1.0
-eth-accounts-index>=0.0.14a1,<0.1.0
+eth-accounts-index>=0.1.2a2,<0.2.0
-cic-eth-registry>=0.5.8a1,<0.6.0
+eth-address-index>=0.2.3a4,<0.3.0
-erc20-faucet>=0.2.4a1,<0.3.0
+cic-eth-registry>=0.6.1a2,<0.7.0
-erc20-transfer-authorization>=0.3.4a1,<0.4.0
+erc20-transfer-authorization>=0.3.5a2,<0.4.0
-sarafu-faucet>=0.0.5a5,<0.1.0
+erc20-faucet>=0.3.2a2,<0.4.0
-chainlib-eth>=0.0.7a1,<0.1.0
+sarafu-faucet>=0.0.7a2,<0.1.0

apps/contract-migration/reset.sh

@@ -67,6 +67,9 @@ echo -n 1 > $init_level_file
 # Wait for the backend to be up, if we know where it is.
 if [[ -n "${ETH_PROVIDER}" ]]; then
 
+	export CONFINI_DIR=$_CONFINI_DIR
+	unset CONFINI_DIR
+
 	if [ ! -z "$DEV_USE_DOCKER_WAIT_SCRIPT" ]; then
 		echo "waiting for ${ETH_PROVIDER}..."
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
@@ -82,8 +85,8 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 			TOKEN_NAME=$TOKEN_SYMBOL
 		fi
 		>&2 echo deploying default token $TOKEN_TYPE
-		echo giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
+		echo giftable-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
-		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
 	elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
 		if [ -z "$TOKEN_SYMBOL" ]; then
 			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
@@ -99,61 +102,57 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 				>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
 			fi
 		fi
-		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww`
+		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s` 
 	else
 		>&2 echo unknown token type $TOKEN_TYPE
 		exit 1
 	fi
-	giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
+
+	echo "giftable-token-gift $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT"
+	giftable-token-gift $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
 
 	>&2 echo "deploy account index contract"
-	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -vv -w`
+	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -vv -s -u -w`
 	>&2 echo "add deployer address as account index writer"
-	eth-accounts-index-writer $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+	eth-accounts-index-writer $fee_price_arg -s -u -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
 	>&2 echo "deploy contract registry contract"
-	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
+	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $ETH_PROVIDER -vv -s -u -w`
-	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
-	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
 
 	# Deploy address declarator registry
 	>&2 echo "deploy address declarator contract"
 	declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
-	DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv $declarator_description`
+	DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv $declarator_description`
-	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS
 
 	# Deploy transfer authorization contact
 	>&2 echo "deploy transfer auth contract"
 	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
-	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv --identifier TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
 
 	# Deploy token index contract
 	>&2 echo "deploy token index contract"
-	DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
+	DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
-	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
+	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
 	>&2 echo "add reserve token to token index"
-	eth-token-index-add $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE  -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv -a $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
+	eth-token-index-add $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE  -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
 
 	# Sarafu faucet contract
 	>&2 echo "deploy token faucet contract"
-	export _CONFINI_DIR=$CONFINI_DIR
-	unset CONFINI_DIR
 	DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s`
 
 	>&2 echo "set token faucet amount"
 	sarafu-faucet-set $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
 
-	export CONFINI_DIR=$_CONFINI_DIR
-
 	>&2 echo "register faucet in registry"
-	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS
+	eth-contract-registry-set -s -u $fee_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS
 
 	>&2 echo "set faucet as token minter"
-	giftable-token-minter $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS
+	giftable-token-minter -s -u $fee_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -e $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS
-
-
-
 
+	export CONFINI_DIR=$_CONFINI_DIR
 else
 	echo "\$ETH_PROVIDER not set!"
 	exit 1

apps/contract-migration/seed_cic_eth.sh

@@ -35,10 +35,13 @@ set -a
 
 #pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL  eth-address-index==0.1.1a7
 
+export CONFINI_DIR=$_CONFINI_DIR
+unset CONFINI_DIR
+
 # get required addresses from registries
-DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry`
+DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $ETH_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw TokenRegistry`
-DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`
+DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $ETH_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw AccountRegistry`
-DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL`
+DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -u -p $ETH_PROVIDER -e $DEV_TOKEN_INDEX_ADDRESS -vv --raw $CIC_DEFAULT_TOKEN_SYMBOL`
 cat <<EOF
 Token registry: $DEV_TOKEN_INDEX_ADDRESS
 Account reigstry: $DEV_ACCOUNT_INDEX_ADDRESS
@@ -72,28 +75,29 @@ DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-c
 echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file
 cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"
-eth-accounts-index-writer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
+eth-accounts-index-writer -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 
 # Transfer gas to custodial gas provider adddress
 _CONFINI_DIR=$CONFINI_DIR
 unset CONFINI_DIR
 >&2 echo gift gas to gas gifter
->&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
+>&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
 
 >&2 echo gift gas to sarafu token owner
->&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
+>&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
 
 >&2 echo gift gas to account index owner
->&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
+>&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
 
 
 # Send token to token creator
 >&2 echo "gift tokens to sarafu owner"
->&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount
+echo "giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount"
+>&2 giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount
 
 # Send token to token gifter
 >&2 echo "gift tokens to keystore address"
->&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount
+>&2 giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount
 
 >&2 echo "set sarafu token to reserve token (temporarily while bancor contracts are not connected)"
 echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >> $env_out_file
@@ -101,15 +105,17 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 # Transfer tokens to gifter address
 >&2 echo "transfer tokens to token gifter address"
->&2 erc20-transfer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
+>&2 erc20-transfer -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --fee-limit 100000 -e $DEV_RESERVE_ADDRESS -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
 
 #echo -n 0 > $init_level_file
 
-CONFINI_DIR=$_CONFINI_DIR
+#CONFINI_DIR=$_CONFINI_DIR
 # Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
-cic-eth-ctl -i :: unlock INIT
+cic-eth-ctl -i $CHAIN_SPEC unlock INIT
-cic-eth-ctl -i :: unlock SEND
+cic-eth-ctl -i $CHAIN_SPEC unlock SEND
-cic-eth-ctl -i :: unlock QUEUE
+cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE
+
+export CONFINI_DIR=$_CONFINI_DIR
 
 set +a
 set +e

apps/data-seeding/cic_eth/import_balance.py

@@ -38,7 +38,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
-from cic_base.eth.syncer import chain_interface
+from cic_eth.cli.chain import chain_interface
 from eth_accounts_index import AccountsIndex
 from eth_contract_registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
@@ -215,8 +215,6 @@ def main():
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
     logg.info('found token address {}'.format(token_address))
-    
-    sys.exit(0)
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -248,7 +246,7 @@ def main():
 
     syncer_backend.set(block_offset, 0)
     syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
-    handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
+    handler = Handler(conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)
     

apps/data-seeding/cic_eth/traffic/cmd/traffic.py

@@ -140,8 +140,11 @@ class TrafficRouter:
         for k in keys:
             if len(k) > 8 and k[:8] == 'TRAFFIC_':
                 v = int(dct.get(k))
-                self.add(k[8:].lower(), v)
+                if v == 0:
-                logg.debug('found traffic item {} weight {}'.format(k, v))
+                    logg.debug('skipping traffic item {} with weight {}'.format(k, v))
+                else:
+                    logg.debug('found traffic item {} weight {}'.format(k, v))
+                    self.add(k[8:].lower(), v)
 
 
 # TODO: This will not work well with big networks. The provisioner should use lazy loading and LRU instead.

apps/data-seeding/cic_eth/traffic/config/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis://localhost:63379
+result_url = redis://localhost:63379

apps/data-seeding/cic_eth/traffic/config/redis.ini

@@ -0,0 +1,3 @@
+[redis]
+host = localhost
+port = 63379

apps/data-seeding/cic_eth/traffic/config/rpc.ini

@@ -0,0 +1,2 @@
+[rpc]
+http_provider = http://localhost:63545

apps/data-seeding/cic_eth/traffic/data/config/traffic.ini

@@ -0,0 +1,4 @@
+[traffic]
+#local.noop_traffic = 1
+local.account = 2
+local.transfer = 2

apps/data-seeding/cic_eth/traffic/traffic.py

@@ -17,14 +17,10 @@ from chainlib.eth.gas import RPCGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.block import block_latest
 from hexathon import strip_0x
-from cic_base import (
+import chainlib.eth.cli
-        argparse,
+import cic_eth.cli
-        config,
+from cic_eth.cli.chain import chain_interface
-        log,
+from chainlib.eth.constant import ZERO_ADDRESS
-        rpc,
-        signer as signer_funcs,
-        )
-from cic_base.eth.syncer import chain_interface
 
 # local imports
 #import common
@@ -42,42 +38,45 @@ from cmd.cache import (
 
 
 # common basics
-script_dir = os.path.realpath(os.path.dirname(__file__))
+script_dir = os.path.dirname(os.path.realpath(__file__))
-logg = log.create()
+traffic_schema_dir = os.path.join(script_dir, 'data', 'config') 
-argparser = argparse.create(script_dir, argparse.full_template)
+logging.basicConfig(level=logging.WARNING)
-argparser = argparse.add(argparser, add_traffic_args, 'traffic')
+logg = logging.getLogger()
-args = argparse.parse(argparser, logg)
-config = config.create(args.c, args, args.env_prefix)
 
-# map custom args to local config entries
+arg_flags = cic_eth.cli.argflag_std_read | cic_eth.cli.Flag.WALLET
-batchsize = args.batch_size
+local_arg_flags = cic_eth.cli.argflag_local_taskcallback | cic_eth.cli.argflag_local_chain
-if batchsize < 1:
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-    batchsize = 1
+argparser.add_argument('--batch-size', default=10, type=int, help='number of events to process simultaneously')
-logg.info('batch size {}'.format(batchsize))
+argparser.process_local_flags(local_arg_flags)
-config.add(batchsize, '_BATCH_SIZE', True)
+args = argparser.parse_args()
 
-config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
+extra_args = {
-config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
+    'batch_size': None,
+        }
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, base_config_dir=traffic_schema_dir, extra_args=extra_args)
 
-config.add(args.y, '_KEYSTORE_FILE', True)
+wallet = chainlib.eth.cli.Wallet()
+wallet.from_config(config)
 
-config.add(args.q, '_CELERY_QUEUE', True)
+rpc = chainlib.eth.cli.Rpc(wallet=wallet)
+conn = rpc.connect_by_config(config)
 
-logg.debug(config)
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+
+class NetworkError(Exception):
+    pass
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 def main():
     # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
-    (signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
+    signer = rpc.get_signer()
+    signer_address = rpc.get_sender_address()
 
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # set up registry
-    rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
-    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
-    #registry = registry.init_legacy(config, w3)
     CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
     registry = CICRegistry(chain_spec, conn)
 
@@ -91,7 +90,7 @@ def main():
     handler = TrafficSyncHandler(config, traffic_router, conn)
 
     # Set up syncer
-    syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
+    syncer_backend = MemBackend(config.get('CHAIN_SPEC'), 0)
     o = block_latest()
     r = conn.do(o)
     block_offset = int(strip_0x(r), 16) + 1
@@ -99,26 +98,28 @@ def main():
 
     # get relevant registry entries
     token_registry = registry.lookup('TokenRegistry')
+    if token_registry == ZERO_ADDRESS:
+        raise NetworkError('TokenRegistry value missing from contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
     logg.info('using token registry {}'.format(token_registry))
     token_cache = TokenRegistryCache(chain_spec, token_registry)
 
     account_registry = registry.lookup('AccountRegistry')
+    if account_registry == ZERO_ADDRESS:
+        raise NetworkError('AccountRegistry value missing from contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
     logg.info('using account registry {}'.format(account_registry))
     account_cache = AccountRegistryCache(chain_spec, account_registry)
    
     # Set up provisioner for common task input data
-    #TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
-    #TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
     TrafficProvisioner.oracles['token'] = token_cache
     TrafficProvisioner.oracles['account'] = account_cache
     
     TrafficProvisioner.default_aux = {
-            'chain_spec': config.get('CIC_CHAIN_SPEC'),
+            'chain_spec': config.get('CHAIN_SPEC'),
             'registry': registry,
             'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
             'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
             'redis_db': config.get('REDIS_DB'),
-            'api_queue': config.get('_CELERY_QUEUE'),
+            'api_queue': config.get('CELERY_QUEUE'),
             }
 
     syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=handler.refresh)

apps/data-seeding/config/traffic.ini

@@ -1,4 +0,0 @@
-[traffic]
-#local.noop_traffic = 2
-#local.account = 2
-local.transfer = 2

apps/data-seeding/eth/import_balance.py

@@ -37,7 +37,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
-from cic_base.eth.syncer import chain_interface
+from cic_eth.cli.chain import chain_interface
 from eth_accounts_index import AccountsIndex
 from eth_contract_registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
@@ -172,6 +172,7 @@ class Handler:
         erc20 = ERC20(self.chain_spec, signer=self.signer, gas_oracle=self.gas_oracle, nonce_oracle=self.nonce_oracle)
         o = erc20.decimals(self.token_address)
         r = conn.do(o)
+        logg.debug('parse dec {}'.format(r))
         decimals = erc20.parse_decimals(r)
         multiplier = 10 ** decimals
         balance_full = balance * multiplier

apps/data-seeding/requirements.txt

@@ -1,13 +1,13 @@
-sarafu-faucet~=0.0.5a2
+sarafu-faucet~=0.0.7a1
-cic-eth[tools]~=0.12.2a3
+cic-eth[tools]~=0.12.4a7
 cic-types~=0.1.0a14
-crypto-dev-signer~=0.4.14rc1
+crypto-dev-signer>=0.4.15a1,<=0.4.15
 faker==4.17.1
-chainsyncer~=0.0.6a1
+chainsyncer~=0.0.6a3
-chainlib-eth~=0.0.7a1
+chainlib-eth~=0.0.9a7
-eth-address-index~=0.1.4a1
+eth-address-index~=0.2.3a4
-eth-contract-registry~=0.5.8a1
+eth-contract-registry~=0.6.3a3
-eth-accounts-index~=0.0.14a1
+eth-accounts-index~=0.1.2a3
-eth-erc20~=0.0.12a1
+eth-erc20~=0.1.2a2
-erc20-faucet~=0.2.4a1
+erc20-faucet~=0.3.2a2
 psycopg2==2.8.6

apps/data-seeding/verify.py

@@ -205,8 +205,9 @@ def send_ussd_request(address, data_dir):
 
 class VerifierState:
 
-    def __init__(self, item_keys, active_tests=None):
+    def __init__(self, item_keys, target_count, active_tests=None):
         self.items = {}
+        self.target_count = target_count
         for k in item_keys:
             self.items[k] = 0
         if active_tests == None:
@@ -223,7 +224,7 @@ class VerifierState:
         r = ''
         for k in self.items.keys():
             if k in self.active_tests:
-                r += '{}: {}\n'.format(k, self.items[k])
+                r += '{}: {}/{}\n'.format(k, self.items[k], self.target_count)
             else:
                 r += '{}: skipped\n'.format(k)
         return r
@@ -244,7 +245,7 @@ class VerifierError(Exception):
 class Verifier:
 
     # TODO: what an awful function signature
-    def __init__(self, conn, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, faucet_address, data_dir, exit_on_error=False):
+    def __init__(self, conn, target_count, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, faucet_address, data_dir, exit_on_error=False):
         self.conn = conn
         self.gas_oracle = gas_oracle
         self.chain_spec = chain_spec
@@ -268,7 +269,7 @@ class Verifier:
         self.faucet_amount = self.faucet_tx_factory.parse_token_amount(r)
         logg.info('faucet amount set to {} at verify initialization time'.format(self.faucet_amount))
 
-        self.state = VerifierState(verifymethods, active_tests=active_tests)
+        self.state = VerifierState(verifymethods, target_count, active_tests=active_tests)
 
 
     def verify_accounts_index(self, address, balance=None):
@@ -481,7 +482,7 @@ def main():
 
     f.close()
 
-    verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, token_address, faucet_address, user_dir, exit_on_error)
+    verifier = Verifier(conn, i, api, gas_oracle, chain_spec, account_index_address, token_address, faucet_address, user_dir, exit_on_error)
 
     user_new_dir = os.path.join(user_dir, 'new')
     i = 0
@@ -517,6 +518,7 @@ def main():
             verifier.verify(new_address, balance, debug_stem=s)
             i += 1
 
+    print()
     print(verifier)
 
 

docker-compose.yml

@@ -80,6 +80,7 @@ services:
         pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
         pip_extra_args: $PIP_EXTRA_ARGS
         EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
+        EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS