Merge branch 'master' into bvander/contract-migration-readyz

bug: cic-eth transaction listings

See merge request grassrootseconomics/cic-internal-integration!285

.gitignore

@@ -15,3 +15,4 @@ build/
 .idea
 **/.vim
 **/*secret.yaml
+**/*.bash_history

README.md

@@ -19,6 +19,11 @@ Run app/contract-migration to deploy contracts
 RUN_MASK=3 docker-compose up contract-migration
 ```
 
+View container status
+```
+docker ps --filter network=cic-network --format "table {{.ID}}\t{{.Names}}\t{{.Status}}" --all
+```
+
 stop cluster
 ```
 docker-compose down

apps/cic-cache/docker/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/cic-cache/docker/start_server.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+set -u
+
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+. ./db.sh
+
+if [ $? -ne "0" ]; then
+	>&2 echo db migrate fail
+	exit 1
+fi
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/uwsgi --wsgi-file /root/cic_cache/runnable/daemons/server.py --http :8000 --pyargv "-vv"
+else
+  /usr/local/bin/uwsgi --wsgi-file /root/cic_cache/runnable/daemons/server.py --http :8000 --pyargv "-vv"
+fi
+

apps/cic-cache/docker/start_tasker.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+set -u
+
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+. ./db.sh
+
+if [ $? -ne "0" ]; then
+	>&2 echo db migrate fail
+	exit 1
+fi
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/cic-cache-taskerd -vv
+else
+  /usr/local/bin/cic-cache-taskerd -vv
+fi
+

apps/cic-cache/docker/start_tracker.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+set -e
+set -u
+
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
 . ./db.sh
 
 if [ $? -ne "0" ]; then
@@ -7,4 +12,12 @@ if [ $? -ne "0" ]; then
 	exit 1
 fi
 
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  docker/wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source docker/get_readyz.sh # set env vars form endpoint
   /usr/local/bin/cic-cache-trackerd $@
+else
+  /usr/local/bin/cic-cache-trackerd $@
+fi
+

apps/cic-cache/docker/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a3,<0.7.0
 hexathon~=0.0.1a8
 chainqueue>=0.0.4a6,<0.1.0
 eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/api/api_task.py

@@ -9,6 +9,7 @@ import logging
 # external imports 
 import celery
 from chainlib.chain import ChainSpec
+from hexathon import strip_0x
 
 # local imports
 from cic_eth.api.base import ApiBase
@@ -254,6 +255,8 @@ class Api(ApiBase):
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        #from_address = strip_0x(from_address)
+        #to_address = strip_0x(to_address)
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [

apps/cic-eth/cic_eth/eth/account.py

@@ -13,7 +13,7 @@ from chainlib.eth.sign import (
         new_account,
         sign_message,
         )
-from chainlib.eth.address import to_checksum_address
+from chainlib.eth.address import to_checksum_address, is_address
 from chainlib.eth.tx import TxFormat
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
@@ -31,6 +31,7 @@ from cic_eth.eth.gas import (
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
+from cic_eth.encode import tx_normalize
 from cic_eth.error import (
         RoleMissingError,
         SignerError,
@@ -49,6 +50,7 @@ from cic_eth.queue.tx import (
 from cic_eth.encode import (
         unpack_normal,
         ZERO_ADDRESS_NORMAL,
+        tx_normalize,
         )
 
 logg = logging.getLogger()
@@ -84,7 +86,7 @@ def create(self, password, chain_spec_dict):
     # TODO: It seems infeasible that a can be None in any case, verify
     if a == None:
         raise SignerError('create account')
-        
+    a = tx_normalize.wallet_address(a)
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -175,6 +177,9 @@ def gift(self, account_address, chain_spec_dict):
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
+    if is_address(account_address):
+        account_address = tx_normalize.wallet_address(account_address)
+
     logg.debug('gift account address {} to index'.format(account_address))
     queue = self.request.delivery_info.get('routing_key')
 
@@ -248,8 +253,9 @@ def have(self, account, chain_spec_dict):
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):
-    if not to_checksum_address(address):
+    if not is_address(address):
-        raise ValueError('invalid checksum address {}'.format(address))
+        raise ValueError('invalid address {}'.format(address))
+    address = tx_normalize.wallet_address(address)
     session = SessionBase.create_session()
     role = AccountRole.set(tag, address, session=session) 
     session.add(role)
@@ -298,13 +304,15 @@ def cache_gift_data(
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
     tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = Faucet.parse_give_to_request(tx['data'])
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx['to'])
 
     session = self.create_session()
 
     tx_dict = {
             'hash': tx['hash'],
-            'from': tx['from'],
+            'from': sender_address,
-            'to': tx['to'],
+            'to': recipient_address,
             'source_token': ZERO_ADDRESS_NORMAL,
             'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,
@@ -338,12 +346,14 @@ def cache_account_data(
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
     tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = AccountsIndex.parse_add_request(tx['data'])
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx['to'])
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx['hash'],
-            'from': tx['from'],
+            'from': sender_address,
-            'to': tx['to'],
+            'to': recipient_address,
             'source_token': ZERO_ADDRESS_NORMAL,
             'destination_token': ZERO_ADDRESS_NORMAL,
             'from_value': 0,

apps/cic-eth/cic_eth/eth/erc20.py

@@ -12,7 +12,10 @@ from chainlib.eth.tx import (
         )
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
 from chainqueue.sql.tx import cache_tx_dict
@@ -38,6 +41,7 @@ from cic_eth.task import (
         CriticalSQLAlchemyAndSignerTask,
     )
 from cic_eth.eth.nonce import CustodialTaskNonceOracle
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -62,7 +66,8 @@ def balance(tokens, holder_address, chain_spec_dict):
 
     for t in tokens:
         address = t['address']
-        token = ERC20Token(chain_spec, rpc, address)
+        logg.debug('address {} {}'.format(address, holder_address))
+        token = ERC20Token(chain_spec, rpc, add_0x(address))
         c = ERC20(chain_spec)
         o = c.balance_of(address, holder_address, sender_address=caller_address)
         r = rpc.do(o)
@@ -371,13 +376,15 @@ def cache_transfer_data(
     tx = unpack(tx_signed_raw_bytes, chain_spec)
 
     tx_data = ERC20.parse_transfer_request(tx['data'])
-    recipient_address = tx_data[0]
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
 
+
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
-            'from': tx['from'],
+            'from': sender_address,
             'to': recipient_address,
             'source_token': tx['to'],
             'destination_token': tx['to'],
@@ -448,13 +455,14 @@ def cache_approve_data(
     tx = unpack(tx_signed_raw_bytes, chain_spec)
 
     tx_data = ERC20.parse_approve_request(tx['data'])
-    recipient_address = tx_data[0]
+    sender_address = tx_normalize.wallet_address(tx['from'])
+    recipient_address = tx_normalize.wallet_address(tx_data[0])
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
     tx_dict = {
             'hash': tx_hash_hex,
-            'from': tx['from'],
+            'from': sender_address,
             'to': recipient_address,
             'source_token': tx['to'],
             'destination_token': tx['to'],

apps/cic-eth/cic_eth/eth/gas.py

@@ -9,7 +9,11 @@ from hexathon import (
         )
 #from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
-from chainlib.eth.address import is_checksum_address
+from chainlib.eth.address import (
+        is_checksum_address,
+        to_checksum_address,
+        is_address
+        )
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
 from chainqueue.sql.tx import cache_tx_dict
@@ -74,7 +78,6 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
-#def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 
@@ -179,8 +182,9 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
+    rpc_format_address = None
     if address != None:
-        if not is_checksum_address(address):
+        if not is_address(address):
             raise ValueError('invalid address {}'.format(address))
         address = tx_normalize.wallet_address(address)
         address = add_0x(address)
@@ -195,7 +199,6 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
         txs.append(tx)
 
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
 
     addresspass = None
     if len(txs) == 0:
@@ -211,13 +214,15 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
                 raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
             addresspass.append(address)
 
+    rpc_format_address = add_0x(to_checksum_address(address))
+
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
 
     gas_balance = 0
     try:
-        o = balance(address)
+        o = balance(rpc_format_address)
         r = conn.do(o)
         conn.disconnect()
         gas_balance = abi_decode_single(ABIContractType.UINT256, r)

apps/cic-eth/cic_eth/eth/meta.py

@@ -2,8 +2,9 @@
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.status import Status as TxStatus
 from cic_eth_registry.erc20 import ERC20Token
+from hexathon import add_0x
 
-# local imports
+# local impor:ts
 from cic_eth.ext.address import translate_address
 
 
@@ -44,8 +45,8 @@ class ExtendedTx:
             destination = source
         if destination_value == None:
             destination_value = source_value
-        st = ERC20Token(self.chain_spec, self.rpc, source)
+        st = ERC20Token(self.chain_spec, self.rpc, add_0x(source))
-        dt = ERC20Token(self.chain_spec, self.rpc, destination)
+        dt = ERC20Token(self.chain_spec, self.rpc, add_0x(destination))
         self.source_token = source
         self.source_token_symbol = st.symbol
         self.source_token_name = st.name

apps/cic-eth/cic_eth/eth/nonce.py

@@ -3,11 +3,12 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.address import is_checksum_address
+from chainlib.eth.address import is_checksum_address, is_address, strip_0x
 
 # local imports
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.nonce import (
         Nonce,
@@ -42,7 +43,8 @@ class CustodialTaskNonceOracle():
         :returns: Nonce
         :rtype: number
         """
-        r = NonceReservation.release(self.address, self.uuid, session=self.session)
+        address = tx_normalize.wallet_address(self.address)
+        r = NonceReservation.release(address, self.uuid, session=self.session)
         return r[1]
 
 
@@ -58,17 +60,18 @@ def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None):
         address = chained_input
         logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
     else:
-        if is_checksum_address(signer_address):
+        if is_address(signer_address):
             address = signer_address
             logg.debug('explicit address for reserve nonce {}'.format(signer_address))
         else:
             address = AccountRole.get_address(signer_address, session=session)
             logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
 
-    if not is_checksum_address(address):
+    if not is_address(address):
         raise ValueError('invalid result when resolving address for nonce {}'.format(address))
 
     root_id = self.request.root_id
+    address = tx_normalize.wallet_address(address)
     r = NonceReservation.next(address, root_id, session=session)
     logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
 

apps/cic-eth/cic_eth/ext/tx.py

@@ -32,6 +32,7 @@ from potaahto.symbols import snake_and_camel
 from cic_eth.queue.time import tx_times
 from cic_eth.task import BaseTask
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -134,7 +135,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                     tx_address = transfer_data[0]
                     tx_token_value = transfer_data[1]
                     
-                    if address == tx_address:
+                    if tx_normalize.wallet_address(address) == tx_normalize.wallet_address(tx_address):
                         status = StatusEnum.SENT
                         try:
                             o = receipt(tx['hash'])
@@ -152,8 +153,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
-                            'sender': tx['from'],
+                            'sender': tx_normalize.wallet_address(tx['from']),
-                            'recipient': tx_address,
+                            'recipient': tx_normalize.wallet_address(tx_address),
                             'source_value': tx_token_value,
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
@@ -164,10 +165,10 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             tx_r['date_created'] = times['queue']
                         else:
                             tx_r['date_created'] = times['network']
-                        txs[tx['hash']] = tx_r
+                        txs[strip_0x(tx['hash'])] = tx_r
                         break
-    return txs
 
+    return txs
 
 
 # TODO: Surely it must be possible to optimize this
@@ -230,6 +231,8 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr
             except UnknownContractError:
                 logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
                 continue
+        tx['recipient'] = tx_normalize.wallet_address(tx['recipient'])
+        tx['sender'] = tx_normalize.wallet_address(tx['sender'])
         txs.append(tx)
 
     return txs

apps/cic-eth/cic_eth/queue/query.py

@@ -58,6 +58,7 @@ def get_tx_local(chain_spec, tx_hash, session=None):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
+    address = tx_normalize.wallet_address(address)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     return get_account_tx_local(chain_spec, address, as_sender=as_sender, as_recipient=as_recipient, counterpart=counterpart)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -21,6 +21,7 @@ from erc20_faucet import Faucet
 # local imports
 from .base import SyncFilter
 from cic_eth.eth.meta import ExtendedTx
+from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger().getChild(__name__)
 
@@ -42,9 +43,9 @@ class CallbackFilter(SyncFilter):
             return (None, None)
         r = ERC20.parse_transfer_request(tx.payload)
         transfer_data = {}
-        transfer_data['to'] = r[0]
+        transfer_data['to'] = tx_normalize.wallet_address(r[0])
         transfer_data['value'] = r[1]
-        transfer_data['from'] = tx.outputs[0]
+        transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
         transfer_data['token_address'] = tx.inputs[0]
         return ('transfer', transfer_data)
 
@@ -54,8 +55,8 @@ class CallbackFilter(SyncFilter):
             return (None, None)
         r = ERC20.parse_transfer_from_request(tx.payload)
         transfer_data = {}
-        transfer_data['from'] = r[0]
+        transfer_data['from'] = tx_normalize.wallet_address(r[0])
-        transfer_data['to'] = r[1]
+        transfer_data['to'] = tx_normalize.wallet_address(r[1])
         transfer_data['value'] = r[2]
         transfer_data['token_address'] = tx.inputs[0]
         return ('transferfrom', transfer_data)
@@ -66,9 +67,9 @@ class CallbackFilter(SyncFilter):
             return (None, None)
         r = Faucet.parse_give_to_request(tx.payload)
         transfer_data = {}
-        transfer_data['to'] = r[0]
+        transfer_data['to'] = tx_normalize.wallet_address(r[0])
         transfer_data['value'] = tx.value
-        transfer_data['from'] = tx.outputs[0]
+        transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -12,7 +12,8 @@ from hexathon import (
 # local imports
 from .base import SyncFilter
 
-logg = logging.getLogger(__name__)
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -17,6 +17,7 @@ from cic_eth_registry import CICRegistry
 from erc20_transfer_authorization import TransferAuthorization
 
 # local imports
+from cic_eth.encode import tx_normalize
 from .base import SyncFilter
 
 
@@ -52,9 +53,9 @@ class TransferAuthFilter(SyncFilter):
 
         r = TransferAuthorization.parse_create_request_request(tx.payload) 
           
-        sender = r[0]
+        sender = tx_normalize.wallet_address(r[0])
-        recipient = r[1]
+        recipient = tx_normalize.wallet_address(r[1])
-        token = r[2]
+        token = tx_normalize.executable_address(r[2])
         value = r[3]
 
         token_data = {

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -203,13 +203,10 @@ def main():
     argv.append(config.get('CELERY_QUEUE'))
     argv.append('-n')
     argv.append(config.get('CELERY_QUEUE'))
-    argv.append('--config')
-    argv.append('celeryconfig')
-
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address)
+    default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
     default_token.load(conn)
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name

apps/cic-eth/crypto_dev_signer_config/config.ini

@@ -0,0 +1,4 @@
+[signer]
+secret = deadbeef
+#database = crypto-dev-signer
+socket_path = ipc:///run/crypto-dev-signer/jsonrpc.ipc

apps/cic-eth/crypto_dev_signer_config/database.ini

@@ -0,0 +1,8 @@
+[database]
+NAME=cic-signer
+USER=
+PASSWORD=
+HOST=
+PORT=
+ENGINE=
+DRIVER=

apps/cic-eth/docker/entrypoints/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/cic-eth/docker/entrypoints/start_dispatcher.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 
+set -e
+set -u
+
 . ./db.sh
 
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
   /usr/local/bin/cic-eth-dispatcherd $@
+else
+  /usr/local/bin/cic-eth-dispatcherd $@
+fi

apps/cic-eth/docker/entrypoints/start_retry.sh

@@ -1,5 +1,18 @@
 #!/bin/bash
 
+set -e
+set -u
+
 . ./db.sh
 
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
   /usr/local/bin/cic-eth-retrierd $@
+else
+  /usr/local/bin/cic-eth-retrierd $@
+fi

apps/cic-eth/docker/entrypoints/start_tasker.sh

@@ -1,31 +1,18 @@
 #!/bin/bash
 
 set -e
+set -u
+
 . ./db.sh
 
-# set CONFINI_ENV_PREFIX to override the env prefix to override env vars
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
 
-#echo "!!! starting signer"
-#python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
 
-echo "!!! starting taskerd"
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
   /usr/local/bin/cic-eth-taskerd $@
-
+else
-# thanks! https://docs.docker.com/config/containers/multi-service_container/
+  /usr/local/bin/cic-eth-taskerd $@
-#sleep 1;
+fi
-#echo "!!! entering monitor loop"
-#while true; do
-#  ps aux | grep crypto-dev-daemon | grep -q -v grep
-#  PROCESS_1_STATUS=$?
-#  ps aux | grep cic-eth-tasker |grep -q -v grep
-#  PROCESS_2_STATUS=$?
-#  # If the greps above find anything, they exit with 0 status
-#  # If they are not both 0, then something is wrong
-#  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
-#    echo "One of the processes has already exited."
-#    exit 1
-#  fi
-#  sleep 15;
-#done
-#
-set +e

apps/cic-eth/docker/entrypoints/start_tracker.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 
+set -e
+set -u
+
 . ./db.sh
 
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
   /usr/local/bin/cic-eth-trackerd $@
+else
+  /usr/local/bin/cic-eth-trackerd $@
+fi

apps/cic-eth/docker/entrypoints/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-eth/requirements.txt

@@ -1,3 +1,4 @@
 celery==4.4.7
-chainlib-eth>=0.0.9a14,<0.1.0
+chainlib-eth>=0.0.9rc2,<0.1.0
 semver==2.13.0
+crypto-dev-signer>=0.4.15rc2,<0.5.0

apps/cic-eth/services_requirements.txt

@@ -8,7 +8,7 @@ pycryptodome==3.10.1
 liveness~=0.0.1a7
 eth-address-index>=0.2.3a4,<0.3.0
 eth-accounts-index>=0.1.2a3,<0.2.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a3,<0.7.0
 erc20-faucet>=0.3.2a2,<0.4.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -18,7 +18,10 @@ from eth_erc20 import ERC20
 from sarafu_faucet import MinterFaucet
 from eth_accounts_index.registry import AccountRegistry
 from potaahto.symbols import snake_and_camel
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
 from cic_eth.runnable.daemons.filters.callback import CallbackFilter
@@ -160,7 +163,7 @@ def test_faucet_gift_to_tx(
     assert transfer_data['token_address'] == foo_token
 
 
-def test_callback_filter(
+def test_callback_filter_filter(
         default_chain_spec,
         init_database,
         eth_rpc,
@@ -213,6 +216,7 @@ def test_callback_filter(
 
         def call_back(self, transfer_type, result):
             self.results[transfer_type] = result
+            logg.debug('result {}'.format(result))
             return self
 
     mock = CallbackMock()
@@ -221,4 +225,4 @@ def test_callback_filter(
     fltr.filter(eth_rpc, mockblock, tx, init_database)
 
     assert mock.results.get('transfer') != None
-    assert mock.results['transfer']['destination_token'] == foo_token
+    assert mock.results['transfer']['destination_token'] == strip_0x(foo_token)

apps/cic-eth/tests/filters/test_register_filter.py

@@ -17,6 +17,9 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
+from chainlib.eth.address import (
+        to_checksum_address,
+        )
 from erc20_faucet import Faucet
 from hexathon import (
         strip_0x,
@@ -25,7 +28,6 @@ from hexathon import (
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
-from cic_eth.encode import tx_normalize
 from cic_eth.queue.query import get_account_tx_local
 
 logg = logging.getLogger()
@@ -70,12 +72,13 @@ def test_register_filter(
     tx = Tx(tx_src, block=block, rcpt=rcpt)
     tx.apply_receipt(rcpt)
 
-    fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None)
+    fltr = RegistrationFilter(default_chain_spec, to_checksum_address(os.urandom(20).hex()), queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
     assert t == None
 
-    fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None)
+    fltr = RegistrationFilter(default_chain_spec, to_checksum_address(account_registry), queue=None)
     t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    logg.debug('t {}'.format(t))
 
     t.get_leaf()
     assert t.successful()
@@ -89,4 +92,4 @@ def test_register_filter(
     gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
 
     gift = Faucet.parse_give_to_request(gift_tx['data'])
-    assert gift[0] == agent_roles['ALICE']
+    assert add_0x(gift[0]) == agent_roles['ALICE']

apps/cic-eth/tests/filters/test_transferauth_filter.py

@@ -19,6 +19,7 @@ from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+from cic_eth.encode import tx_normalize
 
 
 def test_filter_transferauth(
@@ -66,7 +67,8 @@ def test_filter_transferauth(
     t.get_leaf()
     assert t.successful()
 
-    approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    #approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    approve_txs = get_account_tx(default_chain_spec.asdict(), tx_normalize.wallet_address(agent_roles['ALICE']), as_sender=True, session=init_database)
     ks = list(approve_txs.keys())
     assert len(ks) == 1
 
@@ -76,4 +78,4 @@ def test_filter_transferauth(
 
     c = ERC20(default_chain_spec)
     approve = c.parse_approve_request(approve_tx['data']) 
-    assert approve[0] == agent_roles['BOB']
+    assert approve[0] == strip_0x(agent_roles['BOB'])

apps/cic-eth/tests/task/api/test_admin.py

@@ -110,8 +110,8 @@ def test_tag_account(
     t = api.tag_account('bar', agent_roles['CAROL'], default_chain_spec)
     t.get()
 
-    assert AccountRole.get_address('foo', init_database) == agent_roles['ALICE']
+    assert AccountRole.get_address('foo', init_database) == tx_normalize.wallet_address(agent_roles['ALICE'])
-    assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
+    assert AccountRole.get_address('bar', init_database) == tx_normalize.wallet_address(agent_roles['CAROL'])
 
 
 def test_tx(

apps/cic-eth/tests/task/api/test_app.py

@@ -10,6 +10,7 @@ from cic_eth_registry.erc20 import ERC20Token
 from chainlib.chain import ChainSpec
 from eth_accounts_index import AccountsIndex
 from chainlib.eth.tx import (
+        receipt,
         transaction,
         )
 from chainqueue.sql.state import (
@@ -29,6 +30,7 @@ def test_account_api(
         init_database,
         init_eth_rpc,
         account_registry,
+        cic_registry,
         custodial_roles,
         celery_session_worker,
         ):
@@ -49,6 +51,7 @@ def test_account_api_register(
         eth_rpc,
         celery_session_worker,
         ):
+
     api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
     t = api.create_account('')
     register_tx_hash = t.get_leaf()
@@ -69,12 +72,18 @@ def test_account_api_register(
     r = t.get_leaf()
     assert t.successful()
 
+    o = receipt(register_tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
     o = transaction(register_tx_hash)
     tx_src = eth_rpc.do(o)
 
     c = AccountsIndex(default_chain_spec)
     address = c.parse_add_request(tx_src['data'])
+    logg.debug('address {} '.format(address))
     o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
+    logg.debug('o {}'.format(o))
     r = eth_rpc.do(o)
     assert c.parse_have(r)
 

apps/cic-eth/tests/task/api/test_balance.py

@@ -3,18 +3,22 @@ import os
 import logging
 
 # external imports
+import pytest
 from chainlib.eth.address import to_checksum_address
+from hexathon import add_0x
 
 # local imports
 from cic_eth.api.api_task import Api
 
 logg = logging.getLogger()
 
+
 def test_balance_simple_api(
         default_chain_spec,
         init_database,
         cic_registry,
         foo_token,
+        register_lookups,
         register_tokens,
         api,
         celery_session_worker,
@@ -22,7 +26,7 @@ def test_balance_simple_api(
 
     chain_str = str(default_chain_spec)
 
-    a = to_checksum_address('0x' + os.urandom(20).hex())
+    a = add_0x(to_checksum_address(os.urandom(20).hex()))
     t = api.balance(a, 'FOO', include_pending=False)
     r = t.get_leaf()
     assert t.successful()
@@ -36,6 +40,7 @@ def test_balance_complex_api(
         init_database,
         cic_registry,
         foo_token,
+        register_lookups,
         register_tokens,
         api,
         celery_session_worker,
@@ -43,7 +48,7 @@ def test_balance_complex_api(
 
     chain_str = str(default_chain_spec)
 
-    a = to_checksum_address('0x' + os.urandom(20).hex())
+    a = add_0x(to_checksum_address(os.urandom(20).hex()))
     t = api.balance(a, 'FOO', include_pending=True)
     r = t.get_leaf()
     assert t.successful()

apps/cic-eth/tests/task/api/test_list.py

@@ -6,6 +6,7 @@ import pytest
 from chainlib.eth.nonce import RPCNonceOracle
 from eth_erc20 import ERC20
 from chainlib.eth.tx import receipt
+from hexathon import strip_0x
 
 # local imports
 from cic_eth.api.api_task import Api
@@ -23,7 +24,6 @@ from cic_eth.pytest.mock.filter import (
 logg = logging.getLogger()
 
 
-@pytest.mark.xfail()
 def test_list_tx(
         default_chain_spec,
         init_database,
@@ -34,8 +34,10 @@ def test_list_tx(
         agent_roles,
         foo_token,
         register_tokens,
+        register_lookups,
         init_eth_tester,
         celery_session_worker,
+        init_celery_tasks,
         ):
 
     tx_hashes = []
@@ -63,13 +65,16 @@ def test_list_tx(
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
     assert r['status'] == 1
+
     a = r['block_number']
-    block_filter.add(a.to_bytes(4, 'big'))
+    ab = a.to_bytes(4, 'big')
+    block_filter.add(ab)
 
-    a = r['block_number'] + r['transaction_index']
+    bb = r['transaction_index'].to_bytes(4, 'big')
-    tx_filter.add(a.to_bytes(4, 'big'))
+    cb = ab + bb
+    tx_filter.add(cb)
 
-    tx_hashes.append(tx_hash_hex)
+    tx_hashes.append(strip_0x(tx_hash_hex))
 
     # external tx two
     Nonce.next(agent_roles['ALICE'], 'foo', session=init_database)
@@ -83,26 +88,29 @@ def test_list_tx(
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
     assert r['status'] == 1
+
     a = r['block_number']
-    block_filter.add(a.to_bytes(4, 'big'))
+    ab = a.to_bytes(4, 'big')
+    block_filter.add(ab)
 
-    a = r['block_number'] + r['transaction_index']
+    bb = r['transaction_index'].to_bytes(4, 'big')
-    tx_filter.add(a.to_bytes(4, 'big'))
+    cb = ab + bb
+    tx_filter.add(cb)
 
-    tx_hashes.append(tx_hash_hex)
+    tx_hashes.append(strip_0x(tx_hash_hex))
 
     init_eth_tester.mine_blocks(28)
 
     # custodial tx 1
     api = Api(str(default_chain_spec), queue=None)
-    t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO') #, 'blinky')
+    t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO')
     r = t.get_leaf()
     assert t.successful()
     tx_hashes.append(r)
 
     # custodial tx 2
     api = Api(str(default_chain_spec), queue=None)
-    t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO') #, 'blinky')
+    t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO')
     r = t.get_leaf()
     assert t.successful()
     tx_hashes.append(r)
@@ -117,7 +125,8 @@ def test_list_tx(
     assert len(r) == 3
     logg.debug('rrrr {}'.format(r))
 
+    logg.debug('testing against hashes {}'.format(tx_hashes))
     for tx in r:
         logg.debug('have tx {}'.format(tx))
-        tx_hashes.remove(tx['hash'])
+        tx_hashes.remove(strip_0x(tx['hash']))
     assert len(tx_hashes) == 1

apps/cic-eth/tests/task/test_ext_tx.py

@@ -10,6 +10,7 @@ from chainlib.eth.tx import (
         )
 from eth_erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
+from hexathon import add_0x 
 
 # local imports
 from cic_eth.db.models.nonce import (
@@ -91,5 +92,5 @@ def test_filter_process(
 
     assert len(r) == 2
     for tx_hash in r.keys():
-        tx_hashes.remove(tx_hash)
+        tx_hashes.remove(add_0x(tx_hash))
     assert len(tx_hashes) == 0

apps/cic-eth/tools_requirements.txt

@@ -1,6 +1,6 @@
 crypto-dev-signer>=0.4.15a7,<=0.4.15
 chainqueue>=0.0.5a1,<0.1.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a3,<0.7.0
 redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1

apps/cic-ussd/cic_ussd/account/balance.py

@@ -101,7 +101,7 @@ def get_cached_available_balance(blockchain_address: str) -> float:
     :return: Operational balance of an account.
     :rtype: float
     """
-    identifier = bytes.fromhex(blockchain_address[2:])
+    identifier = bytes.fromhex(blockchain_address)
     key = cache_data_key(identifier, salt=':cic.balances')
     cached_balances = get_cached_data(key=key)
     if cached_balances:

apps/cic-ussd/cic_ussd/account/metadata.py

@@ -4,7 +4,6 @@ import logging
 from typing import Optional
 
 # external imports
-from chainlib.hash import strip_0x
 from cic_types.models.person import Person
 
 # local imports
@@ -20,7 +19,7 @@ def get_cached_preferred_language(blockchain_address: str) -> Optional[str]:
     :return: Account's set preferred language | Fallback preferred language.
     :rtype: str
     """
-    identifier = bytes.fromhex(strip_0x(blockchain_address))
+    identifier = bytes.fromhex(blockchain_address)
     preferences_metadata_handler = PreferencesMetadata(identifier)
     cached_preferences_metadata = preferences_metadata_handler.get_cached_metadata()
     if cached_preferences_metadata:

apps/cic-ussd/cic_ussd/account/statement.py

@@ -86,7 +86,7 @@ def query_statement(blockchain_address: str, limit: int = 9):
     :param limit: Number of transactions to be returned.
     :type limit: int
     """
-    logg.debug(f'retrieving balance for address: {blockchain_address}')
+    logg.debug(f'retrieving statement for address: {blockchain_address}')
     chain_str = Chain.spec.__str__()
     cic_eth_api = Api(
         chain_str=chain_str,

apps/cic-ussd/cic_ussd/db/models/account.py

@@ -2,7 +2,6 @@
 import json
 
 # external imports
-from chainlib.hash import strip_0x
 from cic_eth.api import Api
 
 # local imports
@@ -101,7 +100,7 @@ class Account(SessionBase):
         session.add(self)
         session.flush()
         SessionBase.release_session(session=session)
-        return f'Pin reset successful.'
+        return 'Pin reset successful.'
 
     def standard_metadata_id(self) -> str:
         """This function creates an account's standard metadata identification information that contains an account owner's
@@ -109,7 +108,7 @@ class Account(SessionBase):
         :return: Standard metadata identification information | e164 formatted phone number.
         :rtype: str
         """
-        identifier = bytes.fromhex(strip_0x(self.blockchain_address))
+        identifier = bytes.fromhex(self.blockchain_address)
         key = cache_data_key(identifier, ':cic.person')
         account_metadata = get_cached_data(key)
         if not account_metadata:

apps/cic-ussd/cic_ussd/processor/menu.py

@@ -37,7 +37,7 @@ class MenuProcessor:
     def __init__(self, account: Account, display_key: str, menu_name: str, session: Session, ussd_session: dict):
         self.account = account
         self.display_key = display_key
-        self.identifier = bytes.fromhex(self.account.blockchain_address[2:])
+        self.identifier = bytes.fromhex(self.account.blockchain_address)
         self.menu_name = menu_name
         self.session = session
         self.ussd_session = ussd_session

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -5,7 +5,6 @@ from datetime import timedelta
 
 # third-party imports
 import celery
-from chainlib.hash import strip_0x
 
 # local imports
 from cic_ussd.account.balance import get_balances, calculate_available_balance
@@ -55,6 +54,7 @@ def account_creation_callback(self, result: str, url: str, status_code: int):
     session.add(account)
     session.commit()
     session.close()
+    logg.debug(f'recorded account with identifier: {result}')
 
     queue = self.request.delivery_info.get('routing_key')
     s_phone_pointer = celery.signature(
@@ -86,7 +86,7 @@ def balances_callback(result: list, param: str, status_code: int):
         raise ValueError(f'Unexpected status code: {status_code}.')
 
     balances = result[0]
-    identifier = bytes.fromhex(strip_0x(param))
+    identifier = bytes.fromhex(param)
     key = cache_data_key(identifier, ':cic.balances')
     cache_data(key, json.dumps(balances))
 
@@ -113,8 +113,10 @@ def statement_callback(self, result, param: str, status_code: int):
     for transaction in statement_transactions:
         recipient_transaction, sender_transaction = transaction_actors(transaction)
         if recipient_transaction.get('blockchain_address') == param:
+            recipient_transaction['alt_blockchain_address'] = sender_transaction.get('blockchain_address')
             generate(param, queue, recipient_transaction)
         if sender_transaction.get('blockchain_address') == param:
+            sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address')
             generate(param, queue, sender_transaction)
 
 

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -3,7 +3,6 @@ import logging
 
 # third-party imports
 import celery
-from hexathon import strip_0x
 
 # local imports
 from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
@@ -21,7 +20,7 @@ def query_person_metadata(blockchain_address: str):
     :return:
     :rtype:
     """
-    identifier = bytes.fromhex(strip_0x(blockchain_address))
+    identifier = bytes.fromhex(blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.query()
 
@@ -36,14 +35,14 @@ def create_person_metadata(blockchain_address: str, data: dict):
     :return:
     :rtype:
     """
-    identifier = bytes.fromhex(strip_0x(blockchain_address))
+    identifier = bytes.fromhex(blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.create(data=data)
 
 
 @celery_app.task
 def edit_person_metadata(blockchain_address: str, data: dict):
-    identifier = bytes.fromhex(strip_0x(blockchain_address))
+    identifier = bytes.fromhex(blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.edit(data=data)
 
@@ -51,21 +50,21 @@ def edit_person_metadata(blockchain_address: str, data: dict):
 @celery_app.task(bind=True, base=CriticalMetadataTask)
 def add_phone_pointer(self, blockchain_address: str, phone_number: str):
     identifier = phone_number.encode('utf-8')
-    stripped_address = strip_0x(blockchain_address)
+    stripped_address = blockchain_address
     phone_metadata_client = PhonePointerMetadata(identifier=identifier)
     phone_metadata_client.create(data=stripped_address)
 
 
 @celery_app.task()
 def add_custom_metadata(blockchain_address: str, data: dict):
-    identifier = bytes.fromhex(strip_0x(blockchain_address))
+    identifier = bytes.fromhex(blockchain_address)
     custom_metadata_client = CustomMetadata(identifier=identifier)
     custom_metadata_client.create(data=data)
 
 
 @celery_app.task()
 def add_preferences_metadata(blockchain_address: str, data: dict):
-    identifier = bytes.fromhex(strip_0x(blockchain_address))
+    identifier = bytes.fromhex(blockchain_address)
     preferences_metadata_client = PreferencesMetadata(identifier=identifier)
     preferences_metadata_client.create(data=data)
 
@@ -76,7 +75,7 @@ def query_preferences_metadata(blockchain_address: str):
     :param blockchain_address: Blockchain address of an account.
     :type blockchain_address: str | Ox-hex
     """
-    identifier = bytes.fromhex(strip_0x(blockchain_address))
+    identifier = bytes.fromhex(blockchain_address)
     logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
     person_metadata_client = PreferencesMetadata(identifier=identifier)
     return person_metadata_client.query()

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -5,7 +5,6 @@ import logging
 # third-party imports
 import celery
 import i18n
-from chainlib.hash import strip_0x
 
 # local imports
 from cic_ussd.account.metadata import get_cached_preferred_language
@@ -24,17 +23,13 @@ logg = logging.getLogger(__file__)
 def generate_statement(self, querying_party: str, transaction: dict):
     """"""
     queue = self.request.delivery_info.get('routing_key')
-
-    s_preferences = celery.signature(
-        'cic_ussd.tasks.metadata.query_preferences_metadata', [querying_party], queue=queue
-    )
     s_parse_transaction = celery.signature(
         'cic_ussd.tasks.processor.parse_transaction', [transaction], queue=queue
     )
     s_cache_statement = celery.signature(
         'cic_ussd.tasks.processor.cache_statement', [querying_party], queue=queue
     )
-    celery.chain(s_preferences, s_parse_transaction, s_cache_statement).apply_async()
+    celery.chain(s_parse_transaction, s_cache_statement).apply_async()
 
 
 @celery_app.task
@@ -53,7 +48,7 @@ def cache_statement(parsed_transaction: dict, querying_party: str):
         statement_transactions = json.loads(cached_statement)
     statement_transactions.append(parsed_transaction)
     data = json.dumps(statement_transactions)
-    identifier = bytes.fromhex(strip_0x(querying_party))
+    identifier = bytes.fromhex(querying_party)
     key = cache_data_key(identifier, ':cic.statement')
     cache_data(key, data)
 

apps/cic-ussd/docker/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/cic-ussd/docker/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-ussd/tests/cic_ussd/account/test_statement.py

@@ -28,7 +28,6 @@ def test_filter_statement_transactions(transactions_list):
 
 def test_generate(activated_account,
                   cache_default_token_data,
-                  cache_statement,
                   cache_preferences,
                   celery_session_worker,
                   init_cache,
@@ -37,18 +36,18 @@ def test_generate(activated_account,
                   preferences,
                   preferences_metadata_url,
                   transactions_list):
-    with requests_mock.Mocker(real_http=False) as request_mocker:
-        request_mocker.register_uri('GET', preferences_metadata_url, status_code=200, reason='OK', json=preferences)
     statement_transactions = filter_statement_transactions(transactions_list)
     for transaction in statement_transactions:
         querying_party = activated_account.blockchain_address
         recipient_transaction, sender_transaction = transaction_actors(transaction)
         if recipient_transaction.get('blockchain_address') == querying_party:
+            recipient_transaction['alt_blockchain_address'] = sender_transaction.get('blockchain_address')
             generate(querying_party, None, recipient_transaction)
         if sender_transaction.get('blockchain_address') == querying_party:
+            sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address')
             generate(querying_party, None, sender_transaction)
     time.sleep(2)
-        identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     key = cache_data_key(identifier, ':cic.statement')
     statement = get_cached_data(key)
     statement = json.loads(statement)

apps/cic-ussd/tests/cic_ussd/processor/test_menu.py

@@ -57,7 +57,7 @@ def test_menu_processor(activated_account,
                                    available_balance=available_balance,
                                    token_symbol=token_symbol)
 
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     key = cache_data_key(identifier, ':cic.adjusted_balance')
     adjusted_balance = 45931650.64654012
     cache_data(key, json.dumps(adjusted_balance))
@@ -108,7 +108,7 @@ def test_menu_processor(activated_account,
     display_key = 'ussd.kenya.display_user_metadata'
     ussd_menu = UssdMenu.find_by_name('display_user_metadata')
     name = ussd_menu.get('name')
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     person_metadata = PersonMetadata(identifier)
     cached_person_metadata = person_metadata.get_cached_metadata()
     resp = response(activated_account, display_key, name, init_database, generic_ussd_session)

apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py

@@ -114,6 +114,7 @@ def test_statement_callback(activated_account, mocker, transactions_list):
     s_statement_callback.apply_async().get()
     statement_transactions = filter_statement_transactions(transactions_list)
     recipient_transaction, sender_transaction = transaction_actors(statement_transactions[0])
+    sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address')
     mock_statement_generate.assert_called_with(
         (activated_account.blockchain_address, sender_transaction), {}, queue='cic-ussd')
 

apps/cic-ussd/tests/fixtures/account.py

@@ -54,7 +54,7 @@ def cache_account_creation_data(init_cache, account_creation_data):
 
 @pytest.fixture(scope='function')
 def cache_balances(activated_account, balances, init_cache):
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     balances = json.dumps(balances[0])
     key = cache_data_key(identifier, ':cic.balances')
     cache_data(key, balances)
@@ -70,7 +70,7 @@ def cache_default_token_data(default_token_data, init_cache, load_chain_spec):
 
 @pytest.fixture(scope='function')
 def cache_person_metadata(activated_account, init_cache, person_metadata):
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     person = json.dumps(person_metadata)
     key = cache_data_key(identifier, ':cic.person')
     cache_data(key, person)
@@ -78,7 +78,7 @@ def cache_person_metadata(activated_account, init_cache, person_metadata):
 
 @pytest.fixture(scope='function')
 def cache_preferences(activated_account, init_cache, preferences):
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     preferences = json.dumps(preferences)
     key = cache_data_key(identifier, ':cic.preferences')
     cache_data(key, preferences)
@@ -86,10 +86,10 @@ def cache_preferences(activated_account, init_cache, preferences):
 
 @pytest.fixture(scope='function')
 def cache_statement(activated_account, init_cache, statement):
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
-    preferences = json.dumps(statement)
+    statement = json.dumps(statement)
     key = cache_data_key(identifier, ':cic.statement')
-    cache_data(key, preferences)
+    cache_data(key, statement)
 
 
 @pytest.fixture(scope='function')

apps/cic-ussd/tests/fixtures/transaction.py

@@ -7,6 +7,7 @@ import pytest
 # local import
 from cic_ussd.account.balance import get_cached_available_balance
 
+
 # tests imports
 
 
@@ -103,8 +104,8 @@ def transactions_list(activated_account, valid_recipient):
             'destination_token': '0x0000000000000000000000000000000000000000',
             'block_number': 80,
             'tx_index': 0,
-            'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
+            'sender': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
-            'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A',
+            'recipient': '103d1ed6e370dBa6267045c70d4999384c18a04A',
             'from_value': 0,
             'to_value': 0,
             'date_created': '2021-07-14T14:13:46.036198',
@@ -122,8 +123,8 @@ def transactions_list(activated_account, valid_recipient):
             'destination_token': '0x0000000000000000000000000000000000000000',
             'block_number': 78,
             'tx_index': 0,
-            'sender': '0xb41BfEE260693A473254D62b81aE1ADCC9E51AFb',
+            'sender': 'b41BfEE260693A473254D62b81aE1ADCC9E51AFb',
-            'recipient': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
+            'recipient': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
             'from_value': 1800000000000000,
             'to_value': 1800000000000000,
             'date_created': '2021-07-14T14:13:35.839638',
@@ -142,8 +143,8 @@ def transactions_list(activated_account, valid_recipient):
             'destination_token': '0x0000000000000000000000000000000000000000',
             'block_number': 79,
             'tx_index': 0,
-            'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
+            'sender': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8',
-            'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A',
+            'recipient': '103d1ed6e370dBa6267045c70d4999384c18a04A',
             'from_value': 0,
             'to_value': 0,
             'date_created': '2021-07-14T14:13:35.638355',
@@ -152,4 +153,3 @@ def transactions_list(activated_account, valid_recipient):
             'timestamp': 1626272015,
             'hash': '0x32ca3dd3bef06463b452f4d32f5f563d083cb4759219eed90f3d2a9c1791c5fc'}
     ]
-

apps/cic-ussd/tests/helpers/accounts.py

@@ -4,7 +4,6 @@ import random
 import uuid
 
 # external imports
-from chainlib.eth.address import to_checksum_address
 from faker import Faker
 from faker_e164.providers import E164Provider
 
@@ -21,7 +20,7 @@ def phone_number() -> str:
 
 
 def blockchain_address() -> str:
-    return to_checksum_address('0x' + os.urandom(20).hex())
+    return os.urandom(20).hex().lower()
 
 
 def session_id() -> str:

apps/contract-migration/docker/init_readyz.sh

@@ -0,0 +1,25 @@
+#! /bin/bash
+
+set -e 
+set -a
+
+mkdir -p $DEV_DATA_DIR/health
+
+source $DEV_DATA_DIR/env_reset
+
+jq --arg CIC_REGISTRY_ADDRESS "$CIC_REGISTRY_ADDRESS" \
+   --arg CIC_TRUST_ADDRESS "$CIC_TRUST_ADDRESS" \
+   --arg CIC_DEFAULT_TOKEN_SYMBOL "$CIC_DEFAULT_TOKEN_SYMBOL"\
+   --arg TOKEN_NAME "$TOKEN_NAME"\
+   --arg RUN_MASK "$RUN_MASK" \
+   -n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS, 
+        "CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS,
+        "CIC_DEFAULT_TOKEN_SYMBOL": $CIC_DEFAULT_TOKEN_SYMBOL,
+        "TOKEN_NAME": $TOKEN_NAME,
+        "RUN_MASK": $RUN_MASK}' > $DEV_DATA_DIR/health/readyz
+
+cd $DEV_DATA_DIR/health
+
+echo "starting health endpoint on :8000/readyz"
+python -m http.server 8000 &> /dev/null 
+

apps/contract-migration/env_reset

@@ -0,0 +1,16 @@
+export DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export DEV_ETH_ACCOUNT_RESERVE_MINTER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export DEV_RESERVE_AMOUNT=10000000000000000000000000000000000
+export DEV_FAUCET_AMOUNT=50000000
+export DEV_GAS_AMOUNT=100000000000000000000000
+export DEV_TOKEN_AMOUNT=100000000000000000000000
+export DEV_ETH_GAS_PRICE=
+export DEV_DATA_DIR=.
+export DEV_PIP_EXTRA_INDEX_URL=https://pip.grassrootseconomics.net:8433
+export DEV_ETH_PROVIDER_HOST=
+export DEV_ETH_PROVIDER_PORT=
+export CIC_TRUST_ADDRESS=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export CIC_DEFAULT_TOKEN_SYMBOL=GFT
+export WALLET_KEY_FILE=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+

apps/contract-migration/reset.sh

@@ -112,10 +112,16 @@ export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
 
 echo "Writing env_reset file ..."
 
-echo "export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
+# note these are also written in init_readyz.sh
+cat <<EOF > $DEV_DATA_DIR/env_reset 
+export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
+export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS
 export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
 export TOKEN_NAME=$TOKEN_NAME
-" >> "${DEV_DATA_DIR}"/env_reset
+EOF
+
+confini-dump -vv --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset
+confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config 
 
 set +a
 set +e

apps/contract-migration/run_job.sh

@@ -12,6 +12,10 @@ if [[ $((RUN_MASK & 1)) -eq 1 ]]
 then
 	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 1 - contract deployment"
 	./reset.sh
+  # cic-eth-xxx services rely on
+  export RUN_MASK_PHASE=1
+  ./docker/init_readyz.sh &
+  echo 0
 	if [ $? -ne "0" ]; then
 		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 1 - contract deployment"
 		exit 1;
@@ -29,3 +33,5 @@ then
 	fi
 	>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 2 - custodial service initialization"
 fi
+# this will leave the container up and serving the results of the migration
+wait

apps/contract-migration/seed_cic_eth.sh

@@ -82,7 +82,7 @@ cic-eth-ctl -i $CHAIN_SPEC unlock INIT
 cic-eth-ctl -i $CHAIN_SPEC unlock SEND
 cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE
 
-#confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config
+confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config 
 
 set +a
 set +e

apps/data-seeding/cic_eth/paths.py

@@ -7,7 +7,6 @@ import json
 # external imports
 import celery
 import confini
-import celery.utils.graph
 
 # local imports
 from cic_eth.api import Api
@@ -36,7 +35,7 @@ class Fmtr(celery.utils.graph.GraphFormatter):
 
 def main():
     api = Api(
-        config.get('CHAIN_SPEC'),
+        config.get('CIC_CHAIN_SPEC'),
         queue='cic-eth',
         #callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
         #callback_task='cic_eth.callbacks.redis.redis',

apps/data-seeding/scripts/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/data-seeding/scripts/run_job.sh

@@ -0,0 +1,15 @@
+#! /bin/bash 
+set -e
+set -u
+env
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./scripts/wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./scripts/get_readyz.sh # set env vars form endpoint
+  ./import_ussd.sh
+else
+  ./import_ussd.sh
+fi
+

apps/data-seeding/scripts/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

docker-compose.yml

@@ -60,6 +60,8 @@ services:
 
   contract-migration:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest}
+    ports:
+      - 8012:8000
     build:
       context: apps/contract-migration
       dockerfile: docker/Dockerfile
@@ -68,7 +70,6 @@ services:
         pip_extra_args: $PIP_EXTRA_ARGS
         EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
         EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
       RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
       ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
@@ -93,9 +94,14 @@ services:
       RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
       DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-50000000}
       DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE
+      #DEV_GAS_AMOUNT: "100000000000000000000000" # removed from env_reset
+      #WALLET_KEY_FILE: ./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c #removed from env_reset 
       TOKEN_NAME: ${TOKEN_NAME:-Giftable Token}
       TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
       TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token}
+      #TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token}
+      #TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET}
+      #TOKEN_TYPE: ${TOKEN_TYPE:-erc20_demurrage_token }
       TOKEN_DECIMALS: $TOKEN_DECIMALS
       TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD
       TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
@@ -103,9 +109,8 @@ services:
       TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL
       TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
       SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
-    restart: on-failure 
     command: ["./run_job.sh"]
-        #command: ["./reset.sh"]
+    #command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
     depends_on:
       - eth
       - postgres
@@ -113,7 +118,7 @@ services:
       - cic-eth-tasker
     volumes:
       - contract-config:/tmp/cic/config
-
+      - ./apps/contract-migration:/root
 
   data-seeding:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:${TAG:-latest}
@@ -153,9 +158,11 @@ services:
       USSD_PROVIDER: http://cic-user-ussd-server:9000
       CELERY_QUEUE: cic-import-ussd
       EXCLUSIONS: ussd
-    command: bash import_ussd.sh
+      CONTRACT_MIGRATION_URL: contract-migration:8000
+    command: bash ./scripts/run_job.sh 
     volumes:
       - contract-config:/tmp/cic/config/:ro
+        #- ./apps/data-seeding:/root
 
   cic-cache-tracker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
@@ -181,17 +188,13 @@ services:
       CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
       CELERY_BROKER_URL: redis://redis:6379
       CELERY_RESULT_URL: redis://redis:6379
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - redis
       - postgres
       - eth
-    command:
+    command: ./start_tracker.sh -c /usr/local/etc/cic-cache -vv
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_tracker.sh -c /usr/local/etc/cic-cache -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -218,17 +221,13 @@ services:
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CELERY_BROKER_URL: redis://redis:6379
       CELERY_RESULT_URL: redis://redis:6379
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: unless-stopped 
     depends_on:
       - redis
       - postgres
       - eth
-    command:
+    command: bash docker/start_tasker.sh
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        /usr/local/bin/cic-cache-taskerd -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -243,10 +242,10 @@ services:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
-        #DATABASE_PASSWORD: ${DATABASE_PASSWORD:-
+      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
       DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
       DATABASE_DEBUG: 1
-        #PGPASSWORD: $DATABASE_PASSWORD
+      CONTRACT_MIGRATION_URL: contract-migration:8000
       SERVER_PORT: 8000
     restart: on-failure 
     ports:
@@ -255,16 +254,7 @@ services:
       - postgres
       - cic-cache-tasker
       - cic-cache-tracker
-    command:
+    command: bash docker/start_server.sh
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        "/usr/local/bin/uwsgi" \
-        --wsgi-file /root/cic_cache/runnable/daemons/server.py \
-        --http :8000 \
-        --pyargv "-vv"
-
 
   cic-eth-tasker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -302,6 +292,7 @@ services:
       ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
       TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
       CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: unless-stopped 
     depends_on:
       - eth
@@ -313,12 +304,7 @@ services:
       - contract-config:/tmp/cic/config/:ro
         #command: ["/usr/local/bin/cic-eth-taskerd"]
         #command: ["sleep", "3600"]
-    command:
+    command: ./start_tasker.sh --aux-all -q cic-eth  -vv
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_tasker.sh --aux-all -q cic-eth  -vv
     
   cic-eth-signer:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -365,13 +351,6 @@ services:
       - signer-data:/run/crypto-dev-signer
       - contract-config:/tmp/cic/config/:ro
     command: ["python", "/usr/local/bin/crypto-dev-daemon", "-c", "/usr/local/etc/crypto-dev-signer", "-vv"]
-        #command:
-        #  - /bin/bash
-        #  - -c
-        #  - |
-        #    if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        #    ./start_tasker.sh --aux-all -q cic-eth  -vv
-    # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -384,6 +363,7 @@ services:
     environment:
       RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
       ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
+      RPC_HTTP_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
@@ -398,6 +378,7 @@ services:
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
       TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - eth
@@ -405,15 +386,7 @@ services:
       - redis
     volumes:
       - contract-config:/tmp/cic/config/:ro
-    command:
+    command: bash ./start_tracker.sh -vv
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        #./start_tracker.sh -vv -c /usr/local/etc/cic-eth
-        ./start_tracker.sh -vv
-    # command: "/root/start_manager.sh head -vv"
-
 
   cic-eth-dispatcher:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -436,12 +409,12 @@ services:
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-        #BANCOR_DIR: $BANCOR_DIR
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
       TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
       DATABASE_DEBUG: ${DATABASE_DEBUG:-false}
         #DATABASE_DEBUG: 1
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - eth
@@ -449,13 +422,7 @@ services:
       - redis
     volumes:
         - contract-config:/tmp/cic/config/:ro
-    command:
+    command: bash start_dispatcher.sh -q cic-eth -vv
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_dispatcher.sh -q cic-eth -vv
-    # command: "/root/start_dispatcher.sh -q cic-eth -vv"
     
 
 
@@ -488,6 +455,7 @@ services:
       CIC_TX_RETRY_DELAY: 60
       BATCH_SIZE: ${RETRIER_BATCH_SIZE:-50}
         #DATABASE_DEBUG: 1
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - eth
@@ -495,15 +463,7 @@ services:
       - redis
     volumes:
         - contract-config:/tmp/cic/config/:ro
-    command:
+    command: bash start_retry.sh -vv 
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_retry.sh -vv 
-    # command: "/root/start_retry.sh -q cic-eth -vv"
-
-
 
   cic-notify-tasker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:${TAG:-latest}

scripts/dump.sh

@@ -1 +1 @@
-docker run -t -v  --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset
+docker run --rm -v cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset