Merge branch 'master' into bvander/contract-migration-readyz

.gitignore

@@ -15,3 +15,4 @@ build/
 .idea
 **/.vim
 **/*secret.yaml
+**/*.bash_history

README.md

@@ -19,6 +19,11 @@ Run app/contract-migration to deploy contracts
 RUN_MASK=3 docker-compose up contract-migration
 ```
 
+View container status
+```
+docker ps --filter network=cic-network --format "table {{.ID}}\t{{.Names}}\t{{.Status}}" --all
+```
+
 stop cluster
 ```
 docker-compose down

apps/cic-cache/docker/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/cic-cache/docker/start_server.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+set -u
+
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+. ./db.sh
+
+if [ $? -ne "0" ]; then
+	>&2 echo db migrate fail
+	exit 1
+fi
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/uwsgi --wsgi-file /root/cic_cache/runnable/daemons/server.py --http :8000 --pyargv "-vv"
+else
+  /usr/local/bin/uwsgi --wsgi-file /root/cic_cache/runnable/daemons/server.py --http :8000 --pyargv "-vv"
+fi
+

apps/cic-cache/docker/start_tasker.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+set -u
+
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+. ./db.sh
+
+if [ $? -ne "0" ]; then
+	>&2 echo db migrate fail
+	exit 1
+fi
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/cic-cache-taskerd -vv
+else
+  /usr/local/bin/cic-cache-taskerd -vv
+fi
+

apps/cic-cache/docker/start_tracker.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+set -e
+set -u
+
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
 . ./db.sh
 
 if [ $? -ne "0" ]; then
@@ -7,4 +12,12 @@ if [ $? -ne "0" ]; then
 	exit 1
 fi
 
-/usr/local/bin/cic-cache-trackerd $@
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  docker/wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source docker/get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/cic-cache-trackerd $@
+else
+  /usr/local/bin/cic-cache-trackerd $@
+fi
+

apps/cic-cache/docker/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a5,<0.7.0
+cic-eth-registry>=0.6.1a3,<0.7.0
 hexathon~=0.0.1a8
 chainqueue>=0.0.4a6,<0.1.0
 eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/admin/token.py

@@ -1,2 +1,21 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
 # local imports
-from cic_eth.eth.erc20 import default_token
+from cic_eth.task import BaseTask
+
+celery_app = celery.current_app
+logg = logging.getLogger()
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def default_token(self):
+    return {
+            'symbol': self.default_token_symbol,
+            'address': self.default_token_address,
+            'name': self.default_token_name,
+            'decimals': self.default_token_decimals,
+            }

apps/cic-eth/cic_eth/api/api_task.py

@@ -17,50 +17,15 @@ from cic_eth.enum import LockEnum
 
 app = celery.current_app
 
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)
 
 
 class Api(ApiBase):
-
-    @staticmethod
-    def to_v_list(v, n):
-        """Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
-
-        :param v: Arguments
-        :type v: str or list
-        :param n: Number of elements to generate arguments for
-        :type n: int
-        :rtype: list
-        :returns: list of assembled arguments
-        """
-        if isinstance(v, str):
-            vv = v
-            v = []
-            for i in range(n):
-                v.append([vv])
-        elif not isinstance(v, list):
-            raise ValueError('argument must be single string, or list or strings or lists')
-        else:
-            if len(v) != n:
-                raise ValueError('v argument count must match integer n')
-            for i in range(n):
-                if isinstance(v[i], str):
-                    v[i] = [v[i]]
-                elif not isinstance(v, list):
-                    raise ValueError('proof argument must be single string, or list or strings or lists')
-
-        return v
-   
+    
 
     def default_token(self):
-        """Retrieves the default fallback token of the custodial network.
-
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
         s_token = celery.signature(
-                'cic_eth.eth.erc20.default_token',
+                'cic_eth.admin.token.default_token',
                 [],
                 queue=self.queue,
                 )
@@ -70,97 +35,6 @@ class Api(ApiBase):
         return s_token.apply_async()
 
 
-    def token(self, token_symbol, proof=None):
-        """Single-token alias for tokens method.
-
-        See tokens method for details.
-
-        :param token_symbol: Token symbol to look up
-        :type token_symbol: str
-        :param proof: Proofs to add to signature verification for the token
-        :type proof: str or list
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        if not isinstance(token_symbol, str):
-            raise ValueError('token symbol must be string')
-
-        return self.tokens([token_symbol], proof=proof)
-
-
-    def tokens(self, token_symbols, proof=None):
-        """Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
-
-        Token symbols are always strings, and should be specified using uppercase letters.
-
-        If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
-
-        The proof argument may be specified in a number of ways:
-
-        - as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
-        - as a single string, where the same proof is used for each token lookup
-        - as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
-        - as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
-
-        The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
-
-        This method is not intended to be used synchronously. Do so at your peril.
-
-        :param token_symbols: Token symbol strings to look up
-        :type token_symbol: list
-        :param proof: Proof(s) to verify tokens against
-        :type proof: None, str or list
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        if not isinstance(token_symbols, list):
-            raise ValueError('token symbols argument must be list')
-
-        if proof == None:
-            logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
-            proof = ''
-
-        logg.debug('proof is {}'.format(proof))
-        l = len(token_symbols)
-        if len(proof) == 0:
-            l = 0 
-        proof = Api.to_v_list(proof, l)
-
-        chain_spec_dict = self.chain_spec.asdict()
-
-        s_token_resolve = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    token_symbols,
-                    chain_spec_dict,
-                    ],
-                queue=self.queue,
-                )
-
-        s_token_info = celery.signature(
-                'cic_eth.eth.erc20.token_info',
-                [
-                    chain_spec_dict,
-                    proof,
-                    ],
-                queue=self.queue,
-                )
-
-        s_token_verify = celery.signature(
-                    'cic_eth.eth.erc20.verify_token_info',
-                    [
-                        chain_spec_dict,
-                        self.callback_success,
-                        self.callback_error,
-                        ],
-                    queue=self.queue,
-                    )
-
-        s_token_info.link(s_token_verify)
-        s_token_resolve.link(s_token_info)
-        return s_token_resolve.apply_async()
-
-
 #    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
 #        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 #

apps/cic-eth/cic_eth/callbacks/noop.py

@@ -1,10 +1,7 @@
-import logging
-
 import celery
 
 celery_app = celery.current_app
-#logg = celery_app.log.get_default_logger()
-logg = logging.getLogger()
+logg = celery_app.log.get_default_logger()
 
 
 @celery_app.task(bind=True)

apps/cic-eth/cic_eth/error.py

@@ -48,6 +48,8 @@ class RoleMissingError(Exception):
     pass
 
 
+
+
 class IntegrityError(Exception):
     """Exception raised to signal irregularities with deduplication and ordering of tasks
 
@@ -83,8 +85,3 @@ class RoleAgencyError(SeppukuError):
 class YouAreBrokeError(Exception):
     """Exception raised when a value transfer is attempted without access to sufficient funds
     """
-
-
-class TrustError(Exception):
-    """Exception raised when required trust proofs are missing for a request
-    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -19,7 +19,6 @@ from hexathon import (
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
 from chainqueue.sql.tx import cache_tx_dict
-from okota.token_index import to_identifier
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -40,11 +39,9 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
         CriticalSQLAlchemyAndSignerTask,
-        BaseTask,
     )
 from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.encode import tx_normalize
-from cic_eth.eth.trust import verify_proofs
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -476,69 +473,3 @@ def cache_approve_data(
     session.close()
     return (tx_hash_hex, cache_id)
 
-
-@celery_app.task(bind=True, base=BaseTask)
-def token_info(self, tokens, chain_spec_dict, proofs=[]):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    i = 0
-
-    for token in tokens:
-        result_data = []
-        token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
-        token_chain_object.load(rpc)
-
-        token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
-        token_proofs = [token_symbol_proof_hex]
-        if len(proofs) > 0:
-            token_proofs += proofs[i]
-
-        tokens[i] = {
-            'decimals': token_chain_object.decimals,
-            'name': token_chain_object.name,
-            'symbol': token_chain_object.symbol,
-            'address': tx_normalize.executable_address(token_chain_object.address),
-            'proofs': token_proofs,
-            'converters': tokens[i]['converters'],
-                }   
-        i += 1
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
-    queue = self.request.delivery_info.get('routing_key')
-
-    for token in tokens:
-        s = celery.signature(
-                'cic_eth.eth.trust.verify_proofs',
-                [
-                    token,
-                    token['address'],
-                    token['proofs'],
-                    chain_spec_dict,
-                    success_callback,
-                    error_callback,
-                    ],
-                queue=queue,
-                )
-
-        if success_callback != None:
-            s.link(success_callback)
-        if error_callback != None:
-            s.on_error(error_callback)
-        s.apply_async()
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-        'symbol': self.default_token_symbol,
-        'address': self.default_token_address,
-        'name': self.default_token_name,
-        'decimals': self.default_token_decimals,
-        }

apps/cic-eth/cic_eth/eth/trust.py

@@ -1,77 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from eth_address_declarator import Declarator
-from chainlib.connection import RPCConnection
-from chainlib.chain import ChainSpec
-from cic_eth.db.models.role import AccountRole
-from cic_eth_registry import CICRegistry
-from hexathon import strip_0x
-
-# local imports
-from cic_eth.task import BaseTask
-from cic_eth.error import TrustError
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
-    proof = strip_0x(proof)
-
-    proofs = []
- 
-    logg.debug('proof count {}'.format(len(proofs)))
-    if len(proofs) == 0:
-        logg.debug('error {}'.format(len(proofs)))
-        raise TrustError('foo')
-
-    return (chained_input, (proof, proofs))
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
-    queue = self.request.delivery_info.get('routing_key')
-
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    session = self.create_session()
-    sender_address = AccountRole.get_address('DEFAULT', session)
-
-    registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-
-    declarator = Declarator(chain_spec)
-
-    have_proofs = {}
-
-    for proof in proofs:
-
-        proof = strip_0x(proof)
-
-        have_proofs[proof] = []
-
-        for trusted_address in self.trusted_addresses:
-            o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
-            r = rpc.do(o)
-            declarations = declarator.parse_declaration(r)
-            logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
-
-            for declaration in declarations:
-                declaration = strip_0x(declaration)
-                if declaration == proof:
-                    logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
-                    have_proofs[proof].append(trusted_address)
-
-    out_proofs = {}
-    for proof in have_proofs.keys():
-        if len(have_proofs[proof]) == 0:
-            logg.error('missing signer for proof {} subject {}'.format(proof, subject))
-            raise TrustError((subject, proof,))
-        out_proofs[proof] = have_proofs[proof]
-       
-    return (chained_input, out_proofs)

apps/cic-eth/cic_eth/pytest/fixtures_celery.py

@@ -4,21 +4,18 @@ import tempfile
 import logging
 import shutil
 
-# local imports
+# local impors
 from cic_eth.task import BaseTask
 
 #logg = logging.getLogger(__name__)
 logg = logging.getLogger()
 
+
 @pytest.fixture(scope='function')
 def init_celery_tasks(
     contract_roles, 
         ):
     BaseTask.call_address = contract_roles['DEFAULT']
-    BaseTask.trusted_addresses = [
-            contract_roles['TRUSTED_DECLARATOR'],
-            contract_roles['CONTRACT_DEPLOYER'],
-            ]
 
 
 # celery fixtures
@@ -41,7 +38,6 @@ def celery_includes():
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
         'cic_eth.pytest.mock.filter',
-        'cic_eth.pytest.mock.callback',
     ]
 
 

apps/cic-eth/cic_eth/pytest/mock/__init__.py

@@ -1,2 +1 @@
 from .filter import *
-from .callback import *

apps/cic-eth/cic_eth/pytest/mock/callback.py

@@ -1,38 +0,0 @@
-# standard imports
-import os
-import logging
-import mmap
-
-# standard imports
-import tempfile
-
-# external imports
-import celery
-
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
-
-celery_app = celery.current_app
-
-
-class CallbackTask(celery.Task):
-
-    mmap_path = tempfile.mkdtemp()
-
-
-@celery_app.task(bind=True, base=CallbackTask)
-def test_callback(self, a, b, c):
-    s = 'ok'
-    if c > 0:
-        s = 'err'
-
-    fp = os.path.join(self.mmap_path, b)
-    f = open(fp, 'wb+')
-    f.write(b'\x00')
-    f.seek(0)
-    m = mmap.mmap(f.fileno(), length=1)
-    m.write(c.to_bytes(1, 'big'))
-    m.close()
-    f.close()
-
-    logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -210,7 +210,6 @@ def main():
     default_token.load(conn)
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name
-    BaseTask.trusted_addresses = trusted_addresses
 
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

apps/cic-eth/cic_eth/task.py

@@ -28,7 +28,6 @@ class BaseTask(celery.Task):
 
     session_func = SessionBase.create_session
     call_address = ZERO_ADDRESS
-    trusted_addresses = []
     create_nonce_oracle = RPCNonceOracle
     create_gas_oracle = RPCGasOracle
     default_token_address = None

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         4,
-        'alpha.11',
+        'alpha.8',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/entrypoints/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/cic-eth/docker/entrypoints/start_dispatcher.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 
+set -e
+set -u
+
 . ./db.sh
 
-/usr/local/bin/cic-eth-dispatcherd $@
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/cic-eth-dispatcherd $@
+else
+  /usr/local/bin/cic-eth-dispatcherd $@
+fi

apps/cic-eth/docker/entrypoints/start_retry.sh

@@ -1,5 +1,18 @@
 #!/bin/bash
 
+set -e
+set -u
+
 . ./db.sh
 
-/usr/local/bin/cic-eth-retrierd $@
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/cic-eth-retrierd $@
+else
+  /usr/local/bin/cic-eth-retrierd $@
+fi

apps/cic-eth/docker/entrypoints/start_tasker.sh

@@ -1,31 +1,18 @@
 #!/bin/bash
 
 set -e
+set -u
+
 . ./db.sh
 
-# set CONFINI_ENV_PREFIX to override the env prefix to override env vars
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
 
-#echo "!!! starting signer"
-#python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
 
-echo "!!! starting taskerd"
-/usr/local/bin/cic-eth-taskerd $@
-
-# thanks! https://docs.docker.com/config/containers/multi-service_container/
-#sleep 1;
-#echo "!!! entering monitor loop"
-#while true; do
-#  ps aux | grep crypto-dev-daemon | grep -q -v grep
-#  PROCESS_1_STATUS=$?
-#  ps aux | grep cic-eth-tasker |grep -q -v grep
-#  PROCESS_2_STATUS=$?
-#  # If the greps above find anything, they exit with 0 status
-#  # If they are not both 0, then something is wrong
-#  if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then
-#    echo "One of the processes has already exited."
-#    exit 1
-#  fi
-#  sleep 15;
-#done
-#
-set +e
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/cic-eth-taskerd $@
+else
+  /usr/local/bin/cic-eth-taskerd $@
+fi

apps/cic-eth/docker/entrypoints/start_tracker.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 
+set -e
+set -u
+
 . ./db.sh
 
-/usr/local/bin/cic-eth-trackerd $@
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./get_readyz.sh # set env vars form endpoint
+  /usr/local/bin/cic-eth-trackerd $@
+else
+  /usr/local/bin/cic-eth-trackerd $@
+fi

apps/cic-eth/docker/entrypoints/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
 celery==4.4.7
-chainlib-eth>=0.0.9rc4,<0.1.0
+chainlib-eth>=0.0.9rc2,<0.1.0
 semver==2.13.0
 crypto-dev-signer>=0.4.15rc2,<0.5.0

apps/cic-eth/services_requirements.txt

@@ -6,11 +6,10 @@ redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.2.4a1,<0.3.0
+eth-address-index>=0.2.3a4,<0.3.0
 eth-accounts-index>=0.1.2a3,<0.2.0
-cic-eth-registry>=0.6.1a5,<0.7.0
+cic-eth-registry>=0.6.1a3,<0.7.0
 erc20-faucet>=0.3.2a2,<0.4.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0
 moolb~=0.1.1b2
-okota>=0.2.4a6,<0.3.0

apps/cic-eth/tests/task/api/test_app_noncritical.py

@@ -1,27 +1,6 @@
-# standard imports
-import logging
-import os
-import uuid
-import time
-import mmap
-
-# external imports
-import celery
-import pytest
-from hexathon import (
-        strip_0x,
-        uniform as hex_uniform,
-        )
-
 # local imports
 from cic_eth.api.api_task import Api
 from cic_eth.task import BaseTask
-from cic_eth.error import TrustError
-from cic_eth.encode import tx_normalize
-from cic_eth.pytest.mock.callback import CallbackTask
-
-logg = logging.getLogger()
-
 
 def test_default_token(
         default_chain_spec,
@@ -38,175 +17,3 @@ def test_default_token(
     t = api.default_token()
     r = t.get_leaf()
     assert r['address'] == foo_token
-
-
-def test_to_v_list():
-    assert Api.to_v_list('', 0) == []
-    assert Api.to_v_list([], 0) == []
-    assert Api.to_v_list('foo', 1) == [['foo']]
-    assert Api.to_v_list(['foo'], 1) == [['foo']]
-    assert Api.to_v_list(['foo', 'bar'], 2) == [['foo'], ['bar']]
-    assert Api.to_v_list('foo', 3) == [['foo'], ['foo'], ['foo']]
-    assert Api.to_v_list([['foo'], ['bar']], 2) == [['foo'], ['bar']]
-    with pytest.raises(ValueError):
-        Api.to_v_list([['foo'], ['bar']], 3)
-    with pytest.raises(ValueError):
-        Api.to_v_list(['foo', 'bar'], 3)
-    with pytest.raises(ValueError):
-        Api.to_v_list([['foo'], ['bar'], ['baz']], 2)
-
-    assert Api.to_v_list([
-            ['foo'],
-            'bar',
-            ['inky', 'pinky', 'blinky', 'clyde'],
-        ], 3) == [
-            ['foo'],
-            ['bar'],
-            ['inky', 'pinky', 'blinky', 'clyde'],
-            ]
-
-
-def test_token_single(
-        default_chain_spec,
-        foo_token,
-        bar_token,
-        token_registry,
-        register_tokens,
-        register_lookups,
-        cic_registry,
-        init_database,
-        init_celery_tasks,
-        custodial_roles,
-        foo_token_declaration,
-        bar_token_declaration,
-        celery_session_worker,
-        ):
-
-    api = Api(str(default_chain_spec), queue=None, callback_param='foo')     
-
-    t = api.token('FOO', proof=None)
-    r = t.get()
-    logg.debug('rr {}'.format(r))
-    assert len(r) == 1
-    assert r[0]['address'] == strip_0x(foo_token)
-
-
-    t = api.token('FOO', proof=foo_token_declaration)
-    r = t.get()
-    assert len(r) == 1
-    assert r[0]['address'] == strip_0x(foo_token)
-
-
-def test_tokens_noproof(
-        default_chain_spec,
-        foo_token,
-        bar_token,
-        token_registry,
-        register_tokens,
-        register_lookups,
-        cic_registry,
-        init_database,
-        init_celery_tasks,
-        custodial_roles,
-        foo_token_declaration,
-        bar_token_declaration,
-        celery_worker,
-        ):
-
-    api = Api(str(default_chain_spec), queue=None, callback_param='foo')     
-
-    t = api.tokens(['FOO'], proof=[])
-    r = t.get()
-    assert len(r) == 1
-    assert r[0]['address'] == strip_0x(foo_token)
-
-    t = api.tokens(['BAR'], proof='')
-    r = t.get()
-    assert len(r) == 1
-    assert r[0]['address'] == strip_0x(bar_token)
-
-    t = api.tokens(['FOO'], proof=None)
-    r = t.get()
-    assert len(r) == 1
-    assert r[0]['address'] == strip_0x(foo_token)
-
-
-def test_tokens(
-        default_chain_spec,
-        foo_token,
-        bar_token,
-        token_registry,
-        register_tokens,
-        register_lookups,
-        cic_registry,
-        init_database,
-        init_celery_tasks,
-        custodial_roles,
-        foo_token_declaration,
-        bar_token_declaration,
-        celery_session_worker,
-        ):
-
-    api = Api(str(default_chain_spec), queue=None, callback_param='foo')     
-
-    t = api.tokens(['FOO'], proof=[[foo_token_declaration]])
-    r = t.get()
-    logg.debug('rr {}'.format(r))
-    assert len(r) == 1
-    assert r[0]['address'] == strip_0x(foo_token)
-    
-    t = api.tokens(['BAR', 'FOO'], proof=[[bar_token_declaration], [foo_token_declaration]])
-    r = t.get()
-    logg.debug('results {}'.format(r))
-    assert len(r) == 2
-    assert r[1]['address'] == strip_0x(foo_token)
-    assert r[0]['address'] == strip_0x(bar_token)
-
-    celery_app = celery.current_app
-
-    results = []
-    targets = []
-
-    api_param = str(uuid.uuid4())
-    api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
-    bogus_proof = os.urandom(32).hex()
-    t = api.tokens(['FOO'], proof=[[bogus_proof]])
-    r = t.get()
-    logg.debug('r {}'.format(r))
-
-    while True:
-        fp = os.path.join(CallbackTask.mmap_path, api_param)
-        try:
-            f = open(fp, 'rb')
-        except FileNotFoundError:
-            time.sleep(0.1)
-            logg.debug('look for {}'.format(fp))
-            continue
-        f = open(fp, 'rb')
-        m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
-        v = m.read(1)
-        m.close()
-        f.close()
-        assert v == b'\x01'
-        break
-
-    api_param = str(uuid.uuid4())
-    api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
-    t = api.tokens(['BAR'], proof=[[bar_token_declaration]])
-    r = t.get()
-    logg.debug('rr  {} {}'.format(r, t.children))
-
-    while True:
-        fp = os.path.join(CallbackTask.mmap_path, api_param)
-        try:
-            f = open(fp, 'rb')
-        except FileNotFoundError:
-            time.sleep(0.1)
-            continue
-        m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
-        v = m.read(1)
-        m.close()
-        f.close()
-        assert v == b'\x00'
-        break
-

apps/cic-eth/tests/unit/admin/test_default_token.py

@@ -10,7 +10,7 @@ def test_default_token(
         ):
       
     s = celery.signature(
-            'cic_eth.eth.erc20.default_token',
+            'cic_eth.admin.token.default_token',
             [],
             queue=None,
             )

apps/cic-eth/tools_requirements.txt

@@ -1,6 +1,6 @@
-crypto-dev-signer>=0.4.15rc2,<=0.4.15
+crypto-dev-signer>=0.4.15a7,<=0.4.15
 chainqueue>=0.0.5a1,<0.1.0
-cic-eth-registry>=0.6.1a5,<0.7.0
+cic-eth-registry>=0.6.1a3,<0.7.0
 redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1

apps/cic-ussd/docker/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/cic-ussd/docker/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

apps/contract-migration/docker/init_readyz.sh

@@ -0,0 +1,25 @@
+#! /bin/bash
+
+set -e 
+set -a
+
+mkdir -p $DEV_DATA_DIR/health
+
+source $DEV_DATA_DIR/env_reset
+
+jq --arg CIC_REGISTRY_ADDRESS "$CIC_REGISTRY_ADDRESS" \
+   --arg CIC_TRUST_ADDRESS "$CIC_TRUST_ADDRESS" \
+   --arg CIC_DEFAULT_TOKEN_SYMBOL "$CIC_DEFAULT_TOKEN_SYMBOL"\
+   --arg TOKEN_NAME "$TOKEN_NAME"\
+   --arg RUN_MASK "$RUN_MASK" \
+   -n '{"CIC_REGISTRY_ADDRESS": $CIC_REGISTRY_ADDRESS, 
+        "CIC_TRUST_ADDRESS": $CIC_TRUST_ADDRESS,
+        "CIC_DEFAULT_TOKEN_SYMBOL": $CIC_DEFAULT_TOKEN_SYMBOL,
+        "TOKEN_NAME": $TOKEN_NAME,
+        "RUN_MASK": $RUN_MASK}' > $DEV_DATA_DIR/health/readyz
+
+cd $DEV_DATA_DIR/health
+
+echo "starting health endpoint on :8000/readyz"
+python -m http.server 8000 &> /dev/null 
+

apps/contract-migration/env_reset

@@ -0,0 +1,16 @@
+export DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export DEV_ETH_ACCOUNT_RESERVE_MINTER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export DEV_RESERVE_AMOUNT=10000000000000000000000000000000000
+export DEV_FAUCET_AMOUNT=50000000
+export DEV_GAS_AMOUNT=100000000000000000000000
+export DEV_TOKEN_AMOUNT=100000000000000000000000
+export DEV_ETH_GAS_PRICE=
+export DEV_DATA_DIR=.
+export DEV_PIP_EXTRA_INDEX_URL=https://pip.grassrootseconomics.net:8433
+export DEV_ETH_PROVIDER_HOST=
+export DEV_ETH_PROVIDER_PORT=
+export CIC_TRUST_ADDRESS=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+export CIC_DEFAULT_TOKEN_SYMBOL=GFT
+export WALLET_KEY_FILE=/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
+

apps/contract-migration/requirements.txt

@@ -1,15 +1,12 @@
-cic-eth[tools]==0.12.4a11
-chainlib-eth>=0.0.9rc4,<0.1.0
-chainlib==0.0.9rc1,<0.1.0
+cic-eth[tools]==0.12.4a8
+chainlib-eth>=0.0.9a14,<0.1.0
 eth-erc20>=0.1.2a3,<0.2.0
 erc20-demurrage-token>=0.0.5a2,<0.1.0
-#eth-accounts-index>=0.1.2a2,<0.2.0
-eth-address-index>=0.2.4a1,<0.3.0
-cic-eth-registry>=0.6.1a5,<0.7.0
+eth-accounts-index>=0.1.2a2,<0.2.0
+eth-address-index>=0.2.3a4,<0.3.0
+cic-eth-registry>=0.6.1a2,<0.7.0
 erc20-transfer-authorization>=0.3.5a2,<0.4.0
 erc20-faucet>=0.3.2a2,<0.4.0
 sarafu-faucet>=0.0.7a2,<0.1.0
 confini>=0.4.2rc3,<1.0.0
-crypto-dev-signer>=0.4.15rc2,<=0.4.15
-eth-token-index>=0.2.4a1,<=0.3.0
-okota>=0.2.4a5,<0.3.0
+crypto-dev-signer>=0.4.15a7,<=0.4.15

apps/contract-migration/reset.sh

@@ -65,25 +65,22 @@ fi
 echo "giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT"
 giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
 
+>&2 echo "deploy account index contract"
+DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w`
+>&2 echo "add deployer address as account index writer"
+eth-accounts-index-writer $fee_price_arg -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+
+>&2 echo "deploy contract registry contract"
+CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $RPC_PROVIDER -vv -s -u -w`
+eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $RPC_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
+eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
+
 # Deploy address declarator registry
 >&2 echo "deploy address declarator contract"
 declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
 DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv $declarator_description`
-
->&2 echo "deploy contract registry contract"
-#CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $RPC_PROVIDER -vv -s -u -w`
-CIC_REGISTRY_ADDRESS=`okota-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry --address-declarator $DEV_DECLARATOR_ADDRESS -p $RPC_PROVIDER -vv -s -u -w`
-eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $RPC_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
-
 eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS
 
->&2 echo "deploy account index contract"
-#DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w`
-DEV_ACCOUNT_INDEX_ADDRESS=`okota-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w --address-declarator $DEV_DECLARATOR_ADDRESS --token-address $DEV_RESERVE_ADDRESS`
-#>&2 echo "add deployer address as account index writer"
-#eth-accounts-index-writer $fee_price_arg -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
-eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
-
 # Deploy transfer authorization contact
 >&2 echo "deploy transfer auth contract"
 DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
@@ -91,8 +88,7 @@ eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_RE
 
 # Deploy token index contract
 >&2 echo "deploy token index contract"
-#DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
-DEV_TOKEN_INDEX_ADDRESS=`okota-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv --address-declarator $DEV_DECLARATOR_ADDRESS`
+DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
 eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
 >&2 echo "add reserve token to token index"
 eth-token-index-add $fee_price_arg -s -u -w -y $WALLET_KEY_FILE  -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
@@ -116,10 +112,16 @@ export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
 
 echo "Writing env_reset file ..."
 
-echo "export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
+# note these are also written in init_readyz.sh
+cat <<EOF > $DEV_DATA_DIR/env_reset 
+export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
+export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS
 export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
 export TOKEN_NAME=$TOKEN_NAME
-" >> "${DEV_DATA_DIR}"/env_reset
+EOF
+
+confini-dump -vv --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset
+confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config 
 
 set +a
 set +e

apps/contract-migration/run_job.sh

@@ -12,6 +12,10 @@ if [[ $((RUN_MASK & 1)) -eq 1 ]]
 then
 	>&2 echo -e "\033[;96mRUNNING\033[;39m RUN_MASK 1 - contract deployment"
 	./reset.sh
+  # cic-eth-xxx services rely on
+  export RUN_MASK_PHASE=1
+  ./docker/init_readyz.sh &
+  echo 0
 	if [ $? -ne "0" ]; then
 		>&2 echo -e "\033[;31mFAILED\033[;39m RUN_MASK 1 - contract deployment"
 		exit 1;
@@ -29,3 +33,5 @@ then
 	fi
 	>&2 echo -e "\033[;32mSUCCEEDED\033[;39m RUN_MASK 2 - custodial service initialization"
 fi
+# this will leave the container up and serving the results of the migration
+wait

apps/contract-migration/seed_cic_eth.sh

@@ -46,7 +46,7 @@ cic-eth-tag -i $CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFE
 DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"
-#eth-accounts-index-writer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $account_index_address -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
+eth-accounts-index-writer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $account_index_address -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 
 # Transfer gas to custodial gas provider adddress
 _CONFINI_DIR=$CONFINI_DIR
@@ -78,11 +78,11 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 >&2 erc20-transfer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER --fee-limit 100000 -e $reserve_address -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${DEV_TOKEN_AMOUNT:0:-1}
 
 # Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
-cic-eth-ctl -vv -i $CHAIN_SPEC unlock INIT
-cic-eth-ctl -vv -i $CHAIN_SPEC unlock SEND
-cic-eth-ctl -vv -i $CHAIN_SPEC unlock QUEUE
+cic-eth-ctl -i $CHAIN_SPEC unlock INIT
+cic-eth-ctl -i $CHAIN_SPEC unlock SEND
+cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE
 
-#confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config
+confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config 
 
 set +a
 set +e

apps/data-seeding/scripts/get_readyz.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+set -u
+
+echo "fetching migration variables from $CONTRACT_MIGRATION_URL"
+for s in $(curl -s "$CONTRACT_MIGRATION_URL/readyz" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "exporting $s"
+    export $s
+done

apps/data-seeding/scripts/run_job.sh

@@ -0,0 +1,15 @@
+#! /bin/bash 
+set -e
+set -u
+env
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-600}
+
+if [[ "$CONTRACT_MIGRATION_URL" ]]; then
+  echo "waiting for $CONTRACT_MIGRATION_URL/readyz"
+  ./scripts/wait-for-it.sh $CONTRACT_MIGRATION_URL  -t $WAIT_FOR_TIMEOUT 
+  source ./scripts/get_readyz.sh # set env vars form endpoint
+  ./import_ussd.sh
+else
+  ./import_ussd.sh
+fi
+

apps/data-seeding/scripts/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

docker-compose.yml

@@ -60,6 +60,8 @@ services:
 
   contract-migration:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest}
+    ports:
+      - 8012:8000
     build:
       context: apps/contract-migration
       dockerfile: docker/Dockerfile
@@ -68,7 +70,6 @@ services:
         pip_extra_args: $PIP_EXTRA_ARGS
         EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
         EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
-    # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
       RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
       ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
@@ -93,9 +94,14 @@ services:
       RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
       DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-50000000}
       DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE
+      #DEV_GAS_AMOUNT: "100000000000000000000000" # removed from env_reset
+      #WALLET_KEY_FILE: ./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c #removed from env_reset 
       TOKEN_NAME: ${TOKEN_NAME:-Giftable Token}
       TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT}
       TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token}
+      #TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token}
+      #TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET}
+      #TOKEN_TYPE: ${TOKEN_TYPE:-erc20_demurrage_token }
       TOKEN_DECIMALS: $TOKEN_DECIMALS
       TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD
       TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
@@ -103,9 +109,8 @@ services:
       TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL
       TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS
       SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000}
-    restart: on-failure 
     command: ["./run_job.sh"]
-        #command: ["./reset.sh"]
+    #command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
     depends_on:
       - eth
       - postgres
@@ -113,7 +118,7 @@ services:
       - cic-eth-tasker
     volumes:
       - contract-config:/tmp/cic/config
-
+      - ./apps/contract-migration:/root
 
   data-seeding:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:${TAG:-latest}
@@ -153,9 +158,11 @@ services:
       USSD_PROVIDER: http://cic-user-ussd-server:9000
       CELERY_QUEUE: cic-import-ussd
       EXCLUSIONS: ussd
-    command: bash import_ussd.sh
+      CONTRACT_MIGRATION_URL: contract-migration:8000
+    command: bash ./scripts/run_job.sh 
     volumes:
       - contract-config:/tmp/cic/config/:ro
+        #- ./apps/data-seeding:/root
 
   cic-cache-tracker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest}
@@ -181,17 +188,13 @@ services:
       CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996}
       CELERY_BROKER_URL: redis://redis:6379
       CELERY_RESULT_URL: redis://redis:6379
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - redis
       - postgres
       - eth
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_tracker.sh -c /usr/local/etc/cic-cache -vv
+    command: ./start_tracker.sh -c /usr/local/etc/cic-cache -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -218,17 +221,13 @@ services:
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CELERY_BROKER_URL: redis://redis:6379
       CELERY_RESULT_URL: redis://redis:6379
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: unless-stopped 
     depends_on:
       - redis
       - postgres
       - eth
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        /usr/local/bin/cic-cache-taskerd -vv
+    command: bash docker/start_tasker.sh
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -243,10 +242,10 @@ services:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
-        #DATABASE_PASSWORD: ${DATABASE_PASSWORD:-
+      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
       DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
       DATABASE_DEBUG: 1
-        #PGPASSWORD: $DATABASE_PASSWORD
+      CONTRACT_MIGRATION_URL: contract-migration:8000
       SERVER_PORT: 8000
     restart: on-failure 
     ports:
@@ -255,16 +254,7 @@ services:
       - postgres
       - cic-cache-tasker
       - cic-cache-tracker
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        "/usr/local/bin/uwsgi" \
-        --wsgi-file /root/cic_cache/runnable/daemons/server.py \
-        --http :8000 \
-        --pyargv "-vv"
-
+    command: bash docker/start_server.sh
 
   cic-eth-tasker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -302,6 +292,7 @@ services:
       ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
       TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
       CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: unless-stopped 
     depends_on:
       - eth
@@ -313,12 +304,7 @@ services:
       - contract-config:/tmp/cic/config/:ro
         #command: ["/usr/local/bin/cic-eth-taskerd"]
         #command: ["sleep", "3600"]
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_tasker.sh --aux-all -q cic-eth  -vv
+    command: ./start_tasker.sh --aux-all -q cic-eth  -vv
     
   cic-eth-signer:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -365,13 +351,6 @@ services:
       - signer-data:/run/crypto-dev-signer
       - contract-config:/tmp/cic/config/:ro
     command: ["python", "/usr/local/bin/crypto-dev-daemon", "-c", "/usr/local/etc/crypto-dev-signer", "-vv"]
-        #command:
-        #  - /bin/bash
-        #  - -c
-        #  - |
-        #    if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        #    ./start_tasker.sh --aux-all -q cic-eth  -vv
-    # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -384,6 +363,7 @@ services:
     environment:
       RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
       ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
+      RPC_HTTP_PROVIDER: ${RPC_PROVIDER:-http://eth:8545}
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
@@ -398,6 +378,7 @@ services:
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
       TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - eth
@@ -405,15 +386,7 @@ services:
       - redis
     volumes:
       - contract-config:/tmp/cic/config/:ro
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        #./start_tracker.sh -vv -c /usr/local/etc/cic-eth
-        ./start_tracker.sh -vv
-    # command: "/root/start_manager.sh head -vv"
-
+    command: bash ./start_tracker.sh -vv
 
   cic-eth-dispatcher:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest}
@@ -436,12 +409,12 @@ services:
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
-        #BANCOR_DIR: $BANCOR_DIR
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
       TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
       DATABASE_DEBUG: ${DATABASE_DEBUG:-false}
         #DATABASE_DEBUG: 1
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - eth
@@ -449,13 +422,7 @@ services:
       - redis
     volumes:
         - contract-config:/tmp/cic/config/:ro
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_dispatcher.sh -q cic-eth -vv
-    # command: "/root/start_dispatcher.sh -q cic-eth -vv"
+    command: bash start_dispatcher.sh -q cic-eth -vv
     
 
 
@@ -488,6 +455,7 @@ services:
       CIC_TX_RETRY_DELAY: 60
       BATCH_SIZE: ${RETRIER_BATCH_SIZE:-50}
         #DATABASE_DEBUG: 1
+      CONTRACT_MIGRATION_URL: contract-migration:8000
     restart: on-failure 
     depends_on:
       - eth
@@ -495,15 +463,7 @@ services:
       - redis
     volumes:
         - contract-config:/tmp/cic/config/:ro
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi 
-        ./start_retry.sh -vv 
-    # command: "/root/start_retry.sh -q cic-eth -vv"
-
-
+    command: bash start_retry.sh -vv 
 
   cic-notify-tasker:
     image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:${TAG:-latest}

scripts/dump.sh

@@ -1 +1 @@
-docker run -t -v  --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset
+docker run --rm -v cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset