Add tasks module to python package build

Update ci_templates/.cic-template.yml

See merge request grassrootseconomics/cic-internal-integration!61

apps/cic-cache/setup.cfg

@@ -25,6 +25,7 @@ licence_files =
 python_requires = >= 3.6
 packages = 
 	cic_cache
+	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
 	cic_cache.runnable
@@ -33,5 +34,6 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-cache-tracker = cic_cache.runnable.tracker:main
-	cic-cache-server = cic_cache.runnable.server:main
+	cic-cache-trackerd = cic_cache.runnable.tracker:main
+	cic-cache-serverd = cic_cache.runnable.server:main
+	cic-cache-taskerd = cic_cache.runnable.tasker:main

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -10,12 +10,15 @@ from cic_registry import zero_address
 from cic_eth.db.enum import LockEnum
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.lock import Lock
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        )
 from cic_eth.error import LockedError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
@@ -33,7 +36,7 @@ def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
@@ -51,7 +54,7 @@ def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set send lock
 
@@ -67,7 +70,7 @@ def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock_send(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset send lock
 
@@ -83,7 +86,7 @@ def unlock_send(chained_input, chain_str, address=zero_address):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set queue direct lock
 
@@ -99,7 +102,7 @@ def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock_queue(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset queue direct lock
 
@@ -115,7 +118,7 @@ def unlock_queue(chained_input, chain_str, address=zero_address):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_str, lock_flags, address=None):
     session = SessionBase.create_session()
     r = Lock.check(chain_str, lock_flags, address=zero_address, session=session)

apps/cic-eth/cic_eth/api/api_task.py

@@ -82,6 +82,7 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -143,6 +144,7 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -340,11 +342,6 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_nonce = celery.signature(
-                'cic_eth.eth.tx.reserve_nonce',
-                [],
-                queue=self.queue,
-                )
         s_account = celery.signature(
                 'cic_eth.eth.account.create',
                 [
@@ -352,12 +349,18 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_nonce.link(s_account)
-        s_check.link(s_nonce)
+        s_check.link(s_account)
         if self.callback_param != None:
             s_account.link(self.callback_success)
 
         if register:
+            s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
+                [
+                    'ACCOUNTS_INDEX_WRITER',
+                    ],
+                queue=self.queue,
+                )
             s_register = celery.signature(
                 'cic_eth.eth.account.register', 
                 [
@@ -365,7 +368,8 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-            s_account.link(s_register)
+            s_nonce.link(s_register)
+            s_account.link(s_nonce)
 
         t = s_check.apply_async(queue=self.queue)
         return t
@@ -390,7 +394,9 @@ class Api:
                 )
         s_nonce = celery.signature(
                 'cic_eth.eth.tx.reserve_nonce',
-                [],
+                [
+                    'GAS_GIFTER',
+                    ],
                 queue=self.queue,
                 )
         s_refill = celery.signature(

apps/cic-eth/cic_eth/error.py

@@ -66,3 +66,17 @@ class LockedError(Exception):
 
     """
     pass
+
+
+class SignerError(Exception):
+    """Exception raised when signer is unavailable or generates an error
+
+    """
+    pass
+
+
+class EthError(Exception):
+    """Exception raised when unspecified error from evm node is encountered
+
+    """
+    pass

apps/cic-eth/cic_eth/eth/account.py

@@ -21,8 +21,14 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.tx import TxCache
 from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.error import RoleMissingError
-from cic_eth.task import CriticalSQLAlchemyTask
+from cic_eth.error import (
+        RoleMissingError,
+        SignerError,
+        )
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        CriticalSQLAlchemyAndSignerTask,
+        )
 
 #logg = logging.getLogger(__name__)
 logg = logging.getLogger()
@@ -139,7 +145,7 @@ def unpack_gift(data):
      
 
 # TODO: Separate out nonce initialization task
-@celery_app.task(base=CriticalSQLAlchemyTask)
+@celery_app.task(base=CriticalSQLAlchemyAndSignerTask)
 def create(password, chain_str):
     """Creates and stores a new ethereum account in the keystore.
 
@@ -154,7 +160,13 @@ def create(password, chain_str):
     """
     chain_spec = ChainSpec.from_chain_str(chain_str)
     c = RpcClient(chain_spec)
-    a = c.w3.eth.personal.new_account(password)
+    a = None
+    try:
+        a = c.w3.eth.personal.new_account(password)
+    except FileNotFoundError:
+        pass
+    if a == None:
+        raise SignerError('create account')
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -165,7 +177,7 @@ def create(password, chain_str):
     return a
 
 
-@celery_app.task(bind=True, throws=(RoleMissingError,), base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, throws=(RoleMissingError,), base=CriticalSQLAlchemyAndSignerTask)
 def register(self, account_address, chain_str, writer_address=None):
     """Creates a transaction to add the given address to the accounts index.
 
@@ -215,7 +227,7 @@ def register(self, account_address, chain_str, writer_address=None):
     return account_address
 
 
-@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def gift(self, account_address, chain_str):
     """Creates a transaction to invoke the faucet contract for the given address.
 

apps/cic-eth/cic_eth/eth/gas.py

@@ -54,6 +54,7 @@ class GasOracle():
         """
         session = SessionBase.create_session()
         a = AccountRole.get_address('GAS_GIFTER', session)
+        logg.debug('gasgifter {}'.format(a))
         session.close()
         return a
 

apps/cic-eth/cic_eth/eth/task.py

@@ -8,6 +8,7 @@ from cic_registry.chain import ChainSpec
 # local imports
 from cic_eth.eth import RpcClient
 from cic_eth.queue.tx import create as queue_create
+from cic_eth.error import SignerError
 
 celery_app = celery.current_app
 logg = celery_app.log.get_default_logger()
@@ -26,7 +27,13 @@ def sign_tx(tx, chain_str):
     """
     chain_spec = ChainSpec.from_chain_str(chain_str)
     c = RpcClient(chain_spec)
-    tx_transfer_signed = c.w3.eth.sign_transaction(tx) 
+    tx_transfer_signed = None
+    try:
+        tx_transfer_signed = c.w3.eth.sign_transaction(tx)
+    except FileNotFoundError:
+        pass
+    if tx_transfer_signed == None:
+        raise SignerError('sign tx')
     logg.debug('tx_transfer_signed {}'.format(tx_transfer_signed))
     tx_hash = c.w3.keccak(hexstr=tx_transfer_signed['raw'])
     tx_hash_hex = tx_hash.hex()

apps/cic-eth/cic_eth/eth/token.py

@@ -24,6 +24,7 @@ from cic_eth.ext.address import translate_address
 from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
+        CriticalSQLAlchemyAndSignerTask,
     )
 
 celery_app = celery.current_app
@@ -212,7 +213,7 @@ def balance(tokens, holder_address, chain_str):
     return tokens
 
 
-@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_str):
     """Transfer ERC20 tokens between addresses
 
@@ -268,7 +269,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_str):
     return tx_hash_hex
 
 
-@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def approve(self, tokens, holder_address, spender_address, value, chain_str):
     """Approve ERC20 transfer on behalf of holder address
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -14,6 +14,7 @@ from cic_eth.db import Otx, SessionBase
 from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.nonce import NonceReservation
 from cic_eth.db.models.lock import Lock
+from cic_eth.db.models.role import AccountRole
 from cic_eth.db.enum import (
         LockEnum,
         StatusBits,
@@ -36,6 +37,9 @@ from cic_eth.admin.ctrl import lock_send
 from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
+        CriticalWeb3AndSignerTask,
+        CriticalSQLAlchemyAndSignerTask,
+        CriticalSQLAlchemyAndWeb3Task,
         )
 
 celery_app = celery.current_app
@@ -45,7 +49,7 @@ MAX_NONCE_ATTEMPTS = 3
 
 
 # TODO this function is too long
-@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
 def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=None):
     """Check the gas level of the sender address of a transaction.
 
@@ -73,6 +77,9 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
             if address == None:
                 address = o['address']
 
+    if not web3.Web3.isChecksumAddress(address):
+        raise ValueError('invalid address {}'.format(address))
+
     chain_spec = ChainSpec.from_chain_str(chain_str)
 
     queue = self.request.delivery_info['routing_key']
@@ -81,7 +88,12 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
     c = RpcClient(chain_spec)
 
     # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
-    balance = c.w3.eth.getBalance(address) 
+    balance = 0
+    try:
+        balance = c.w3.eth.getBalance(address) 
+    except ValueError as e:
+        raise EthError('balance call for {}'.format())
+
     logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required))
 
     if gas_required > balance:
@@ -418,7 +430,7 @@ def send(self, txs, chain_str):
 
 # TODO: if this method fails the nonce will be out of sequence. session needs to be extended to include the queue create, so that nonce is rolled back if the second sql query fails. Better yet, split each state change into separate tasks.
 # TODO: method is too long, factor out code for clarity
-@celery_app.task(bind=True, throws=(web3.exceptions.TransactionNotFound,), base=CriticalWeb3Task)
+@celery_app.task(bind=True, throws=(web3.exceptions.TransactionNotFound,), base=CriticalWeb3AndSignerTask)
 def refill_gas(self, recipient_address, chain_str):
     """Executes a native token transaction to fund the recipient's gas expenditures.
 
@@ -511,7 +523,7 @@ def refill_gas(self, recipient_address, chain_str):
     return tx_send_gas_signed['raw']
 
 
-@celery_app.task(bind=True)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_factor=1.1):
     """Create a new transaction from an existing one with same nonce and higher gas price.
 
@@ -588,11 +600,23 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
 
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
-def reserve_nonce(self, chained_input, address=None):
+def reserve_nonce(self, chained_input, signer=None):
     session = SessionBase.create_session()
 
-    if address == None:
+    address = None
+    if signer == None:
         address = chained_input
+        logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
+    else:
+        if web3.Web3.isChecksumAddress(signer):
+            address = signer
+            logg.debug('explicit address for reserve nonce {}'.format(signer))
+        else:
+            address = AccountRole.get_address(signer, session=session)
+            logg.debug('role for reserve nonce {} -> {}'.format(signer, address))
+
+    if not web3.Web3.isChecksumAddress(address):
+        raise ValueError('invalid result when resolving address for nonce {}'.format(address))
 
     root_id = self.request.root_id
     nonce = NonceReservation.next(address, root_id)

apps/cic-eth/cic_eth/runnable/create.py

@@ -5,6 +5,7 @@ import os
 import logging
 import uuid
 import json
+from xdg.BaseDirectory import xdg_config_home
 
 import celery
 from cic_eth.api import Api

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -25,7 +25,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
 
 
 default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')

apps/cic-eth/cic_eth/runnable/resend.py

@@ -22,7 +22,7 @@ logg = logging.getLogger()
 logging.getLogger('web3').setLevel(logging.WARNING)
 logging.getLogger('urllib3').setLevel(logging.WARNING)
 
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 
 argparser = argparse.ArgumentParser()

apps/cic-eth/cic_eth/runnable/view.py

@@ -37,7 +37,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
 
 
 default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)')

apps/cic-eth/cic_eth/task.py

@@ -5,6 +5,12 @@ import requests
 import celery
 import sqlalchemy
 
+# local imports
+from cic_eth.error import (
+        SignerError,
+        EthError,
+        )
+
 
 class CriticalTask(celery.Task):
     retry_jitter = True
@@ -30,4 +36,18 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         requests.exceptions.ConnectionError,
+        EthError,
+        )
+
+class CriticalSQLAlchemyAndSignerTask(CriticalTask):
+     autoretry_for = (
+        sqlalchemy.exc.DatabaseError,
+        sqlalchemy.exc.TimeoutError,
+        SignerError,
+        ) 
+
+class CriticalWeb3AndSignerTask(CriticalTask):
+    autoretry_for = (
+        requests.exceptions.ConnectionError,
+        SignerError,
         )

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         0,
-        'alpha.39',
+        'alpha.41',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=
+DEBUG=0

apps/cic-eth/config/tasks.ini

@@ -1,3 +1,3 @@
 [tasks]
 transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
-trace_queue_status = 
+trace_queue_status = 1

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
-cic-base~=0.1.1a10
+cic-base~=0.1.1a20
 web3==5.12.2
 celery==4.4.7
 crypto-dev-signer~=0.4.13rc4

apps/cic-eth/tests/tasks/test_debug.py

@@ -1,29 +0,0 @@
-# external imports
-import celery
-
-# local imports
-from cic_eth.db.models.debug import Debug
-
-
-def test_debug_alert(
-        init_database,
-        celery_session_worker,
-        ):
-
-    s = celery.signature(
-            'cic_eth.admin.debug.alert',
-            [
-                'foo',
-                'bar',
-                'baz',
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get()
-    assert r == 'foo'
-
-    q = init_database.query(Debug)
-    q = q.filter(Debug.tag=='bar')
-    o = q.first()
-    assert o.description == 'baz'

apps/cic-notify/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-notify
-version= attr: cic_notify.version.__version_string__
+version= 0.4.0a2
 description = CIC notifications service
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -21,10 +21,10 @@ from confini import Config
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = os.path.join(xdg_config_home, 'cli-ussd')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser(description='CLI tool to interface a Sempo USSD session')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 #argparser.add_argument('-d', type=str, default='local', help='deployment name to interface (config root subdirectory)')
 argparser.add_argument('--host', type=str, default='localhost')
 argparser.add_argument('--port', type=int, default=9000)

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.2')
+version = (0, 3, 0, 'alpha.5')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/requirements.txt

@@ -1,3 +1,4 @@
+cic-types==0.1.0a8
 alembic==1.4.2
 amqp==2.6.1
 attrs==20.2.0
@@ -6,12 +7,10 @@ betterpath==0.2.2
 billiard==3.6.3.0
 celery==4.4.7
 cffi==1.14.3
-chainlib~=0.0.1a15
-cic-eth==0.10.0a39
-cic-notify~=0.4.0a2
-cic-types==0.1.0a8
+cic-eth~=0.10.0a22
+cic-notify==0.3.1
 click==7.1.2
-confini~=0.3.6rc3
+confini~=0.3.6a1
 cryptography==3.2.1
 faker==4.17.1
 iniconfig==1.1.1
@@ -36,7 +35,6 @@ python-i18n==0.3.9
 pytz==2020.1
 PyYAML==5.3.1
 redis==3.5.3
-requests==2.24.0
 semver==2.13.0
 six==1.15.0
 SQLAlchemy==1.3.20

apps/contract-migration/docker/Dockerfile

@@ -107,10 +107,10 @@ RUN cd cic-bancor/python && \
 
 
 RUN apt-get install -y cargo
-ARG cic_base_version=0.1.1a10
+ARG cic_base_version=0.1.1a23
 RUN pip install  --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version
 
-ARG cic_registry_version=0.5.3a22
+ARG cic_registry_version=0.5.3a24
 RUN pip install  --extra-index-url $pip_extra_index_url  cic-registry==$cic_registry_version
 
 WORKDIR /root
@@ -127,5 +127,6 @@ COPY contract-migration/from_env.sh from_env.sh
 COPY contract-migration/seed_cic_eth.sh seed_cic_eth.sh
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
+COPY contract-migration/envlist .
 
-ENTRYPOINT [ "/bin/bash" ]
+# ENTRYPOINT [ "/bin/bash" ]

apps/contract-migration/envlist

@@ -0,0 +1,61 @@
+SYNCER_LOOP_INTERVAL
+SSL_ENABLE_CLIENT
+SSL_CERT_FILE
+SSL_KEY_FILE
+SSL_PASSWORD
+SSL_CA_FILE
+BANCOR_DIR
+REDIS_HOST
+REDIS_PORT
+REDIS_DB
+PGP_EXPORTS_DIR
+PGP_PRIVATEKEY_FILE
+PGP_PASSPHRASE
+DATABASE_USER
+DATABASE_PASSWORD
+DATABASE_NAME
+DATABASE_HOST
+DATABASE_PORT
+DATABASE_ENGINE
+DATABASE_DRIVER
+DATABASE_DEBUG
+TASKS_AFRICASTALKING
+TASKS_SMS_DB
+TASKS_LOG
+TASKS_TRACE_QUEUE_STATUS
+TASKS_TRANSFER_CALLBACKS
+DEV_MNEMONIC
+DEV_ETH_RESERVE_ADDRESS
+DEV_ETH_ACCOUNTS_INDEX_ADDRESS
+DEV_ETH_RESERVE_AMOUNT
+DEV_ETH_ACCOUNT_BANCOR_DEPLOYER
+DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+DEV_ETH_ACCOUNT_GAS_PROVIDER
+DEV_ETH_ACCOUNT_RESERVE_OWNER
+DEV_ETH_ACCOUNT_RESERVE_MINTER
+DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER
+DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
+DEV_ETH_ACCOUNT_SARAFU_OWNER
+DEV_ETH_ACCOUNT_SARAFU_GIFTER
+DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
+DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
+DEV_ETH_SARAFU_TOKEN_NAME
+DEV_ETH_SARAFU_TOKEN_SYMBOL
+DEV_ETH_SARAFU_TOKEN_DECIMALS
+DEV_ETH_SARAFU_TOKEN_ADDRESS
+DEV_PGP_PUBLICKEYS_ACTIVE_FILE
+DEV_PGP_PUBLICKEYS_TRUSTED_FILE
+DEV_PGP_PUBLICKEYS_ENCRYPT_FILE
+CIC_REGISTRY_ADDRESS
+CIC_APPROVAL_ESCROW_ADDRESS
+CIC_TOKEN_INDEX_ADDRESS
+CIC_ACCOUNTS_INDEX_ADDRESS
+CIC_DECLARATOR_ADDRESS
+CIC_CHAIN_SPEC
+ETH_PROVIDER
+ETH_ABI_DIR
+SIGNER_SOCKET_PATH
+SIGNER_SECRET
+CELERY_BROKER_URL
+CELERY_RESULT_URL
+META_PROVIDER

apps/contract-migration/reset.sh

@@ -64,7 +64,7 @@ export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
 EOF
 
-cat $CIC_DATA_DIR/envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
+cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
 # popd
 
 set +a

apps/contract-migration/scripts/create_import_users.py

@@ -191,9 +191,16 @@ if __name__ == '__main__':
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
 
-    for i in range(user_count):
-    
-        (eth, phone, o) = gen()
+    i = 0
+    while i < user_count:
+        eth = None
+        phone = None
+        o = None
+        try:
+            (eth, phone, o) = gen()
+        except Exception as e:
+            logg.warning('generate failed, trying anew: {}'.format(e))
+            continue
         uid = eth[2:].upper()
 
         print(o)
@@ -212,5 +219,7 @@ if __name__ == '__main__':
         amount = genAmount()
         fa.write('{},{}\n'.format(eth,amount))
         logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
+        
+        i += 1
 
     fa.close()

apps/contract-migration/scripts/import_users.py

@@ -38,7 +38,7 @@ argparser.add_argument('--redis-host-callback', dest='redis_host_callback', defa
 argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
 argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
 argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches')
-argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
+argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout')
 argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
 argparser.add_argument('-v', action='store_true', help='Be verbose')
 argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@@ -97,6 +97,7 @@ def register_eth(i, u):
         callback_queue=args.q,
         )
     t = api.create_account(register=True)
+    logg.debug('register {} -> {}'.format(u, t))
 
     while True:
         ps.get_message()
@@ -112,11 +113,11 @@ def register_eth(i, u):
             r = json.loads(m['data'])
             address = r['result']
             break
-        except TypeError as e:
+        except Exception as e:
             if m == None:
-                logg.critical('empty response from redis callback (did the service crash?)')
+                logg.critical('empty response from redis callback (did the service crash?) {}'.format(e))
             else:
-                logg.critical('unexpected response from redis callback: {}'.format(m))
+                logg.critical('unexpected response from redis callback: {} {}'.format(m, e))
             sys.exit(1)
         logg.debug('[{}] register eth {} {}'.format(i, u, address))
 

apps/contract-migration/scripts/requirements.txt

@@ -1,3 +1,3 @@
-cic-base[full_graph]==0.1.1a12
-cic-eth==0.10.0a38
+cic-base[full_graph]==0.1.1a23
+cic-eth==0.10.0a41
 cic-types==0.1.0a8

apps/contract-migration/seed_cic_eth.sh

@@ -31,8 +31,7 @@ set -e
 set -a
 
 # We need to not install these here...
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a38 chainlib==0.0.1a19 cic-contracts==0.0.2a2
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10
+pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a41 chainlib==0.0.1a21 cic-contracts==0.0.2a2
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER

ci_templates/.cic-template.yml

@@ -1,54 +1,35 @@
-image: docker:19.03.13
+image:
+  name: gcr.io/kaniko-project/executor:debug
+  entrypoint: [""]
 
 variables:
-    # docker host
-    DOCKER_HOST: tcp://docker:2376
-    # container, thanks to volume mount from config.toml
-    DOCKER_TLS_CERTDIR: "/certs"
-    # These are usually specified by the entrypoint, however the
-    # Kubernetes executor doesn't run entrypoints
-    # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125
-    DOCKER_TLS_VERIFY: 1
-    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-    # We are building these from the apps dir to easily share the requirements file there.
-    # It would be nicer to build from the app dir context. TODO figure out a nice way to do this in local DOCKER_TLS_VERIFY
-    CONTEXT: apps/
-
-services:
-  - docker:19.03.13-dind
-
-before_script:
-  - docker info
+  KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
+  CONTEXT: $CI_PROJECT_DIR/apps/
 
 .py_build_merge_request:
   stage: build
-  before_script:
-    - cd $CONTEXT
   variables:
-    CI_DEBUG_TRACE: "true"
-    IMAGE_TAG: $APP_NAME:$CI_COMMIT_SHORT_SHA
+    - CI_DEBUG_TRACE: "true"
   script:
-    - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
 
 .py_build_push:
-    stage: build
-    before_script:
-      - cd $CONTEXT
-      - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" $CI_REGISTRY --password-stdin
-    variables:
-        CI_DEBUG_TRACE: "true"
-        IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
-        LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
-    script:
-      - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
-      - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH .
-      - docker push $IMAGE_TAG
-      - docker tag $IMAGE_TAG $LATEST_TAG
-      - docker push $LATEST_TAG
-    rules:
+  stage: build
+  variables:
+      IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
+      LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest
+  script:
+    - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
+  rules:
       - if: $CI_COMMIT_BRANCH == "master"
         when: always
 

docker-compose.yml

@@ -162,7 +162,43 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        /usr/local/bin/cic-cache-tracker -vv
+        /usr/local/bin/cic-cache-trackerd -vv
+    volumes:
+      - contract-config:/tmp/cic/config/:ro
+
+  cic-cache-tasker:
+    build:
+      context: apps
+      dockerfile: cic-cache/docker/Dockerfile
+    environment:
+      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
+      ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
+      DATABASE_USER: ${DATABASE_USER:-grassroots}
+      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
+      DATABASE_HOST: ${DATABASE_HOST:-postgres}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
+      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
+      DATABASE_DEBUG: 1
+      ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
+      CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
+      CELERY_BROKER_URL: redis://redis:6379
+      CELERY_RESULT_URL: redis://redis:6379
+    deploy: 
+      restart_policy:
+        condition: on-failure
+    depends_on:
+      - redis
+      - postgres
+      - eth
+    command:
+      - /bin/bash
+      - -c
+      - |
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
+        /usr/local/bin/cic-cache-taskerd -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -192,7 +228,7 @@ services:
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         "/usr/local/bin/uwsgi" \
-        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \
+        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \
         --http :8000 \
         --pyargv -vv
 
@@ -238,7 +274,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_tasker.sh -q cic-eth 
+        ./start_tasker.sh -q cic-eth  -vv
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker: