Rehabilitate CLI and API after nonce changes

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -10,12 +10,15 @@ from cic_registry import zero_address
 from cic_eth.db.enum import LockEnum
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.lock import Lock
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        )
 from cic_eth.error import LockedError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
@@ -33,7 +36,7 @@ def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
@@ -51,7 +54,7 @@ def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set send lock
 
@@ -67,7 +70,7 @@ def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock_send(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset send lock
 
@@ -83,7 +86,7 @@ def unlock_send(chained_input, chain_str, address=zero_address):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set queue direct lock
 
@@ -99,7 +102,7 @@ def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def unlock_queue(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset queue direct lock
 
@@ -115,7 +118,7 @@ def unlock_queue(chained_input, chain_str, address=zero_address):
     return chained_input
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_str, lock_flags, address=None):
     session = SessionBase.create_session()
     r = Lock.check(chain_str, lock_flags, address=zero_address, session=session)

apps/cic-eth/cic_eth/api/api_task.py

@@ -82,6 +82,7 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -143,6 +144,7 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
@@ -340,11 +342,6 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_nonce = celery.signature(
-                'cic_eth.eth.tx.reserve_nonce',
-                [],
-                queue=self.queue,
-                )
         s_account = celery.signature(
                 'cic_eth.eth.account.create',
                 [
@@ -352,12 +349,18 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_nonce.link(s_account)
-        s_check.link(s_nonce)
+        s_check.link(s_account)
         if self.callback_param != None:
             s_account.link(self.callback_success)
 
         if register:
+            s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
+                [
+                    'ACCOUNTS_INDEX_WRITER',
+                    ],
+                queue=self.queue,
+                )
             s_register = celery.signature(
                 'cic_eth.eth.account.register', 
                 [
@@ -365,7 +368,8 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-            s_account.link(s_register)
+            s_nonce.link(s_register)
+            s_account.link(s_nonce)
 
         t = s_check.apply_async(queue=self.queue)
         return t
@@ -390,7 +394,9 @@ class Api:
                 )
         s_nonce = celery.signature(
                 'cic_eth.eth.tx.reserve_nonce',
-                [],
+                [
+                    'GAS_GIFTER',
+                    ],
                 queue=self.queue,
                 )
         s_refill = celery.signature(

apps/cic-eth/cic_eth/error.py

@@ -73,3 +73,10 @@ class SignerError(Exception):
 
     """
     pass
+
+
+class EthError(Exception):
+    """Exception raised when unspecified error from evm node is encountered
+
+    """
+    pass

apps/cic-eth/cic_eth/eth/gas.py

@@ -54,6 +54,7 @@ class GasOracle():
         """
         session = SessionBase.create_session()
         a = AccountRole.get_address('GAS_GIFTER', session)
+        logg.debug('gasgifter {}'.format(a))
         session.close()
         return a
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -14,6 +14,7 @@ from cic_eth.db import Otx, SessionBase
 from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.nonce import NonceReservation
 from cic_eth.db.models.lock import Lock
+from cic_eth.db.models.role import AccountRole
 from cic_eth.db.enum import (
         LockEnum,
         StatusBits,
@@ -38,6 +39,7 @@ from cic_eth.task import (
         CriticalWeb3Task,
         CriticalWeb3AndSignerTask,
         CriticalSQLAlchemyAndSignerTask,
+        CriticalSQLAlchemyAndWeb3Task,
         )
 
 celery_app = celery.current_app
@@ -47,7 +49,7 @@ MAX_NONCE_ATTEMPTS = 3
 
 
 # TODO this function is too long
-@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
 def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=None):
     """Check the gas level of the sender address of a transaction.
 
@@ -75,6 +77,9 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
             if address == None:
                 address = o['address']
 
+    if not web3.Web3.isChecksumAddress(address):
+        raise ValueError('invalid address {}'.format(address))
+
     chain_spec = ChainSpec.from_chain_str(chain_str)
 
     queue = self.request.delivery_info['routing_key']
@@ -83,7 +88,12 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
     c = RpcClient(chain_spec)
 
     # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
-    balance = c.w3.eth.getBalance(address) 
+    balance = 0
+    try:
+        balance = c.w3.eth.getBalance(address) 
+    except ValueError as e:
+        raise EthError('balance call for {}'.format())
+
     logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required))
 
     if gas_required > balance:
@@ -590,11 +600,23 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
 
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
-def reserve_nonce(self, chained_input, address=None):
+def reserve_nonce(self, chained_input, signer=None):
     session = SessionBase.create_session()
 
-    if address == None:
+    address = None
+    if signer == None:
         address = chained_input
+        logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
+    else:
+        if web3.Web3.isChecksumAddress(signer):
+            address = signer
+            logg.debug('explicit address for reserve nonce {}'.format(signer))
+        else:
+            address = AccountRole.get_address(signer, session=session)
+            logg.debug('role for reserve nonce {} -> {}'.format(signer, address))
+
+    if not web3.Web3.isChecksumAddress(address):
+        raise ValueError('invalid result when resolving address for nonce {}'.format(address))
 
     root_id = self.request.root_id
     nonce = NonceReservation.next(address, root_id)

apps/cic-eth/cic_eth/runnable/create.py

@@ -5,6 +5,7 @@ import os
 import logging
 import uuid
 import json
+from xdg.BaseDirectory import xdg_config_home
 
 import celery
 from cic_eth.api import Api

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -25,7 +25,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
 
 
 default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')

apps/cic-eth/cic_eth/runnable/resend.py

@@ -22,7 +22,7 @@ logg = logging.getLogger()
 logging.getLogger('web3').setLevel(logging.WARNING)
 logging.getLogger('urllib3').setLevel(logging.WARNING)
 
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 
 argparser = argparse.ArgumentParser()

apps/cic-eth/cic_eth/runnable/view.py

@@ -37,7 +37,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
 
 
 default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)')

apps/cic-eth/cic_eth/task.py

@@ -6,7 +6,10 @@ import celery
 import sqlalchemy
 
 # local imports
-from cic_eth.error import SignerError
+from cic_eth.error import (
+        SignerError,
+        EthError,
+        )
 
 
 class CriticalTask(celery.Task):
@@ -33,6 +36,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         requests.exceptions.ConnectionError,
+        EthError,
         )
 
 class CriticalSQLAlchemyAndSignerTask(CriticalTask):

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         0,
-        'alpha.39',
+        'alpha.41',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
-cic-base~=0.1.1a10
+cic-base~=0.1.1a20
 web3==5.12.2
 celery==4.4.7
 crypto-dev-signer~=0.4.13rc4

apps/cic-notify/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-notify
-version= attr: cic_notify.version.__version_string__
+version= 0.4.0a2
 description = CIC notifications service
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -21,10 +21,10 @@ from confini import Config
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = os.path.join(xdg_config_home, 'cli-ussd')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser(description='CLI tool to interface a Sempo USSD session')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 #argparser.add_argument('-d', type=str, default='local', help='deployment name to interface (config root subdirectory)')
 argparser.add_argument('--host', type=str, default='localhost')
 argparser.add_argument('--port', type=int, default=9000)
@@ -64,7 +64,7 @@ else:
     ssl = True
 
 
-if __name__ == "__main__":
+def main():
 
     # TODO: improve url building 
     url = 'http'
@@ -101,3 +101,7 @@ if __name__ == "__main__":
         state = response_data[:3]
         out = response_data[4:]
         print(out)
+
+
+if __name__ == "__main__":
+    main()

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.1')
+version = (0, 3, 0, 'alpha.5')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/emulator/browser/index.html

@@ -16,9 +16,16 @@ div#session {
 		<textarea id="monitor" disabled="1"></textarea>
 		<div id="login">
 			<label for="user">API username</label>
-			<input type="text" id="user" name="user" type="text" /><br/>
+			<input type="text" id="user" name="user" />  
 			<label for="user">API password</label>
-			<input type="text" id="pass" name="pass" type="text" /><br/>
+			<input type="text" id="pass" name="pass" /> <br/>
+
+			<label for="host">API host</label>
+			 <input type="text" id="host" name="host" />
+			<label for="host">API port</label>
+			 <input type="text" id="port" name="port" />
+			<label for="host">SSL</label>
+			<input type="checkbox" id="ssl" name="ssl" checked="1"/> <br/>
 			<hr/>
 			<input type="text" id="phone" /> <button onclick="setPhone(document.getElementById('phone').value);" id="send_phone">set phone number</button> 
 		</div>

apps/cic-ussd/emulator/browser/ussd.js

@@ -1,10 +1,12 @@
-//var proto = 'http';
-//var host = 'localhost:9000';
+var ssl = false;
+var host = 'localhost';
+var port = 9000;
 var proto = 'https';
 var host = 'staging.sarafu.network';
-var user = 'admin_bert_token_inc.';
-var pass = '197781ed60bf16d5dc12d84e3df37e35';
-var serviceCode = '*483*061#';
+var user = 'foo';
+var pass = 'bar';
+var path = '/';
+var serviceCode = '*483*46#';
 
 // cheekily stolen from https://www.tutorialspoint.com/how-to-create-guid-uuid-in-javascript
 function createUUID() {
@@ -23,9 +25,17 @@ function send(s) {
 	document.getElementById('send_input').disabled = true;
 	var xhr = new XMLHttpRequest();
 	xhr.responseType = 'text';
-	current_user = document.getElementById('user').value;
-	current_pass = document.getElementById('pass').value;
-	xhr.open('POST', proto + '://' + host + '/api/v1/ussd/kenya?username=' + current_user + '&password=' + current_pass, true);
+	const current_user = document.getElementById('user').value;
+	const current_pass = document.getElementById('pass').value;
+	const current_host = document.getElementById('host').value;
+	const current_port = document.getElementById('port').value;
+	let current_scheme = 'http';
+	if (document.getElementById('ssl').checked) {
+		current_scheme += 's';
+	}
+	const url = current_scheme + '://' + current_host + ':' + current_port + '?username=' + current_user + '&password=' + current_pass
+	console.debug('connecting to', url);
+	xhr.open('POST', url, true);
 	xhr.setRequestHeader('Content-Type', 'application/json');
 	data = {
 		sessionId: uuid,
@@ -106,6 +116,8 @@ function abort() {
 window.addEventListener('load', () => {
 	document.getElementById('user').value = user;
 	document.getElementById('pass').value = pass;
+	document.getElementById('host').value = host;
+	document.getElementById('port').value = port;
 	document.getElementById('phone').addEventListener('keyup', (e) => {
 		if (e.keyCode == '13') {
 			document.getElementById('input').value = '';

apps/cic-ussd/requirements.txt

@@ -1,49 +1,21 @@
+cic_base[full-graph]~=0.1.1a17
 alembic==1.4.2
-amqp==2.6.1
-attrs==20.2.0
 bcrypt==3.2.0
-betterpath==0.2.2
-billiard==3.6.3.0
 celery==4.4.7
-cffi==1.14.3
-chainlib~=0.0.1a15
-cic-eth==0.10.0a39
-cic-notify==0.3.1
+chainlib~=0.0.1a20
+cic-eth~=0.10.0a40
+cic-notify~=0.4.0a2
 cic-types==0.1.0a8
-click==7.1.2
-confini==0.3.5
+confini~=0.3.6rc3
 cryptography==3.2.1
 faker==4.17.1
-iniconfig==1.1.1
-kombu==4.6.11
-Mako==1.1.3
-MarkupSafe==1.1.1
-mirakuru==2.3.0
-more-itertools==8.5.0
-packaging==20.4
 phonenumbers==8.12.12
-pluggy==0.13.1
-port-for==0.4
-psutil==5.7.3
 psycopg2==2.8.6
-py==1.9.0
-pycparser==2.20
-pyparsing==2.4.7
-python-dateutil==2.8.1
-python-editor==1.0.4
 python-gnupg==0.4.6
 python-i18n==0.3.9
-pytz==2020.1
-PyYAML==5.3.1
 redis==3.5.3
-requests==2.24.0
 semver==2.13.0
-six==1.15.0
 SQLAlchemy==1.3.20
 tinydb==4.2.0
-toml==0.10.1
 transitions==0.8.4
 uWSGI==2.0.19.1
-vcversioner==2.16.0.0
-vine==1.3.0
-zope.interface==5.1.2

apps/cic-ussd/setup.cfg

@@ -45,3 +45,4 @@ scripts =
 [options.entry_points]
 console_scripts =
 	cic-ussd-tasker = cic_ussd.runnable.tasker:main
+	cic-ussd-client = cic_ussd.runnable.client:main

apps/contract-migration/docker/Dockerfile

@@ -107,10 +107,10 @@ RUN cd cic-bancor/python && \
 
 
 RUN apt-get install -y cargo
-ARG cic_base_version=0.1.1a10
+ARG cic_base_version=0.1.1a23
 RUN pip install  --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version
 
-ARG cic_registry_version=0.5.3a22
+ARG cic_registry_version=0.5.3a24
 RUN pip install  --extra-index-url $pip_extra_index_url  cic-registry==$cic_registry_version
 
 WORKDIR /root

apps/contract-migration/scripts/create_import_users.py

@@ -191,9 +191,16 @@ if __name__ == '__main__':
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
 
-    for i in range(user_count):
-    
-        (eth, phone, o) = gen()
+    i = 0
+    while i < user_count:
+        eth = None
+        phone = None
+        o = None
+        try:
+            (eth, phone, o) = gen()
+        except Exception as e:
+            logg.warning('generate failed, trying anew: {}'.format(e))
+            continue
         uid = eth[2:].upper()
 
         print(o)
@@ -213,4 +220,6 @@ if __name__ == '__main__':
         fa.write('{},{}\n'.format(eth,amount))
         logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
         
+        i += 1
+
     fa.close()

apps/contract-migration/scripts/import_meta.js

@@ -74,7 +74,7 @@ new cic.PGPKeyStore(
 	importMeta,
 );
 
-const batchSize = 50;
+const batchSize = 16;
 const batchDelay = 1000;
 const total = parseInt(process.argv[3]);
 const workDir = path.join(process.argv[2], 'meta');

apps/contract-migration/scripts/import_users.py

@@ -38,7 +38,7 @@ argparser.add_argument('--redis-host-callback', dest='redis_host_callback', defa
 argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
 argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
 argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches')
-argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
+argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout')
 argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
 argparser.add_argument('-v', action='store_true', help='Be verbose')
 argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@@ -97,6 +97,7 @@ def register_eth(i, u):
         callback_queue=args.q,
         )
     t = api.create_account(register=True)
+    logg.debug('register {} -> {}'.format(u, t))
 
     while True:
         ps.get_message()
@@ -112,11 +113,11 @@ def register_eth(i, u):
             r = json.loads(m['data'])
             address = r['result']
             break
-        except TypeError as e:
+        except Exception as e:
             if m == None:
-                logg.critical('empty response from redis callback (did the service crash?)')
+                logg.critical('empty response from redis callback (did the service crash?) {}'.format(e))
             else:
-                logg.critical('unexpected response from redis callback: {}'.format(m))
+                logg.critical('unexpected response from redis callback: {} {}'.format(m, e))
             sys.exit(1)
         logg.debug('[{}] register eth {} {}'.format(i, u, address))
 

apps/contract-migration/scripts/requirements.txt

@@ -1,3 +1,3 @@
-cic-base[full_graph]==0.1.1a12
-cic-eth==0.10.0a38
+cic-base[full_graph]==0.1.1a23
+cic-eth==0.10.0a41
 cic-types==0.1.0a8

apps/contract-migration/seed_cic_eth.sh

@@ -31,8 +31,7 @@ set -e
 set -a
 
 # We need to not install these here...
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a38 chainlib==0.0.1a19 cic-contracts==0.0.2a2
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10
+pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a41 chainlib==0.0.1a21 cic-contracts==0.0.2a2
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER

docker-compose.yml

@@ -238,7 +238,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_tasker.sh -q cic-eth 
+        ./start_tasker.sh -q cic-eth  -vv
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker: