Fix type typo for task base

apps/cic-eth/cic_eth/error.py

@@ -66,3 +66,10 @@ class LockedError(Exception):
 
     """
     pass
+
+
+class SignerError(Exception):
+    """Exception raised when signer is unavailable or generates an error
+
+    """
+    pass

apps/cic-eth/cic_eth/eth/account.py

@@ -21,8 +21,14 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.tx import TxCache
 from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.error import RoleMissingError
-from cic_eth.task import CriticalSQLAlchemyTask
+from cic_eth.error import (
+        RoleMissingError,
+        SignerError,
+        )
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        CriticalSQLAlchemyAndSignerTask,
+        )
 
 #logg = logging.getLogger(__name__)
 logg = logging.getLogger()
@@ -139,7 +145,7 @@ def unpack_gift(data):
      
 
 # TODO: Separate out nonce initialization task
-@celery_app.task(base=CriticalSQLAlchemyTask)
+@celery_app.task(base=CriticalSQLAlchemyAndSignerTask)
 def create(password, chain_str):
     """Creates and stores a new ethereum account in the keystore.
 
@@ -154,7 +160,13 @@ def create(password, chain_str):
     """
     chain_spec = ChainSpec.from_chain_str(chain_str)
     c = RpcClient(chain_spec)
-    a = c.w3.eth.personal.new_account(password)
+    a = None
+    try:
+        a = c.w3.eth.personal.new_account(password)
+    except FileNotFoundError:
+        pass
+    if a == None:
+        raise SignerError('create account')
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -165,7 +177,7 @@ def create(password, chain_str):
     return a
 
 
-@celery_app.task(bind=True, throws=(RoleMissingError,), base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, throws=(RoleMissingError,), base=CriticalSQLAlchemyAndSignerTask)
 def register(self, account_address, chain_str, writer_address=None):
     """Creates a transaction to add the given address to the accounts index.
 
@@ -215,7 +227,7 @@ def register(self, account_address, chain_str, writer_address=None):
     return account_address
 
 
-@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def gift(self, account_address, chain_str):
     """Creates a transaction to invoke the faucet contract for the given address.
 

apps/cic-eth/cic_eth/eth/task.py

@@ -8,6 +8,7 @@ from cic_registry.chain import ChainSpec
 # local imports
 from cic_eth.eth import RpcClient
 from cic_eth.queue.tx import create as queue_create
+from cic_eth.error import SignerError
 
 celery_app = celery.current_app
 logg = celery_app.log.get_default_logger()
@@ -26,7 +27,13 @@ def sign_tx(tx, chain_str):
     """
     chain_spec = ChainSpec.from_chain_str(chain_str)
     c = RpcClient(chain_spec)
-    tx_transfer_signed = c.w3.eth.sign_transaction(tx) 
+    tx_transfer_signed = None
+    try:
+        tx_transfer_signed = c.w3.eth.sign_transaction(tx)
+    except FileNotFoundError:
+        pass
+    if tx_transfer_signed == None:
+        raise SignerError('sign tx')
     logg.debug('tx_transfer_signed {}'.format(tx_transfer_signed))
     tx_hash = c.w3.keccak(hexstr=tx_transfer_signed['raw'])
     tx_hash_hex = tx_hash.hex()

apps/cic-eth/cic_eth/eth/token.py

@@ -24,6 +24,7 @@ from cic_eth.ext.address import translate_address
 from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
+        CriticalSQLAlchemyAndSignerTask,
     )
 
 celery_app = celery.current_app
@@ -212,7 +213,7 @@ def balance(tokens, holder_address, chain_str):
     return tokens
 
 
-@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_str):
     """Transfer ERC20 tokens between addresses
 
@@ -268,7 +269,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_str):
     return tx_hash_hex
 
 
-@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def approve(self, tokens, holder_address, spender_address, value, chain_str):
     """Approve ERC20 transfer on behalf of holder address
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -36,6 +36,8 @@ from cic_eth.admin.ctrl import lock_send
 from cic_eth.task import (
         CriticalSQLAlchemyTask,
         CriticalWeb3Task,
+        CriticalWeb3AndSignerTask,
+        CriticalSQLAlchemyAndSignerTask,
         )
 
 celery_app = celery.current_app
@@ -418,7 +420,7 @@ def send(self, txs, chain_str):
 
 # TODO: if this method fails the nonce will be out of sequence. session needs to be extended to include the queue create, so that nonce is rolled back if the second sql query fails. Better yet, split each state change into separate tasks.
 # TODO: method is too long, factor out code for clarity
-@celery_app.task(bind=True, throws=(web3.exceptions.TransactionNotFound,), base=CriticalWeb3Task)
+@celery_app.task(bind=True, throws=(web3.exceptions.TransactionNotFound,), base=CriticalWeb3AndSignerTask)
 def refill_gas(self, recipient_address, chain_str):
     """Executes a native token transaction to fund the recipient's gas expenditures.
 
@@ -511,7 +513,7 @@ def refill_gas(self, recipient_address, chain_str):
     return tx_send_gas_signed['raw']
 
 
-@celery_app.task(bind=True)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_factor=1.1):
     """Create a new transaction from an existing one with same nonce and higher gas price.
 

apps/cic-eth/cic_eth/task.py

@@ -5,6 +5,9 @@ import requests
 import celery
 import sqlalchemy
 
+# local imports
+from cic_eth.error import SignerError
+
 
 class CriticalTask(celery.Task):
     retry_jitter = True
@@ -31,3 +34,16 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
         sqlalchemy.exc.TimeoutError,
         requests.exceptions.ConnectionError,
         )
+
+class CriticalSQLAlchemyAndSignerTask(CriticalTask):
+     autoretry_for = (
+        sqlalchemy.exc.DatabaseError,
+        sqlalchemy.exc.TimeoutError,
+        SignerError,
+        ) 
+
+class CriticalWeb3AndSignerTask(CriticalTask):
+    autoretry_for = (
+        requests.exceptions.ConnectionError,
+        SignerError,
+        )

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         0,
-        'alpha.39',
+        'alpha.38',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=
+DEBUG=0

apps/cic-eth/config/tasks.ini

@@ -1,3 +1,3 @@
 [tasks]
 transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
-trace_queue_status = 
+trace_queue_status = 1

apps/cic-eth/tests/tasks/test_debug.py

@@ -1,29 +0,0 @@
-# external imports
-import celery
-
-# local imports
-from cic_eth.db.models.debug import Debug
-
-
-def test_debug_alert(
-        init_database,
-        celery_session_worker,
-        ):
-
-    s = celery.signature(
-            'cic_eth.admin.debug.alert',
-            [
-                'foo',
-                'bar',
-                'baz',
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get()
-    assert r == 'foo'
-
-    q = init_database.query(Debug)
-    q = q.filter(Debug.tag=='bar')
-    o = q.first()
-    assert o.description == 'baz'

apps/cic-ussd/.config_client/app.ini

@@ -1,2 +0,0 @@
-[app]
-service_code = *483*46#

apps/cic-ussd/.config_client/client.ini

@@ -1,4 +0,0 @@
-[client]
-host = 
-port = 
-ssl =

apps/cic-ussd/.config_client/ussd.ini

@@ -1,3 +0,0 @@
-[ussd]
-user =
-pass =

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -1,107 +0,0 @@
-#!/usr/bin/python3
-
-# Author: Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
-# Description: interactive console for Sempo USSD session
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import os
-import sys
-import uuid
-import json
-import argparse
-import logging
-import urllib
-from xdg.BaseDirectory import xdg_config_home
-from urllib import request
-
-# third-party imports
-from confini import Config
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-config_dir = os.path.join(xdg_config_home, 'cli-ussd')
-
-argparser = argparse.ArgumentParser(description='CLI tool to interface a Sempo USSD session')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
-#argparser.add_argument('-d', type=str, default='local', help='deployment name to interface (config root subdirectory)')
-argparser.add_argument('--host', type=str, default='localhost')
-argparser.add_argument('--port', type=int, default=9000)
-argparser.add_argument('--nossl', help='do not use ssl (careful)', action='store_true')
-argparser.add_argument('phone', help='phone number for USSD session')
-argparser.add_argument('-v', help='be verbose', action='store_true')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-
-args = argparser.parse_args(sys.argv[1:])
-
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-#config_dir = os.path.join(args.c, args.d)
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-
-config = Config(config_dir)
-config.process()
-logg.debug('config loaded from {}'.format(config_dir))
-
-host = config.get('CLIENT_HOST')
-port = config.get('CLIENT_PORT')
-ssl = config.get('CLIENT_SSL')
-
-if host == None:
-    host = args.host
-if port == None:
-    port = args.port
-if ssl == None:
-    ssl = not args.nossl
-elif ssl == 0:
-    ssl = False
-else:
-    ssl = True
-
-
-def main():
-
-    # TODO: improve url building 
-    url = 'http'
-    if ssl:
-        url += 's'
-    url += '://{}:{}'.format(host, port)
-    url += '/?username={}&password={}'.format(config.get('USSD_USER'), config.get('USSD_PASS'))
-
-    logg.info('service url {}'.format(url))
-    logg.info('phone {}'.format(args.phone))
-
-    session = uuid.uuid4().hex
-    data = {
-            'sessionId': session,
-            'serviceCode': config.get('APP_SERVICE_CODE'),
-            'phoneNumber': args.phone,
-            'text': config.get('APP_SERVICE_CODE'),
-        }
-
-    state = "_BEGIN"
-    while state != "END":
-
-        if state != "_BEGIN":
-            user_input = input('next> ')
-            data['text'] = user_input
-
-        req = urllib.request.Request(url)
-        data_str = json.dumps(data)
-        data_bytes = data_str.encode('utf-8')
-        req.add_header('Content-Type', 'application/json')
-        req.data = data_bytes
-        response = urllib.request.urlopen(req)
-        response_data = response.read().decode('utf-8')
-        state = response_data[:3]
-        out = response_data[4:]
-        print(out)
-
-
-if __name__ == "__main__":
-    main()

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.2')
+version = (0, 3, 0, 'alpha.1')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/emulator/browser/index.html

@@ -16,16 +16,9 @@ div#session {
 		<textarea id="monitor" disabled="1"></textarea>
 		<div id="login">
 			<label for="user">API username</label>
-			<input type="text" id="user" name="user" />  
+			<input type="text" id="user" name="user" type="text" /><br/>
 			<label for="user">API password</label>
-			<input type="text" id="pass" name="pass" /> <br/>
-
-			<label for="host">API host</label>
-			 <input type="text" id="host" name="host" />
-			<label for="host">API port</label>
-			 <input type="text" id="port" name="port" />
-			<label for="host">SSL</label>
-			<input type="checkbox" id="ssl" name="ssl" checked="1"/> <br/>
+			<input type="text" id="pass" name="pass" type="text" /><br/>
 			<hr/>
 			<input type="text" id="phone" /> <button onclick="setPhone(document.getElementById('phone').value);" id="send_phone">set phone number</button> 
 		</div>

apps/cic-ussd/emulator/browser/ussd.js

@@ -1,12 +1,10 @@
-var ssl = false;
-var host = 'localhost';
-var port = 9000;
+//var proto = 'http';
+//var host = 'localhost:9000';
 var proto = 'https';
 var host = 'staging.sarafu.network';
-var user = 'foo';
-var pass = 'bar';
-var path = '/';
-var serviceCode = '*483*46#';
+var user = 'admin_bert_token_inc.';
+var pass = '197781ed60bf16d5dc12d84e3df37e35';
+var serviceCode = '*483*061#';
 
 // cheekily stolen from https://www.tutorialspoint.com/how-to-create-guid-uuid-in-javascript
 function createUUID() {
@@ -25,17 +23,9 @@ function send(s) {
 	document.getElementById('send_input').disabled = true;
 	var xhr = new XMLHttpRequest();
 	xhr.responseType = 'text';
-	const current_user = document.getElementById('user').value;
-	const current_pass = document.getElementById('pass').value;
-	const current_host = document.getElementById('host').value;
-	const current_port = document.getElementById('port').value;
-	let current_scheme = 'http';
-	if (document.getElementById('ssl').checked) {
-		current_scheme += 's';
-	}
-	const url = current_scheme + '://' + current_host + ':' + current_port + '?username=' + current_user + '&password=' + current_pass
-	console.debug('connecting to', url);
-	xhr.open('POST', url, true);
+	current_user = document.getElementById('user').value;
+	current_pass = document.getElementById('pass').value;
+	xhr.open('POST', proto + '://' + host + '/api/v1/ussd/kenya?username=' + current_user + '&password=' + current_pass, true);
 	xhr.setRequestHeader('Content-Type', 'application/json');
 	data = {
 		sessionId: uuid,
@@ -116,8 +106,6 @@ function abort() {
 window.addEventListener('load', () => {
 	document.getElementById('user').value = user;
 	document.getElementById('pass').value = pass;
-	document.getElementById('host').value = host;
-	document.getElementById('port').value = port;
 	document.getElementById('phone').addEventListener('keyup', (e) => {
 		if (e.keyCode == '13') {
 			document.getElementById('input').value = '';

apps/cic-ussd/requirements.txt

@@ -7,11 +7,11 @@ billiard==3.6.3.0
 celery==4.4.7
 cffi==1.14.3
 chainlib~=0.0.1a15
-cic-eth==0.10.0a39
-cic-notify~=0.4.0a2
+cic-eth==0.10.0a38
+cic-notify==0.3.1
 cic-types==0.1.0a8
 click==7.1.2
-confini~=0.3.6rc3
+confini==0.3.5
 cryptography==3.2.1
 faker==4.17.1
 iniconfig==1.1.1

apps/cic-ussd/setup.cfg

@@ -44,5 +44,4 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-ussd-tasker = cic_ussd.runnable.tasker:main
-	cic-ussd-client = cic_ussd.runnable.client:main
+	cic-ussd-tasker = cic_ussd.runnable.tasker:main

apps/contract-migration/scripts/import_meta.js

@@ -74,7 +74,7 @@ new cic.PGPKeyStore(
 	importMeta,
 );
 
-const batchSize = 16;
+const batchSize = 50;
 const batchDelay = 1000;
 const total = parseInt(process.argv[3]);
 const workDir = path.join(process.argv[2], 'meta');