Merge branch 'lash/cli-rehabilitations' into 'master'

Rehabilitate CLI and API after nonce changes

See merge request grassrootseconomics/cic-internal-integration!57
This commit is contained in:
Louis Holbrook 2021-03-07 18:01:44 +00:00
commit 8da5219290
29 changed files with 139 additions and 90 deletions

View File

@ -10,12 +10,15 @@ from cic_registry import zero_address
from cic_eth.db.enum import LockEnum
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.lock import Lock
from cic_eth.task import (
CriticalSQLAlchemyTask,
)
from cic_eth.error import LockedError
celery_app = celery.current_app
logg = logging.getLogger()
@celery_app.task()
@celery_app.task(base=CriticalSQLAlchemyTask)
def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_hash=None):
"""Task wrapper to set arbitrary locks
@ -33,7 +36,7 @@ def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_
return chained_input
@celery_app.task()
@celery_app.task(base=CriticalSQLAlchemyTask)
def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
"""Task wrapper to reset arbitrary locks
@ -51,7 +54,7 @@ def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
return chained_input
@celery_app.task()
@celery_app.task(base=CriticalSQLAlchemyTask)
def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
"""Task wrapper to set send lock
@ -67,7 +70,7 @@ def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
return chained_input
@celery_app.task()
@celery_app.task(base=CriticalSQLAlchemyTask)
def unlock_send(chained_input, chain_str, address=zero_address):
"""Task wrapper to reset send lock
@ -83,7 +86,7 @@ def unlock_send(chained_input, chain_str, address=zero_address):
return chained_input
@celery_app.task()
@celery_app.task(base=CriticalSQLAlchemyTask)
def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
"""Task wrapper to set queue direct lock
@ -99,7 +102,7 @@ def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
return chained_input
@celery_app.task()
@celery_app.task(base=CriticalSQLAlchemyTask)
def unlock_queue(chained_input, chain_str, address=zero_address):
"""Task wrapper to reset queue direct lock
@ -115,7 +118,7 @@ def unlock_queue(chained_input, chain_str, address=zero_address):
return chained_input
@celery_app.task()
@celery_app.task(base=CriticalSQLAlchemyTask)
def check_lock(chained_input, chain_str, lock_flags, address=None):
session = SessionBase.create_session()
r = Lock.check(chain_str, lock_flags, address=zero_address, session=session)

View File

@ -82,6 +82,7 @@ class Api:
:returns: uuid of root task
:rtype: celery.Task
"""
raise NotImplementedError('out of service until new DEX migration is done')
s_check = celery.signature(
'cic_eth.admin.ctrl.check_lock',
[
@ -143,6 +144,7 @@ class Api:
:returns: uuid of root task
:rtype: celery.Task
"""
raise NotImplementedError('out of service until new DEX migration is done')
s_check = celery.signature(
'cic_eth.admin.ctrl.check_lock',
[
@ -340,11 +342,6 @@ class Api:
],
queue=self.queue,
)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[],
queue=self.queue,
)
s_account = celery.signature(
'cic_eth.eth.account.create',
[
@ -352,12 +349,18 @@ class Api:
],
queue=self.queue,
)
s_nonce.link(s_account)
s_check.link(s_nonce)
s_check.link(s_account)
if self.callback_param != None:
s_account.link(self.callback_success)
if register:
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[
'ACCOUNTS_INDEX_WRITER',
],
queue=self.queue,
)
s_register = celery.signature(
'cic_eth.eth.account.register',
[
@ -365,7 +368,8 @@ class Api:
],
queue=self.queue,
)
s_account.link(s_register)
s_nonce.link(s_register)
s_account.link(s_nonce)
t = s_check.apply_async(queue=self.queue)
return t
@ -390,7 +394,9 @@ class Api:
)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[],
[
'GAS_GIFTER',
],
queue=self.queue,
)
s_refill = celery.signature(

View File

@ -73,3 +73,10 @@ class SignerError(Exception):
"""
pass
class EthError(Exception):
"""Exception raised when unspecified error from evm node is encountered
"""
pass

View File

@ -54,6 +54,7 @@ class GasOracle():
"""
session = SessionBase.create_session()
a = AccountRole.get_address('GAS_GIFTER', session)
logg.debug('gasgifter {}'.format(a))
session.close()
return a

View File

@ -14,6 +14,7 @@ from cic_eth.db import Otx, SessionBase
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import NonceReservation
from cic_eth.db.models.lock import Lock
from cic_eth.db.models.role import AccountRole
from cic_eth.db.enum import (
LockEnum,
StatusBits,
@ -38,6 +39,7 @@ from cic_eth.task import (
CriticalWeb3Task,
CriticalWeb3AndSignerTask,
CriticalSQLAlchemyAndSignerTask,
CriticalSQLAlchemyAndWeb3Task,
)
celery_app = celery.current_app
@ -47,7 +49,7 @@ MAX_NONCE_ATTEMPTS = 3
# TODO this function is too long
@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyTask)
@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=None):
"""Check the gas level of the sender address of a transaction.
@ -75,6 +77,9 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
if address == None:
address = o['address']
if not web3.Web3.isChecksumAddress(address):
raise ValueError('invalid address {}'.format(address))
chain_spec = ChainSpec.from_chain_str(chain_str)
queue = self.request.delivery_info['routing_key']
@ -83,7 +88,12 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
c = RpcClient(chain_spec)
# TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
balance = c.w3.eth.getBalance(address)
balance = 0
try:
balance = c.w3.eth.getBalance(address)
except ValueError as e:
raise EthError('balance call for {}'.format())
logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required))
if gas_required > balance:
@ -590,11 +600,23 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
def reserve_nonce(self, chained_input, address=None):
def reserve_nonce(self, chained_input, signer=None):
session = SessionBase.create_session()
if address == None:
address = None
if signer == None:
address = chained_input
logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
else:
if web3.Web3.isChecksumAddress(signer):
address = signer
logg.debug('explicit address for reserve nonce {}'.format(signer))
else:
address = AccountRole.get_address(signer, session=session)
logg.debug('role for reserve nonce {} -> {}'.format(signer, address))
if not web3.Web3.isChecksumAddress(address):
raise ValueError('invalid result when resolving address for nonce {}'.format(address))
root_id = self.request.root_id
nonce = NonceReservation.next(address, root_id)

View File

@ -5,6 +5,7 @@ import os
import logging
import uuid
import json
from xdg.BaseDirectory import xdg_config_home
import celery
from cic_eth.api import Api

View File

@ -25,7 +25,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
default_abi_dir = '/usr/share/local/cic/solidity/abi'
default_config_dir = os.path.join('/usr/local/etc/cic-eth')
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')

View File

@ -22,7 +22,7 @@ logg = logging.getLogger()
logging.getLogger('web3').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
default_config_dir = os.path.join('/usr/local/etc/cic-eth')
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser()

View File

@ -37,7 +37,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING)
default_abi_dir = '/usr/share/local/cic/solidity/abi'
default_config_dir = os.path.join('/usr/local/etc/cic-eth')
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)')

View File

@ -6,7 +6,10 @@ import celery
import sqlalchemy
# local imports
from cic_eth.error import SignerError
from cic_eth.error import (
SignerError,
EthError,
)
class CriticalTask(celery.Task):
@ -33,6 +36,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
sqlalchemy.exc.DatabaseError,
sqlalchemy.exc.TimeoutError,
requests.exceptions.ConnectionError,
EthError,
)
class CriticalSQLAlchemyAndSignerTask(CriticalTask):

View File

@ -10,7 +10,7 @@ version = (
0,
10,
0,
'alpha.39',
'alpha.41',
)
version_object = semver.VersionInfo(

View File

@ -1,4 +1,4 @@
cic-base~=0.1.1a10
cic-base~=0.1.1a20
web3==5.12.2
celery==4.4.7
crypto-dev-signer~=0.4.13rc4

View File

@ -1,6 +1,6 @@
[metadata]
name = cic-notify
version= attr: cic_notify.version.__version_string__
version= 0.4.0a2
description = CIC notifications service
author = Louis Holbrook
author_email = dev@holbrook.no

View File

@ -21,10 +21,10 @@ from confini import Config
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
config_dir = os.path.join(xdg_config_home, 'cli-ussd')
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser(description='CLI tool to interface a Sempo USSD session')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
#argparser.add_argument('-d', type=str, default='local', help='deployment name to interface (config root subdirectory)')
argparser.add_argument('--host', type=str, default='localhost')
argparser.add_argument('--port', type=int, default=9000)
@ -64,7 +64,7 @@ else:
ssl = True
if __name__ == "__main__":
def main():
# TODO: improve url building
url = 'http'
@ -101,3 +101,7 @@ if __name__ == "__main__":
state = response_data[:3]
out = response_data[4:]
print(out)
if __name__ == "__main__":
main()

View File

@ -1,7 +1,7 @@
# standard imports
import semver
version = (0, 3, 0, 'alpha.1')
version = (0, 3, 0, 'alpha.5')
version_object = semver.VersionInfo(
major=version[0],

View File

@ -16,9 +16,16 @@ div#session {
<textarea id="monitor" disabled="1"></textarea>
<div id="login">
<label for="user">API username</label>
<input type="text" id="user" name="user" type="text" /><br/>
<input type="text" id="user" name="user" />
<label for="user">API password</label>
<input type="text" id="pass" name="pass" type="text" /><br/>
<input type="text" id="pass" name="pass" /> <br/>
<label for="host">API host</label>
<input type="text" id="host" name="host" />
<label for="host">API port</label>
<input type="text" id="port" name="port" />
<label for="host">SSL</label>
<input type="checkbox" id="ssl" name="ssl" checked="1"/> <br/>
<hr/>
<input type="text" id="phone" /> <button onclick="setPhone(document.getElementById('phone').value);" id="send_phone">set phone number</button>
</div>

View File

@ -1,10 +1,12 @@
//var proto = 'http';
//var host = 'localhost:9000';
var ssl = false;
var host = 'localhost';
var port = 9000;
var proto = 'https';
var host = 'staging.sarafu.network';
var user = 'admin_bert_token_inc.';
var pass = '197781ed60bf16d5dc12d84e3df37e35';
var serviceCode = '*483*061#';
var user = 'foo';
var pass = 'bar';
var path = '/';
var serviceCode = '*483*46#';
// cheekily stolen from https://www.tutorialspoint.com/how-to-create-guid-uuid-in-javascript
function createUUID() {
@ -23,9 +25,17 @@ function send(s) {
document.getElementById('send_input').disabled = true;
var xhr = new XMLHttpRequest();
xhr.responseType = 'text';
current_user = document.getElementById('user').value;
current_pass = document.getElementById('pass').value;
xhr.open('POST', proto + '://' + host + '/api/v1/ussd/kenya?username=' + current_user + '&password=' + current_pass, true);
const current_user = document.getElementById('user').value;
const current_pass = document.getElementById('pass').value;
const current_host = document.getElementById('host').value;
const current_port = document.getElementById('port').value;
let current_scheme = 'http';
if (document.getElementById('ssl').checked) {
current_scheme += 's';
}
const url = current_scheme + '://' + current_host + ':' + current_port + '?username=' + current_user + '&password=' + current_pass
console.debug('connecting to', url);
xhr.open('POST', url, true);
xhr.setRequestHeader('Content-Type', 'application/json');
data = {
sessionId: uuid,
@ -106,6 +116,8 @@ function abort() {
window.addEventListener('load', () => {
document.getElementById('user').value = user;
document.getElementById('pass').value = pass;
document.getElementById('host').value = host;
document.getElementById('port').value = port;
document.getElementById('phone').addEventListener('keyup', (e) => {
if (e.keyCode == '13') {
document.getElementById('input').value = '';

View File

@ -1,49 +1,21 @@
cic_base[full-graph]~=0.1.1a17
alembic==1.4.2
amqp==2.6.1
attrs==20.2.0
bcrypt==3.2.0
betterpath==0.2.2
billiard==3.6.3.0
celery==4.4.7
cffi==1.14.3
chainlib~=0.0.1a15
cic-eth==0.10.0a39
cic-notify==0.3.1
chainlib~=0.0.1a20
cic-eth~=0.10.0a40
cic-notify~=0.4.0a2
cic-types==0.1.0a8
click==7.1.2
confini==0.3.5
confini~=0.3.6rc3
cryptography==3.2.1
faker==4.17.1
iniconfig==1.1.1
kombu==4.6.11
Mako==1.1.3
MarkupSafe==1.1.1
mirakuru==2.3.0
more-itertools==8.5.0
packaging==20.4
phonenumbers==8.12.12
pluggy==0.13.1
port-for==0.4
psutil==5.7.3
psycopg2==2.8.6
py==1.9.0
pycparser==2.20
pyparsing==2.4.7
python-dateutil==2.8.1
python-editor==1.0.4
python-gnupg==0.4.6
python-i18n==0.3.9
pytz==2020.1
PyYAML==5.3.1
redis==3.5.3
requests==2.24.0
semver==2.13.0
six==1.15.0
SQLAlchemy==1.3.20
tinydb==4.2.0
toml==0.10.1
transitions==0.8.4
uWSGI==2.0.19.1
vcversioner==2.16.0.0
vine==1.3.0
zope.interface==5.1.2

View File

@ -44,4 +44,5 @@ scripts =
[options.entry_points]
console_scripts =
cic-ussd-tasker = cic_ussd.runnable.tasker:main
cic-ussd-tasker = cic_ussd.runnable.tasker:main
cic-ussd-client = cic_ussd.runnable.client:main

View File

@ -107,10 +107,10 @@ RUN cd cic-bancor/python && \
RUN apt-get install -y cargo
ARG cic_base_version=0.1.1a10
ARG cic_base_version=0.1.1a23
RUN pip install --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version
ARG cic_registry_version=0.5.3a22
ARG cic_registry_version=0.5.3a24
RUN pip install --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version
WORKDIR /root

View File

@ -191,9 +191,16 @@ if __name__ == '__main__':
fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
for i in range(user_count):
(eth, phone, o) = gen()
i = 0
while i < user_count:
eth = None
phone = None
o = None
try:
(eth, phone, o) = gen()
except Exception as e:
logg.warning('generate failed, trying anew: {}'.format(e))
continue
uid = eth[2:].upper()
print(o)
@ -212,5 +219,7 @@ if __name__ == '__main__':
amount = genAmount()
fa.write('{},{}\n'.format(eth,amount))
logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
i += 1
fa.close()

View File

@ -74,7 +74,7 @@ new cic.PGPKeyStore(
importMeta,
);
const batchSize = 50;
const batchSize = 16;
const batchDelay = 1000;
const total = parseInt(process.argv[3]);
const workDir = path.join(process.argv[2], 'meta');

View File

@ -38,7 +38,7 @@ argparser.add_argument('--redis-host-callback', dest='redis_host_callback', defa
argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches')
argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout')
argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
argparser.add_argument('-v', action='store_true', help='Be verbose')
argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@ -97,6 +97,7 @@ def register_eth(i, u):
callback_queue=args.q,
)
t = api.create_account(register=True)
logg.debug('register {} -> {}'.format(u, t))
while True:
ps.get_message()
@ -112,11 +113,11 @@ def register_eth(i, u):
r = json.loads(m['data'])
address = r['result']
break
except TypeError as e:
except Exception as e:
if m == None:
logg.critical('empty response from redis callback (did the service crash?)')
logg.critical('empty response from redis callback (did the service crash?) {}'.format(e))
else:
logg.critical('unexpected response from redis callback: {}'.format(m))
logg.critical('unexpected response from redis callback: {} {}'.format(m, e))
sys.exit(1)
logg.debug('[{}] register eth {} {}'.format(i, u, address))

View File

@ -1,3 +1,3 @@
cic-base[full_graph]==0.1.1a12
cic-eth==0.10.0a38
cic-base[full_graph]==0.1.1a23
cic-eth==0.10.0a41
cic-types==0.1.0a8

View File

@ -31,8 +31,7 @@ set -e
set -a
# We need to not install these here...
pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a38 chainlib==0.0.1a19 cic-contracts==0.0.2a2
pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10
pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a41 chainlib==0.0.1a21 cic-contracts==0.0.2a2
>&2 echo "create account for gas gifter"
old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER

View File

@ -238,7 +238,7 @@ services:
- -c
- |
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
./start_tasker.sh -q cic-eth
./start_tasker.sh -q cic-eth -vv
# command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
cic-eth-tracker: