From 5f01135b04d285c6143b3d5202ff5a978dc18ce7 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Sun, 7 Mar 2021 18:01:44 +0000 Subject: [PATCH] Rehabilitate CLI and API after nonce changes --- apps/cic-eth/cic_eth/admin/ctrl.py | 17 +++++---- apps/cic-eth/cic_eth/api/api_task.py | 24 +++++++----- apps/cic-eth/cic_eth/error.py | 7 ++++ apps/cic-eth/cic_eth/eth/gas.py | 1 + apps/cic-eth/cic_eth/eth/tx.py | 30 +++++++++++++-- apps/cic-eth/cic_eth/runnable/create.py | 1 + apps/cic-eth/cic_eth/runnable/ctrl.py | 2 +- apps/cic-eth/cic_eth/runnable/resend.py | 2 +- apps/cic-eth/cic_eth/runnable/view.py | 2 +- apps/cic-eth/cic_eth/task.py | 6 ++- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/requirements.txt | 2 +- apps/cic-notify/setup.cfg | 2 +- .../cli/config => .config_client}/app.ini | 0 .../cli/config => .config_client}/client.ini | 0 .../cli/config => .config_client}/ussd.ini | 0 .../cli.py => cic_ussd/runnable/client.py} | 10 +++-- apps/cic-ussd/cic_ussd/version.py | 2 +- apps/cic-ussd/emulator/browser/index.html | 11 +++++- apps/cic-ussd/emulator/browser/ussd.js | 28 ++++++++++---- apps/cic-ussd/requirements.txt | 38 +++---------------- apps/cic-ussd/setup.cfg | 3 +- apps/contract-migration/docker/Dockerfile | 4 +- .../scripts/create_import_users.py | 15 ++++++-- .../contract-migration/scripts/import_meta.js | 2 +- .../scripts/import_users.py | 9 +++-- .../scripts/requirements.txt | 4 +- apps/contract-migration/seed_cic_eth.sh | 3 +- docker-compose.yml | 2 +- 29 files changed, 139 insertions(+), 90 deletions(-) rename apps/cic-ussd/{emulator/cli/config => .config_client}/app.ini (100%) rename apps/cic-ussd/{emulator/cli/config => .config_client}/client.ini (100%) rename apps/cic-ussd/{emulator/cli/config => .config_client}/ussd.ini (100%) rename apps/cic-ussd/{emulator/cli/cli.py => cic_ussd/runnable/client.py} (93%) diff --git a/apps/cic-eth/cic_eth/admin/ctrl.py b/apps/cic-eth/cic_eth/admin/ctrl.py index 5c9d769..1868238 100644 --- a/apps/cic-eth/cic_eth/admin/ctrl.py +++ b/apps/cic-eth/cic_eth/admin/ctrl.py @@ -10,12 +10,15 @@ from cic_registry import zero_address from cic_eth.db.enum import LockEnum from cic_eth.db.models.base import SessionBase from cic_eth.db.models.lock import Lock +from cic_eth.task import ( + CriticalSQLAlchemyTask, + ) from cic_eth.error import LockedError celery_app = celery.current_app logg = logging.getLogger() -@celery_app.task() +@celery_app.task(base=CriticalSQLAlchemyTask) def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_hash=None): """Task wrapper to set arbitrary locks @@ -33,7 +36,7 @@ def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_ return chained_input -@celery_app.task() +@celery_app.task(base=CriticalSQLAlchemyTask) def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL): """Task wrapper to reset arbitrary locks @@ -51,7 +54,7 @@ def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL): return chained_input -@celery_app.task() +@celery_app.task(base=CriticalSQLAlchemyTask) def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None): """Task wrapper to set send lock @@ -67,7 +70,7 @@ def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None): return chained_input -@celery_app.task() +@celery_app.task(base=CriticalSQLAlchemyTask) def unlock_send(chained_input, chain_str, address=zero_address): """Task wrapper to reset send lock @@ -83,7 +86,7 @@ def unlock_send(chained_input, chain_str, address=zero_address): return chained_input -@celery_app.task() +@celery_app.task(base=CriticalSQLAlchemyTask) def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None): """Task wrapper to set queue direct lock @@ -99,7 +102,7 @@ def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None): return chained_input -@celery_app.task() +@celery_app.task(base=CriticalSQLAlchemyTask) def unlock_queue(chained_input, chain_str, address=zero_address): """Task wrapper to reset queue direct lock @@ -115,7 +118,7 @@ def unlock_queue(chained_input, chain_str, address=zero_address): return chained_input -@celery_app.task() +@celery_app.task(base=CriticalSQLAlchemyTask) def check_lock(chained_input, chain_str, lock_flags, address=None): session = SessionBase.create_session() r = Lock.check(chain_str, lock_flags, address=zero_address, session=session) diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index f6e8704..9013f26 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -82,6 +82,7 @@ class Api: :returns: uuid of root task :rtype: celery.Task """ + raise NotImplementedError('out of service until new DEX migration is done') s_check = celery.signature( 'cic_eth.admin.ctrl.check_lock', [ @@ -143,6 +144,7 @@ class Api: :returns: uuid of root task :rtype: celery.Task """ + raise NotImplementedError('out of service until new DEX migration is done') s_check = celery.signature( 'cic_eth.admin.ctrl.check_lock', [ @@ -340,11 +342,6 @@ class Api: ], queue=self.queue, ) - s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', - [], - queue=self.queue, - ) s_account = celery.signature( 'cic_eth.eth.account.create', [ @@ -352,12 +349,18 @@ class Api: ], queue=self.queue, ) - s_nonce.link(s_account) - s_check.link(s_nonce) + s_check.link(s_account) if self.callback_param != None: s_account.link(self.callback_success) if register: + s_nonce = celery.signature( + 'cic_eth.eth.tx.reserve_nonce', + [ + 'ACCOUNTS_INDEX_WRITER', + ], + queue=self.queue, + ) s_register = celery.signature( 'cic_eth.eth.account.register', [ @@ -365,7 +368,8 @@ class Api: ], queue=self.queue, ) - s_account.link(s_register) + s_nonce.link(s_register) + s_account.link(s_nonce) t = s_check.apply_async(queue=self.queue) return t @@ -390,7 +394,9 @@ class Api: ) s_nonce = celery.signature( 'cic_eth.eth.tx.reserve_nonce', - [], + [ + 'GAS_GIFTER', + ], queue=self.queue, ) s_refill = celery.signature( diff --git a/apps/cic-eth/cic_eth/error.py b/apps/cic-eth/cic_eth/error.py index 8f76926..b6d615c 100644 --- a/apps/cic-eth/cic_eth/error.py +++ b/apps/cic-eth/cic_eth/error.py @@ -73,3 +73,10 @@ class SignerError(Exception): """ pass + + +class EthError(Exception): + """Exception raised when unspecified error from evm node is encountered + + """ + pass diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 8cf9905..34a2ff1 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -54,6 +54,7 @@ class GasOracle(): """ session = SessionBase.create_session() a = AccountRole.get_address('GAS_GIFTER', session) + logg.debug('gasgifter {}'.format(a)) session.close() return a diff --git a/apps/cic-eth/cic_eth/eth/tx.py b/apps/cic-eth/cic_eth/eth/tx.py index 2780faa..ec87ba3 100644 --- a/apps/cic-eth/cic_eth/eth/tx.py +++ b/apps/cic-eth/cic_eth/eth/tx.py @@ -14,6 +14,7 @@ from cic_eth.db import Otx, SessionBase from cic_eth.db.models.tx import TxCache from cic_eth.db.models.nonce import NonceReservation from cic_eth.db.models.lock import Lock +from cic_eth.db.models.role import AccountRole from cic_eth.db.enum import ( LockEnum, StatusBits, @@ -38,6 +39,7 @@ from cic_eth.task import ( CriticalWeb3Task, CriticalWeb3AndSignerTask, CriticalSQLAlchemyAndSignerTask, + CriticalSQLAlchemyAndWeb3Task, ) celery_app = celery.current_app @@ -47,7 +49,7 @@ MAX_NONCE_ATTEMPTS = 3 # TODO this function is too long -@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyTask) +@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task) def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=None): """Check the gas level of the sender address of a transaction. @@ -75,6 +77,9 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non if address == None: address = o['address'] + if not web3.Web3.isChecksumAddress(address): + raise ValueError('invalid address {}'.format(address)) + chain_spec = ChainSpec.from_chain_str(chain_str) queue = self.request.delivery_info['routing_key'] @@ -83,7 +88,12 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non c = RpcClient(chain_spec) # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx - balance = c.w3.eth.getBalance(address) + balance = 0 + try: + balance = c.w3.eth.getBalance(address) + except ValueError as e: + raise EthError('balance call for {}'.format()) + logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required)) if gas_required > balance: @@ -590,11 +600,23 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa @celery_app.task(bind=True, base=CriticalSQLAlchemyTask) -def reserve_nonce(self, chained_input, address=None): +def reserve_nonce(self, chained_input, signer=None): session = SessionBase.create_session() - if address == None: + address = None + if signer == None: address = chained_input + logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input)) + else: + if web3.Web3.isChecksumAddress(signer): + address = signer + logg.debug('explicit address for reserve nonce {}'.format(signer)) + else: + address = AccountRole.get_address(signer, session=session) + logg.debug('role for reserve nonce {} -> {}'.format(signer, address)) + + if not web3.Web3.isChecksumAddress(address): + raise ValueError('invalid result when resolving address for nonce {}'.format(address)) root_id = self.request.root_id nonce = NonceReservation.next(address, root_id) diff --git a/apps/cic-eth/cic_eth/runnable/create.py b/apps/cic-eth/cic_eth/runnable/create.py index 8371b0e..532878b 100644 --- a/apps/cic-eth/cic_eth/runnable/create.py +++ b/apps/cic-eth/cic_eth/runnable/create.py @@ -5,6 +5,7 @@ import os import logging import uuid import json +from xdg.BaseDirectory import xdg_config_home import celery from cic_eth.api import Api diff --git a/apps/cic-eth/cic_eth/runnable/ctrl.py b/apps/cic-eth/cic_eth/runnable/ctrl.py index 2aae7be..b73f3bd 100644 --- a/apps/cic-eth/cic_eth/runnable/ctrl.py +++ b/apps/cic-eth/cic_eth/runnable/ctrl.py @@ -25,7 +25,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING) default_abi_dir = '/usr/share/local/cic/solidity/abi' -default_config_dir = os.path.join('/usr/local/etc/cic-eth') +default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') argparser = argparse.ArgumentParser() argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') diff --git a/apps/cic-eth/cic_eth/runnable/resend.py b/apps/cic-eth/cic_eth/runnable/resend.py index b455b77..77faf95 100644 --- a/apps/cic-eth/cic_eth/runnable/resend.py +++ b/apps/cic-eth/cic_eth/runnable/resend.py @@ -22,7 +22,7 @@ logg = logging.getLogger() logging.getLogger('web3').setLevel(logging.WARNING) logging.getLogger('urllib3').setLevel(logging.WARNING) -default_config_dir = os.path.join('/usr/local/etc/cic-eth') +default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') argparser = argparse.ArgumentParser() diff --git a/apps/cic-eth/cic_eth/runnable/view.py b/apps/cic-eth/cic_eth/runnable/view.py index 144f094..0cd6f8d 100644 --- a/apps/cic-eth/cic_eth/runnable/view.py +++ b/apps/cic-eth/cic_eth/runnable/view.py @@ -37,7 +37,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING) default_abi_dir = '/usr/share/local/cic/solidity/abi' -default_config_dir = os.path.join('/usr/local/etc/cic-eth') +default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') argparser = argparse.ArgumentParser() argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)') diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index 2b5f6af..5d34e5a 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -6,7 +6,10 @@ import celery import sqlalchemy # local imports -from cic_eth.error import SignerError +from cic_eth.error import ( + SignerError, + EthError, + ) class CriticalTask(celery.Task): @@ -33,6 +36,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask): sqlalchemy.exc.DatabaseError, sqlalchemy.exc.TimeoutError, requests.exceptions.ConnectionError, + EthError, ) class CriticalSQLAlchemyAndSignerTask(CriticalTask): diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 66a1f09..6da7a12 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 10, 0, - 'alpha.39', + 'alpha.41', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 96b4c93..ef85b3c 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ -cic-base~=0.1.1a10 +cic-base~=0.1.1a20 web3==5.12.2 celery==4.4.7 crypto-dev-signer~=0.4.13rc4 diff --git a/apps/cic-notify/setup.cfg b/apps/cic-notify/setup.cfg index d421d37..4ef87f2 100644 --- a/apps/cic-notify/setup.cfg +++ b/apps/cic-notify/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = cic-notify -version= attr: cic_notify.version.__version_string__ +version= 0.4.0a2 description = CIC notifications service author = Louis Holbrook author_email = dev@holbrook.no diff --git a/apps/cic-ussd/emulator/cli/config/app.ini b/apps/cic-ussd/.config_client/app.ini similarity index 100% rename from apps/cic-ussd/emulator/cli/config/app.ini rename to apps/cic-ussd/.config_client/app.ini diff --git a/apps/cic-ussd/emulator/cli/config/client.ini b/apps/cic-ussd/.config_client/client.ini similarity index 100% rename from apps/cic-ussd/emulator/cli/config/client.ini rename to apps/cic-ussd/.config_client/client.ini diff --git a/apps/cic-ussd/emulator/cli/config/ussd.ini b/apps/cic-ussd/.config_client/ussd.ini similarity index 100% rename from apps/cic-ussd/emulator/cli/config/ussd.ini rename to apps/cic-ussd/.config_client/ussd.ini diff --git a/apps/cic-ussd/emulator/cli/cli.py b/apps/cic-ussd/cic_ussd/runnable/client.py similarity index 93% rename from apps/cic-ussd/emulator/cli/cli.py rename to apps/cic-ussd/cic_ussd/runnable/client.py index bf28075..f6fc050 100644 --- a/apps/cic-ussd/emulator/cli/cli.py +++ b/apps/cic-ussd/cic_ussd/runnable/client.py @@ -21,10 +21,10 @@ from confini import Config logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -config_dir = os.path.join(xdg_config_home, 'cli-ussd') +default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') argparser = argparse.ArgumentParser(description='CLI tool to interface a Sempo USSD session') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') +argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') #argparser.add_argument('-d', type=str, default='local', help='deployment name to interface (config root subdirectory)') argparser.add_argument('--host', type=str, default='localhost') argparser.add_argument('--port', type=int, default=9000) @@ -64,7 +64,7 @@ else: ssl = True -if __name__ == "__main__": +def main(): # TODO: improve url building url = 'http' @@ -101,3 +101,7 @@ if __name__ == "__main__": state = response_data[:3] out = response_data[4:] print(out) + + +if __name__ == "__main__": + main() diff --git a/apps/cic-ussd/cic_ussd/version.py b/apps/cic-ussd/cic_ussd/version.py index 1743252..2161a75 100644 --- a/apps/cic-ussd/cic_ussd/version.py +++ b/apps/cic-ussd/cic_ussd/version.py @@ -1,7 +1,7 @@ # standard imports import semver -version = (0, 3, 0, 'alpha.1') +version = (0, 3, 0, 'alpha.5') version_object = semver.VersionInfo( major=version[0], diff --git a/apps/cic-ussd/emulator/browser/index.html b/apps/cic-ussd/emulator/browser/index.html index 1dd5c89..642ddfe 100644 --- a/apps/cic-ussd/emulator/browser/index.html +++ b/apps/cic-ussd/emulator/browser/index.html @@ -16,9 +16,16 @@ div#session {
-
+ -
+
+ + + + + + +
diff --git a/apps/cic-ussd/emulator/browser/ussd.js b/apps/cic-ussd/emulator/browser/ussd.js index 61b3ef6..3670fea 100644 --- a/apps/cic-ussd/emulator/browser/ussd.js +++ b/apps/cic-ussd/emulator/browser/ussd.js @@ -1,10 +1,12 @@ -//var proto = 'http'; -//var host = 'localhost:9000'; +var ssl = false; +var host = 'localhost'; +var port = 9000; var proto = 'https'; var host = 'staging.sarafu.network'; -var user = 'admin_bert_token_inc.'; -var pass = '197781ed60bf16d5dc12d84e3df37e35'; -var serviceCode = '*483*061#'; +var user = 'foo'; +var pass = 'bar'; +var path = '/'; +var serviceCode = '*483*46#'; // cheekily stolen from https://www.tutorialspoint.com/how-to-create-guid-uuid-in-javascript function createUUID() { @@ -23,9 +25,17 @@ function send(s) { document.getElementById('send_input').disabled = true; var xhr = new XMLHttpRequest(); xhr.responseType = 'text'; - current_user = document.getElementById('user').value; - current_pass = document.getElementById('pass').value; - xhr.open('POST', proto + '://' + host + '/api/v1/ussd/kenya?username=' + current_user + '&password=' + current_pass, true); + const current_user = document.getElementById('user').value; + const current_pass = document.getElementById('pass').value; + const current_host = document.getElementById('host').value; + const current_port = document.getElementById('port').value; + let current_scheme = 'http'; + if (document.getElementById('ssl').checked) { + current_scheme += 's'; + } + const url = current_scheme + '://' + current_host + ':' + current_port + '?username=' + current_user + '&password=' + current_pass + console.debug('connecting to', url); + xhr.open('POST', url, true); xhr.setRequestHeader('Content-Type', 'application/json'); data = { sessionId: uuid, @@ -106,6 +116,8 @@ function abort() { window.addEventListener('load', () => { document.getElementById('user').value = user; document.getElementById('pass').value = pass; + document.getElementById('host').value = host; + document.getElementById('port').value = port; document.getElementById('phone').addEventListener('keyup', (e) => { if (e.keyCode == '13') { document.getElementById('input').value = ''; diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 85ffea9..c28b33e 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,49 +1,21 @@ +cic_base[full-graph]~=0.1.1a17 alembic==1.4.2 -amqp==2.6.1 -attrs==20.2.0 bcrypt==3.2.0 -betterpath==0.2.2 -billiard==3.6.3.0 celery==4.4.7 -cffi==1.14.3 -chainlib~=0.0.1a15 -cic-eth==0.10.0a39 -cic-notify==0.3.1 +chainlib~=0.0.1a20 +cic-eth~=0.10.0a40 +cic-notify~=0.4.0a2 cic-types==0.1.0a8 -click==7.1.2 -confini==0.3.5 +confini~=0.3.6rc3 cryptography==3.2.1 faker==4.17.1 -iniconfig==1.1.1 -kombu==4.6.11 -Mako==1.1.3 -MarkupSafe==1.1.1 -mirakuru==2.3.0 -more-itertools==8.5.0 -packaging==20.4 phonenumbers==8.12.12 -pluggy==0.13.1 -port-for==0.4 -psutil==5.7.3 psycopg2==2.8.6 -py==1.9.0 -pycparser==2.20 -pyparsing==2.4.7 -python-dateutil==2.8.1 -python-editor==1.0.4 python-gnupg==0.4.6 python-i18n==0.3.9 -pytz==2020.1 -PyYAML==5.3.1 redis==3.5.3 -requests==2.24.0 semver==2.13.0 -six==1.15.0 SQLAlchemy==1.3.20 tinydb==4.2.0 -toml==0.10.1 transitions==0.8.4 uWSGI==2.0.19.1 -vcversioner==2.16.0.0 -vine==1.3.0 -zope.interface==5.1.2 diff --git a/apps/cic-ussd/setup.cfg b/apps/cic-ussd/setup.cfg index 620483a..716e595 100644 --- a/apps/cic-ussd/setup.cfg +++ b/apps/cic-ussd/setup.cfg @@ -44,4 +44,5 @@ scripts = [options.entry_points] console_scripts = - cic-ussd-tasker = cic_ussd.runnable.tasker:main \ No newline at end of file + cic-ussd-tasker = cic_ussd.runnable.tasker:main + cic-ussd-client = cic_ussd.runnable.client:main diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index c257aa5..9b589c0 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -107,10 +107,10 @@ RUN cd cic-bancor/python && \ RUN apt-get install -y cargo -ARG cic_base_version=0.1.1a10 +ARG cic_base_version=0.1.1a23 RUN pip install --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version -ARG cic_registry_version=0.5.3a22 +ARG cic_registry_version=0.5.3a24 RUN pip install --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version WORKDIR /root diff --git a/apps/contract-migration/scripts/create_import_users.py b/apps/contract-migration/scripts/create_import_users.py index 0ef540b..7222d03 100644 --- a/apps/contract-migration/scripts/create_import_users.py +++ b/apps/contract-migration/scripts/create_import_users.py @@ -191,9 +191,16 @@ if __name__ == '__main__': fa = open(os.path.join(user_dir, 'balances.csv'), 'w') - for i in range(user_count): - - (eth, phone, o) = gen() + i = 0 + while i < user_count: + eth = None + phone = None + o = None + try: + (eth, phone, o) = gen() + except Exception as e: + logg.warning('generate failed, trying anew: {}'.format(e)) + continue uid = eth[2:].upper() print(o) @@ -212,5 +219,7 @@ if __name__ == '__main__': amount = genAmount() fa.write('{},{}\n'.format(eth,amount)) logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount)) + + i += 1 fa.close() diff --git a/apps/contract-migration/scripts/import_meta.js b/apps/contract-migration/scripts/import_meta.js index 3ea82e9..56c754f 100644 --- a/apps/contract-migration/scripts/import_meta.js +++ b/apps/contract-migration/scripts/import_meta.js @@ -74,7 +74,7 @@ new cic.PGPKeyStore( importMeta, ); -const batchSize = 50; +const batchSize = 16; const batchDelay = 1000; const total = parseInt(process.argv[3]); const workDir = path.join(process.argv[2], 'meta'); diff --git a/apps/contract-migration/scripts/import_users.py b/apps/contract-migration/scripts/import_users.py index cb2d1d7..8264a31 100644 --- a/apps/contract-migration/scripts/import_users.py +++ b/apps/contract-migration/scripts/import_users.py @@ -38,7 +38,7 @@ argparser.add_argument('--redis-host-callback', dest='redis_host_callback', defa argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node') argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') +argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout') argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') argparser.add_argument('-v', action='store_true', help='Be verbose') argparser.add_argument('-vv', action='store_true', help='Be more verbose') @@ -97,6 +97,7 @@ def register_eth(i, u): callback_queue=args.q, ) t = api.create_account(register=True) + logg.debug('register {} -> {}'.format(u, t)) while True: ps.get_message() @@ -112,11 +113,11 @@ def register_eth(i, u): r = json.loads(m['data']) address = r['result'] break - except TypeError as e: + except Exception as e: if m == None: - logg.critical('empty response from redis callback (did the service crash?)') + logg.critical('empty response from redis callback (did the service crash?) {}'.format(e)) else: - logg.critical('unexpected response from redis callback: {}'.format(m)) + logg.critical('unexpected response from redis callback: {} {}'.format(m, e)) sys.exit(1) logg.debug('[{}] register eth {} {}'.format(i, u, address)) diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index fa1cf34..b16cc54 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,3 +1,3 @@ -cic-base[full_graph]==0.1.1a12 -cic-eth==0.10.0a38 +cic-base[full_graph]==0.1.1a23 +cic-eth==0.10.0a41 cic-types==0.1.0a8 diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index dda61b9..d59efef 100644 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -31,8 +31,7 @@ set -e set -a # We need to not install these here... -pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a38 chainlib==0.0.1a19 cic-contracts==0.0.2a2 -pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10 +pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a41 chainlib==0.0.1a21 cic-contracts==0.0.2a2 >&2 echo "create account for gas gifter" old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER diff --git a/docker-compose.yml b/docker-compose.yml index 6543053..7bd962b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -238,7 +238,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tasker.sh -q cic-eth + ./start_tasker.sh -q cic-eth -vv # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ] cic-eth-tracker: