From 12ab5c2f66664a9c8730608e8426c737f0deeb3d Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Wed, 23 Jun 2021 04:29:38 +0000 Subject: [PATCH] Rehabilitate ussd import scripts --- apps/cic-ussd/cic_ussd/processor.py | 8 +--- .../runnable/daemons/cic_user_ussd_server.py | 5 ++- apps/cic-ussd/requirements.txt | 4 +- apps/data-seeding/README.md | 8 ++-- apps/data-seeding/cic_ussd/import_balance.py | 3 +- apps/data-seeding/cic_ussd/import_pins.py | 4 +- apps/data-seeding/cic_ussd/import_users.py | 43 ++++++------------- .../data-seeding/cic_ussd/import_ussd_data.py | 13 +++--- apps/data-seeding/cic_ussd/import_util.py | 8 ++-- apps/data-seeding/create_import_pins.py | 3 +- apps/data-seeding/verify.py | 12 +++--- 11 files changed, 46 insertions(+), 65 deletions(-) diff --git a/apps/cic-ussd/cic_ussd/processor.py b/apps/cic-ussd/cic_ussd/processor.py index a838d308..d9842df8 100644 --- a/apps/cic-ussd/cic_ussd/processor.py +++ b/apps/cic-ussd/cic_ussd/processor.py @@ -423,12 +423,6 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict] :return: A ussd menu's corresponding text value. :rtype: Document """ - # retrieve metadata before any transition - key = generate_metadata_pointer( - identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), - cic_type=':cic.person' - ) - person_metadata = get_cached_data(key=key) if ussd_session: if user_input == "0": @@ -452,7 +446,7 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict] 'exit_pin_mismatch', 'exit_invalid_request', 'exit_successful_transaction' - ] and person_metadata is not None: + ]: return UssdMenu.find_by_name(name='start') else: return UssdMenu.find_by_name(name=last_state) diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py index 6aad181f..c20aec18 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py @@ -145,7 +145,7 @@ def application(env, start_response): if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/': if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded': - start_response('405 Play by the rules', errors_headers) + start_response('405 Urlencoded, please', errors_headers) return [] post_data = env.get('wsgi.input').read() @@ -213,6 +213,9 @@ def application(env, start_response): return [response_bytes] else: + logg.error('invalid query {}'.format(env)) + for r in env: + logg.debug('{}: {}'.format(r, env)) start_response('405 Play by the rules', errors_headers) return [] diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index ddaa7abf..7b5ef9df 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,4 +1,4 @@ -cic_base[full_graph]~=0.1.2b15 -cic-eth~=0.11.0b16 +cic_base[full_graph]~=0.1.2b17 +cic-eth~=0.11.0b17 cic-notify~=0.4.0a5 cic-types~=0.1.0a10 diff --git a/apps/data-seeding/README.md b/apps/data-seeding/README.md index cb33352a..11043675 100644 --- a/apps/data-seeding/README.md +++ b/apps/data-seeding/README.md @@ -136,7 +136,7 @@ First, make a note of the **block height** before running anything: To import, run to _completion_: -`python eth/import_users.py -v -c config -p -r -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` +`python eth/import_users.py -v -c config -p -r -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` After the script completes, keystore files for all generated accouts will be found in `/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one). @@ -150,7 +150,7 @@ Then run: Run in sequence, in first terminal: -`python cic_eth/import_balance.py -v -c config -p -r --token-symbol -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out` +`python cic_eth/import_balance.py -v -c config -p -r --token-symbol -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out` In another terminal: @@ -226,7 +226,7 @@ The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in ### Step 5 - Verify -`python verify.py -v -c config -r -p ` +`python verify.py -v -c config -r -p --token-symbol ` Included checks: * Private key is in cic-eth keystore @@ -262,3 +262,5 @@ Should exit with code 0 if all input data is found in the respective services. - MacOS BigSur issue when installing psycopg2: ld: library not found for -lssl -> https://github.com/psycopg/psycopg2/issues/1115#issuecomment-831498953 - `cic_ussd` imports is poorly implemented, and consumes a lot of resources. Therefore it takes a long time to complete. Reducing the amount of polls for the phone pointer would go a long way to improve it. + +- A strict constraint is maintained insistin the use of postgresql-12. diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 35430e95..5383bfd4 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -114,7 +114,7 @@ def main(): conn = EthHTTPConnection(config.get('ETH_PROVIDER')) ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer) - ImportTask.balance_processor.init() + ImportTask.balance_processor.init(token_symbol) # TODO get decimals from token balances = {} @@ -139,6 +139,7 @@ def main(): ImportTask.balances = balances ImportTask.count = i + ImportTask.import_dir = user_dir s = celery.signature( 'import_task.send_txs', diff --git a/apps/data-seeding/cic_ussd/import_pins.py b/apps/data-seeding/cic_ussd/import_pins.py index 37bc5eb4..119f8d07 100644 --- a/apps/data-seeding/cic_ussd/import_pins.py +++ b/apps/data-seeding/cic_ussd/import_pins.py @@ -39,6 +39,7 @@ elif args.vv: config_dir = args.c config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) config.process() +logg.debug('config loaded from {}:\n{}'.format(args.c, config)) celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) @@ -62,9 +63,6 @@ def main(): ) s_import_pins.apply_async() - argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG'] - celery_app.worker_main(argv) - if __name__ == '__main__': main() diff --git a/apps/data-seeding/cic_ussd/import_users.py b/apps/data-seeding/cic_ussd/import_users.py index 1898c17c..7dd4ff5e 100644 --- a/apps/data-seeding/cic_ussd/import_users.py +++ b/apps/data-seeding/cic_ussd/import_users.py @@ -1,29 +1,21 @@ # standard imports -import os -import sys +import argparse import json import logging -import argparse -import uuid -import datetime +import os +import sys import time import urllib.request -from glob import glob +import uuid +from urllib.parse import urlencode -# third-party imports -import redis -import confini +# external imports import celery -from hexathon import ( - add_0x, - strip_0x, - ) -from chainlib.eth.address import to_checksum -from cic_types.models.person import Person -from cic_eth.api.api_task import Api -from chainlib.chain import ChainSpec -from cic_types.processor import generate_metadata_pointer +import confini import phonenumbers +import redis +from chainlib.chain import ChainSpec +from cic_types.models.person import Person logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -87,21 +79,13 @@ chain_str = str(chain_spec) batch_size = args.batch_size batch_delay = args.batch_delay -db_configs = { - 'database': config.get('DATABASE_NAME'), - 'host': config.get('DATABASE_HOST'), - 'port': config.get('DATABASE_PORT'), - 'user': config.get('DATABASE_USER'), - 'password': config.get('DATABASE_PASSWORD') -} - def build_ussd_request(phone, host, port, service_code, username, password, ssl=False): url = 'http' if ssl: url += 's' url += '://{}:{}'.format(host, port) - url += '/?username={}&password={}'.format(username, password) #config.get('USSD_USER'), config.get('USSD_PASS')) + url += '/?username={}&password={}'.format(username, password) logg.info('ussd service url {}'.format(url)) logg.info('ussd phone {}'.format(phone)) @@ -114,9 +98,10 @@ def build_ussd_request(phone, host, port, service_code, username, password, ssl= 'text': service_code, } req = urllib.request.Request(url) - data_str = json.dumps(data) + req.method=('POST') + data_str = urlencode(data) data_bytes = data_str.encode('utf-8') - req.add_header('Content-Type', 'application/json') + req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.data = data_bytes return req diff --git a/apps/data-seeding/cic_ussd/import_ussd_data.py b/apps/data-seeding/cic_ussd/import_ussd_data.py index d2283000..fddd8bb6 100644 --- a/apps/data-seeding/cic_ussd/import_ussd_data.py +++ b/apps/data-seeding/cic_ussd/import_ussd_data.py @@ -31,9 +31,9 @@ elif args.vv: config_dir = args.c config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) config.process() +logg.debug('config loaded from {}:\n{}'.format(args.c, config)) -user_old_dir = os.path.join(args.user_dir, 'old') -os.stat(user_old_dir) +ussd_data_dir = os.path.join(args.user_dir, 'ussd') db_configs = { 'database': config.get('DATABASE_NAME'), @@ -45,18 +45,15 @@ db_configs = { celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) if __name__ == '__main__': - for x in os.walk(user_old_dir): + for x in os.walk(ussd_data_dir): for y in x[2]: - if y[len(y) - 5:] != '.json': - continue - - # handle ussd_data json object - if y[:15] == '_ussd_data.json': + if y[len(y) - 5:] == '.json': filepath = os.path.join(x[0], y) f = open(filepath, 'r') try: ussd_data = json.load(f) + logg.debug(f'LOADING USSD DATA: {ussd_data}') except json.decoder.JSONDecodeError as e: f.close() logg.error('load error for {}: {}'.format(y, e)) diff --git a/apps/data-seeding/cic_ussd/import_util.py b/apps/data-seeding/cic_ussd/import_util.py index ebe706bb..d88b354a 100644 --- a/apps/data-seeding/cic_ussd/import_util.py +++ b/apps/data-seeding/cic_ussd/import_util.py @@ -6,7 +6,7 @@ from eth_contract_registry import Registry from eth_token_index import TokenUniqueSymbolIndex from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import OverrideNonceOracle -from chainlib.eth.erc20 import ERC20 +from eth_erc20 import ERC20 from chainlib.eth.tx import ( count, TxFormat, @@ -37,7 +37,7 @@ class BalanceProcessor: self.value_multiplier = 1 - def init(self): + def init(self, token_symbol): # Get Token registry address registry = Registry(self.chain_spec) o = registry.address_of(self.registry_address, 'TokenRegistry') @@ -46,10 +46,10 @@ class BalanceProcessor: logg.info('found token index address {}'.format(self.token_index_address)) token_registry = TokenUniqueSymbolIndex(self.chain_spec) - o = token_registry.address_of(self.token_index_address, 'SRF') + o = token_registry.address_of(self.token_index_address, token_symbol) r = self.conn.do(o) self.token_address = token_registry.parse_address_of(r) - logg.info('found SRF token address {}'.format(self.token_address)) + logg.info('found {} token address {}'.format(token_symbol, self.token_address)) tx_factory = ERC20(self.chain_spec) o = tx_factory.decimals(self.token_address) diff --git a/apps/data-seeding/create_import_pins.py b/apps/data-seeding/create_import_pins.py index daebe3ba..d4dd9716 100644 --- a/apps/data-seeding/create_import_pins.py +++ b/apps/data-seeding/create_import_pins.py @@ -3,6 +3,7 @@ import argparse import json import logging import os +import uuid # third-party imports import bcrypt @@ -83,7 +84,7 @@ if __name__ == '__main__': phone_object = phonenumbers.parse(u.tel) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) - password_hash = generate_password_hash() + password_hash = uuid.uuid4().hex pins_file.write(f'{phone},{password_hash}\n') logg.info(f'Writing phone: {phone}, password_hash: {password_hash}') diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 2e265264..af90c04d 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -9,6 +9,7 @@ import sys import urllib import urllib.request import uuid +import urllib.parse # external imports import celery @@ -72,7 +73,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification') -argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') +argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error') @@ -185,9 +186,9 @@ def send_ussd_request(address, data_dir): } req = urllib.request.Request(config.get('_USSD_PROVIDER')) - data_str = json.dumps(data) - data_bytes = data_str.encode('utf-8') - req.add_header('Content-Type', 'application/json') + urlencoded_data = urllib.parse.urlencode(data) + data_bytes = urlencoded_data.encode('utf-8') + req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.data = data_bytes response = urllib.request.urlopen(req) return response.read().decode('utf-8') @@ -388,10 +389,9 @@ class Verifier: def verify_ussd_pins(self, address, balance): response_data = send_ussd_request(address, self.data_dir) - if response_data[:11] != 'CON Balance': + if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio': raise VerifierError(response_data, 'pins') - def verify(self, address, balance, debug_stem=None): for k in active_tests: