diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py index 92c4147f..6a7485e9 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py @@ -31,13 +31,17 @@ from cic_ussd.processor import get_default_token_data from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore from cic_ussd.requests import (get_request_endpoint, get_request_method) -from cic_ussd.runnable.server_base import exportable_parser, logg +#from cic_ussd.runnable.server_base import exportable_parser, logg +from cic_ussd.runnable.server_base import exportable_parser from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession from cic_ussd.state_machine import UssdStateMachine from cic_ussd.validator import check_ip, check_request_content_length, validate_phone_number, validate_presence args = exportable_parser.parse_args() +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() + # define log levels if args.vv: logging.getLogger().setLevel(logging.DEBUG) @@ -139,6 +143,7 @@ def application(env, start_response): # define headers errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')] headers = [('Content-Type', 'text/plain')] + logg.debug('foo {}'.format(headers)) if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/': @@ -211,6 +216,9 @@ def application(env, start_response): return [response_bytes] else: + logg.error('invalid query {}'.format(env)) + for r in env: + logg.debug('{}: {}'.format(r, env)) start_response('405 Play by the rules', errors_headers) return [] diff --git a/apps/data-seeding/README.md b/apps/data-seeding/README.md index 32dde8d6..3583eed1 100644 --- a/apps/data-seeding/README.md +++ b/apps/data-seeding/README.md @@ -168,13 +168,38 @@ If you have previously run the `cic_ussd` import incompletely, it could be a goo Then, in sequence, run in first terminal: -`python cic_eth/import_balance.py -v -c config -p -r -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out` +`python cic_eth/import_balance.py -v -c config -p -r --token-symbol -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out` In second terminal: `python cic_ussd/import_users.py -v -c config out` + +### Step 4 - Metadata import (optional) + +The metadata import scripts can be run at any time after step 1 has been completed. + + +#### Importing user metadata + +To import the main user metadata structs, run: + +`node cic_meta/import_meta.js ` + +Monitors a folder for output from the `import_users.py` script, adding the metadata found to the `cic-meta` service. + +If _number of users_ is omitted the script will run until manually interrupted. + + + +#### Importing phone pointer + +`node cic_meta/import_meta_phone.js ` + +If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing. + + ##### Importing pins and ussd data (optional) Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps: @@ -199,29 +224,6 @@ The balance script is a celery task worker, and will not exit by itself in its c The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file. -### Step 4 - Metadata import (optional) - -The metadata import scripts can be run at any time after step 1 has been completed. - - -#### Importing user metadata - -To import the main user metadata structs, run: - -`node cic_meta/import_meta.js ` - -Monitors a folder for output from the `import_users.py` script, adding the metadata found to the `cic-meta` service. - -If _number of users_ is omitted the script will run until manually interrupted. - - -#### Importing phone pointer - -`node cic_meta/import_meta_phone.js ` - -If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing. - - ### Step 5 - Verify `python verify.py -v -c config -r -p ` diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 35430e95..a39cda63 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -114,7 +114,7 @@ def main(): conn = EthHTTPConnection(config.get('ETH_PROVIDER')) ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer) - ImportTask.balance_processor.init() + ImportTask.balance_processor.init(token_symbol) # TODO get decimals from token balances = {} diff --git a/apps/data-seeding/cic_ussd/import_users.py b/apps/data-seeding/cic_ussd/import_users.py index 1898c17c..57b37f15 100644 --- a/apps/data-seeding/cic_ussd/import_users.py +++ b/apps/data-seeding/cic_ussd/import_users.py @@ -8,9 +8,10 @@ import uuid import datetime import time import urllib.request +from urllib.parse import urlencode from glob import glob -# third-party imports +# external imports import redis import confini import celery @@ -87,21 +88,13 @@ chain_str = str(chain_spec) batch_size = args.batch_size batch_delay = args.batch_delay -db_configs = { - 'database': config.get('DATABASE_NAME'), - 'host': config.get('DATABASE_HOST'), - 'port': config.get('DATABASE_PORT'), - 'user': config.get('DATABASE_USER'), - 'password': config.get('DATABASE_PASSWORD') -} - def build_ussd_request(phone, host, port, service_code, username, password, ssl=False): url = 'http' if ssl: url += 's' url += '://{}:{}'.format(host, port) - url += '/?username={}&password={}'.format(username, password) #config.get('USSD_USER'), config.get('USSD_PASS')) + url += '/?username={}&password={}'.format(username, password) logg.info('ussd service url {}'.format(url)) logg.info('ussd phone {}'.format(phone)) @@ -114,9 +107,10 @@ def build_ussd_request(phone, host, port, service_code, username, password, ssl= 'text': service_code, } req = urllib.request.Request(url) - data_str = json.dumps(data) + req.method=('POST') + data_str = urlencode(data) data_bytes = data_str.encode('utf-8') - req.add_header('Content-Type', 'application/json') + req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.data = data_bytes return req diff --git a/apps/data-seeding/cic_ussd/import_util.py b/apps/data-seeding/cic_ussd/import_util.py index ebe706bb..607189af 100644 --- a/apps/data-seeding/cic_ussd/import_util.py +++ b/apps/data-seeding/cic_ussd/import_util.py @@ -6,7 +6,7 @@ from eth_contract_registry import Registry from eth_token_index import TokenUniqueSymbolIndex from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import OverrideNonceOracle -from chainlib.eth.erc20 import ERC20 +from eth_erc20 import ERC20 from chainlib.eth.tx import ( count, TxFormat, @@ -37,7 +37,7 @@ class BalanceProcessor: self.value_multiplier = 1 - def init(self): + def init(self, token_symbol): # Get Token registry address registry = Registry(self.chain_spec) o = registry.address_of(self.registry_address, 'TokenRegistry') @@ -46,7 +46,7 @@ class BalanceProcessor: logg.info('found token index address {}'.format(self.token_index_address)) token_registry = TokenUniqueSymbolIndex(self.chain_spec) - o = token_registry.address_of(self.token_index_address, 'SRF') + o = token_registry.address_of(self.token_index_address, token_symbol) r = self.conn.do(o) self.token_address = token_registry.parse_address_of(r) logg.info('found SRF token address {}'.format(self.token_address))