kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'lash/check-import-ussd' into 'master'

Browse Source 
Rehabilitate ussd import scripts

Closes #57

See merge request grassrootseconomics/cic-internal-integration!166
This commit is contained in:
Philip Wafula 2021-06-23 04:29:39 +00:00
commit 71e0973020
11 changed files with 46 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/cic-ussd/cic_ussd/processor.py
View File

@ -423,12 +423,6 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
:return: A ussd menu's corresponding text value.
:rtype: Document
"""
# retrieve metadata before any transition
key = generate_metadata_pointer(
identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
cic_type=':cic.person'
)
person_metadata = get_cached_data(key=key)
if ussd_session:
if user_input == "0":
@ -452,7 +446,7 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
'exit_pin_mismatch',
'exit_invalid_request',
'exit_successful_transaction'
] and person_metadata is not None:
]:
return UssdMenu.find_by_name(name='start')
else:
return UssdMenu.find_by_name(name=last_state)

5
apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py
View File

@ -145,7 +145,7 @@ def application(env, start_response):
if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/':
if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded':
start_response('405 Play by the rules', errors_headers)
start_response('405 Urlencoded, please', errors_headers)
return []
post_data = env.get('wsgi.input').read()
@ -213,6 +213,9 @@ def application(env, start_response):
return [response_bytes]
else:
logg.error('invalid query {}'.format(env))
for r in env:
logg.debug('{}: {}'.format(r, env))
start_response('405 Play by the rules', errors_headers)
return []

4
apps/cic-ussd/requirements.txt
View File

@ -1,4 +1,4 @@
cic_base[full_graph]~=0.1.2b15
cic-eth~=0.11.0b16
cic_base[full_graph]~=0.1.2b17
cic-eth~=0.11.0b17
cic-notify~=0.4.0a5
cic-types~=0.1.0a10

8
apps/data-seeding/README.md
View File

@ -136,7 +136,7 @@ First, make a note of the **block height** before running anything:
To import, run to _completion_:
`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
After the script completes, keystore files for all generated accouts will be found in `<datadir>/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one).
@ -150,7 +150,7 @@ Then run:
Run in sequence, in first terminal:
`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
In another terminal:
@ -226,7 +226,7 @@ The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in
### Step 5 - Verify
`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> <datadir>`
`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> <datadir>`
Included checks:
* Private key is in cic-eth keystore
@ -262,3 +262,5 @@ Should exit with code 0 if all input data is found in the respective services.
- MacOS BigSur issue when installing psycopg2: ld: library not found for -lssl -> https://github.com/psycopg/psycopg2/issues/1115#issuecomment-831498953
- `cic_ussd` imports is poorly implemented, and consumes a lot of resources. Therefore it takes a long time to complete. Reducing the amount of polls for the phone pointer would go a long way to improve it.
- A strict constraint is maintained insistin the use of postgresql-12.

3
apps/data-seeding/cic_ussd/import_balance.py
View File

@ -114,7 +114,7 @@ def main():
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer)
ImportTask.balance_processor.init()
ImportTask.balance_processor.init(token_symbol)
# TODO get decimals from token
balances = {}
@ -139,6 +139,7 @@ def main():
ImportTask.balances = balances
ImportTask.count = i
ImportTask.import_dir = user_dir
s = celery.signature(
'import_task.send_txs',

4
apps/data-seeding/cic_ussd/import_pins.py
View File

@ -39,6 +39,7 @@ elif args.vv:
config_dir = args.c
config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config.process()
logg.debug('config loaded from {}:\n{}'.format(args.c, config))
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@ -62,9 +63,6 @@ def main():
)
s_import_pins.apply_async()
argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
celery_app.worker_main(argv)
if __name__ == '__main__':
main()

43
apps/data-seeding/cic_ussd/import_users.py
View File

@ -1,29 +1,21 @@
# standard imports
import os
import sys
import argparse
import json
import logging
import argparse
import uuid
import datetime
import os
import sys
import time
import urllib.request
from glob import glob
import uuid
from urllib.parse import urlencode
# third-party imports
import redis
import confini
# external imports
import celery
from hexathon import (
add_0x,
strip_0x,
)
from chainlib.eth.address import to_checksum
from cic_types.models.person import Person
from cic_eth.api.api_task import Api
from chainlib.chain import ChainSpec
from cic_types.processor import generate_metadata_pointer
import confini
import phonenumbers
import redis
from chainlib.chain import ChainSpec
from cic_types.models.person import Person
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
@ -87,21 +79,13 @@ chain_str = str(chain_spec)
batch_size = args.batch_size
batch_delay = args.batch_delay
db_configs = {
'database': config.get('DATABASE_NAME'),
'host': config.get('DATABASE_HOST'),
'port': config.get('DATABASE_PORT'),
'user': config.get('DATABASE_USER'),
'password': config.get('DATABASE_PASSWORD')
}
def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
url = 'http'
if ssl:
url += 's'
url += '://{}:{}'.format(host, port)
url += '/?username={}&password={}'.format(username, password) #config.get('USSD_USER'), config.get('USSD_PASS'))
url += '/?username={}&password={}'.format(username, password)
logg.info('ussd service url {}'.format(url))
logg.info('ussd phone {}'.format(phone))
@ -114,9 +98,10 @@ def build_ussd_request(phone, host, port, service_code, username, password, ssl=
'text': service_code,
}
req = urllib.request.Request(url)
data_str = json.dumps(data)
req.method=('POST')
data_str = urlencode(data)
data_bytes = data_str.encode('utf-8')
req.add_header('Content-Type', 'application/json')
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
req.data = data_bytes
return req

13
apps/data-seeding/cic_ussd/import_ussd_data.py
View File

@ -31,9 +31,9 @@ elif args.vv:
config_dir = args.c
config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config.process()
logg.debug('config loaded from {}:\n{}'.format(args.c, config))
user_old_dir = os.path.join(args.user_dir, 'old')
os.stat(user_old_dir)
ussd_data_dir = os.path.join(args.user_dir, 'ussd')
db_configs = {
'database': config.get('DATABASE_NAME'),
@ -45,18 +45,15 @@ db_configs = {
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
if __name__ == '__main__':
for x in os.walk(user_old_dir):
for x in os.walk(ussd_data_dir):
for y in x[2]:
if y[len(y) - 5:] != '.json':
continue
# handle ussd_data json object
if y[:15] == '_ussd_data.json':
if y[len(y) - 5:] == '.json':
filepath = os.path.join(x[0], y)
f = open(filepath, 'r')
try:
ussd_data = json.load(f)
logg.debug(f'LOADING USSD DATA: {ussd_data}')
except json.decoder.JSONDecodeError as e:
f.close()
logg.error('load error for {}: {}'.format(y, e))

8
apps/data-seeding/cic_ussd/import_util.py
View File

@ -6,7 +6,7 @@ from eth_contract_registry import Registry
from eth_token_index import TokenUniqueSymbolIndex
from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.nonce import OverrideNonceOracle
from chainlib.eth.erc20 import ERC20
from eth_erc20 import ERC20
from chainlib.eth.tx import (
count,
TxFormat,
@ -37,7 +37,7 @@ class BalanceProcessor:
self.value_multiplier = 1
def init(self):
def init(self, token_symbol):
# Get Token registry address
registry = Registry(self.chain_spec)
o = registry.address_of(self.registry_address, 'TokenRegistry')
@ -46,10 +46,10 @@ class BalanceProcessor:
logg.info('found token index address {}'.format(self.token_index_address))
token_registry = TokenUniqueSymbolIndex(self.chain_spec)
o = token_registry.address_of(self.token_index_address, 'SRF')
o = token_registry.address_of(self.token_index_address, token_symbol)
r = self.conn.do(o)
self.token_address = token_registry.parse_address_of(r)
logg.info('found SRF token address {}'.format(self.token_address))
logg.info('found {} token address {}'.format(token_symbol, self.token_address))
tx_factory = ERC20(self.chain_spec)
o = tx_factory.decimals(self.token_address)

3
apps/data-seeding/create_import_pins.py
View File

@ -3,6 +3,7 @@ import argparse
import json
import logging
import os
import uuid
# third-party imports
import bcrypt
@ -83,7 +84,7 @@ if __name__ == '__main__':
phone_object = phonenumbers.parse(u.tel)
phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
password_hash = generate_password_hash()
password_hash = uuid.uuid4().hex
pins_file.write(f'{phone},{password_hash}\n')
logg.info(f'Writing phone: {phone}, password_hash: {password_hash}')

12
apps/data-seeding/verify.py
View File

@ -9,6 +9,7 @@ import sys
import urllib
import urllib.request
import uuid
import urllib.parse
# external imports
import celery
@ -72,7 +73,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification')
argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@ -185,9 +186,9 @@ def send_ussd_request(address, data_dir):
}
req = urllib.request.Request(config.get('_USSD_PROVIDER'))
data_str = json.dumps(data)
data_bytes = data_str.encode('utf-8')
req.add_header('Content-Type', 'application/json')
urlencoded_data = urllib.parse.urlencode(data)
data_bytes = urlencoded_data.encode('utf-8')
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
req.data = data_bytes
response = urllib.request.urlopen(req)
return response.read().decode('utf-8')
@ -388,10 +389,9 @@ class Verifier:
def verify_ussd_pins(self, address, balance):
response_data = send_ussd_request(address, self.data_dir)
if response_data[:11] != 'CON Balance':
if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
raise VerifierError(response_data, 'pins')
def verify(self, address, balance, debug_stem=None):
for k in active_tests: