diff --git a/.gitignore b/.gitignore index 459ec2b9..dabaefe2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ build/ **/*sqlite **/.nyc_output **/coverage +**/.venv +.idea diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index 0746df7f..ec03f6b7 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -16,6 +16,7 @@ import cic_base.config import cic_base.log import cic_base.argparse import cic_base.rpc +from cic_base.eth.syncer import chain_interface from cic_eth_registry import CICRegistry from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec @@ -28,10 +29,8 @@ from hexathon import ( strip_0x, ) from chainsyncer.backend.sql import SQLBackend -from chainsyncer.driver import ( - HeadSyncer, - HistorySyncer, - ) +from chainsyncer.driver.head import HeadSyncer +from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase # local imports @@ -113,10 +112,10 @@ def main(): logg.info('resuming sync session {}'.format(syncer_backend)) for syncer_backend in syncer_backends: - syncers.append(HistorySyncer(syncer_backend)) + syncers.append(HistorySyncer(syncer_backend, chain_interface)) syncer_backend = SQLBackend.live(chain_spec, block_offset+1) - syncers.append(HeadSyncer(syncer_backend)) + syncers.append(HeadSyncer(syncer_backend, chain_interface)) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index ad1232c3..2bc3a75a 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,12 +1,13 @@ -cic-base~=0.1.2b10 +cic-base==0.1.3a3+build.4aa03607 alembic==1.4.2 confini~=0.3.6rc3 uwsgi==2.0.19.1 moolb~=0.1.0 -cic-eth-registry~=0.5.5a4 +cic-eth-registry~=0.5.6a1 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainsyncer[sql]~=0.0.2a4 +chainsyncer[sql]~=0.0.3a3 +erc20-faucet~=0.2.2a1 diff --git a/apps/cic-cache/scripts/migrate.py b/apps/cic-cache/scripts/migrate.py index 6db1ed64..b6925144 100644 --- a/apps/cic-cache/scripts/migrate.py +++ b/apps/cic-cache/scripts/migrate.py @@ -2,6 +2,7 @@ import os import argparse import logging +import re import alembic from alembic.config import Config as AlembicConfig @@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser() argparser.add_argument('-c', type=str, default=config_dir, help='config file') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') +argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') +argparser.add_argument('-f', action='store_true', help='force action') argparser.add_argument('-v', action='store_true', help='be verbose') argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() @@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini')) ac.set_main_option('sqlalchemy.url', dsn) ac.set_main_option('script_location', migrations_dir) +if args.reset: + if not args.f: + if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')): + logg.error('user chickened out on requested reset, bailing') + sys.exit(1) + alembic.command.downgrade(ac, 'base') alembic.command.upgrade(ac, 'head') diff --git a/apps/cic-cache/test_requirements.txt b/apps/cic-cache/test_requirements.txt index e0addadd..f0c5fd81 100644 --- a/apps/cic-cache/test_requirements.txt +++ b/apps/cic-cache/test_requirements.txt @@ -6,6 +6,5 @@ sqlparse==0.4.1 pytest-celery==0.0.0a1 eth_tester==0.5.0b3 py-evm==0.3.0a20 -web3==5.12.2 -cic-eth-registry~=0.5.5a3 -cic-base[full]==0.1.2b8 +cic_base[full]==0.1.3a3+build.4aa03607 +sarafu-faucet~=0.0.4a1 diff --git a/apps/cic-eth/MANIFEST.in b/apps/cic-eth/MANIFEST.in new file mode 100644 index 00000000..739a53c6 --- /dev/null +++ b/apps/cic-eth/MANIFEST.in @@ -0,0 +1,2 @@ +include *requirements.txt + diff --git a/apps/cic-eth/cic_eth/api/api_admin.py b/apps/cic-eth/cic_eth/api/api_admin.py index c5c1e142..34fc4d68 100644 --- a/apps/cic-eth/cic_eth/api/api_admin.py +++ b/apps/cic-eth/cic_eth/api/api_admin.py @@ -562,13 +562,13 @@ class AdminApi: tx['source_token_symbol'] = source_token.symbol o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address) r = self.rpc.do(o) - tx['sender_token_balance'] = erc20_c.parse_balance_of(r) + tx['sender_token_balance'] = erc20_c.parse_balance(r) if destination_token != None: tx['destination_token_symbol'] = destination_token.symbol o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address) r = self.rpc.do(o) - tx['recipient_token_balance'] = erc20_c.parse_balance_of(r) + tx['recipient_token_balance'] = erc20_c.parse_balance(r) #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call() # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which diff --git a/apps/cic-eth/cic_eth/runnable/daemons/server.py b/apps/cic-eth/cic_eth/runnable/daemons/server.py deleted file mode 100644 index 875c581e..00000000 --- a/apps/cic-eth/cic_eth/runnable/daemons/server.py +++ /dev/null @@ -1,136 +0,0 @@ -# standard imports -import os -import re -import logging -import argparse -import json - -# third-party imports -import web3 -import confini -import celery -from json.decoder import JSONDecodeError -from cic_registry.chain import ChainSpec - -# local imports -from cic_eth.db import dsn_from_config -from cic_eth.db.models.base import SessionBase -from cic_eth.eth.util import unpack_signed_raw_tx - -logging.basicConfig(level=logging.WARNING) -logg = logging.getLogger() - -rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -dbdir = os.path.join(rootdir, 'cic_eth', 'db') -migrationsdir = os.path.join(dbdir, 'migrations') - -config_dir = os.path.join('/usr/local/etc/cic-eth') - -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks') -argparser.add_argument('-v', action='store_true', help='be verbose') -argparser.add_argument('-vv', action='store_true', help='be more verbose') -args = argparser.parse_args() - -if args.vv: - logging.getLogger().setLevel(logging.DEBUG) -elif args.v: - logging.getLogger().setLevel(logging.INFO) - -config = confini.Config(args.c, args.env_prefix) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config:\n{}'.format(config)) - -dsn = dsn_from_config(config) -SessionBase.connect(dsn) - -celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) -queue = args.q - -re_something = r'^/something/?' - -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) - - -def process_something(session, env): - r = re.match(re_something, env.get('PATH_INFO')) - if not r: - return None - - #if env.get('CONTENT_TYPE') != 'application/json': - # raise AttributeError('content type') - - #if env.get('REQUEST_METHOD') != 'POST': - # raise AttributeError('method') - - #post_data = json.load(env.get('wsgi.input')) - - #return ('text/plain', 'foo'.encode('utf-8'),) - - -# uwsgi application -def application(env, start_response): - - for k in env.keys(): - logg.debug('env {} {}'.format(k, env[k])) - - headers = [] - content = b'' - err = None - - session = SessionBase.create_session() - for handler in [ - process_something, - ]: - try: - r = handler(session, env) - except AttributeError as e: - logg.error('handler fail attribute {}'.format(e)) - err = '400 Impertinent request' - break - except JSONDecodeError as e: - logg.error('handler fail json {}'.format(e)) - err = '400 Invalid data format' - break - except KeyError as e: - logg.error('handler fail key {}'.format(e)) - err = '400 Invalid JSON' - break - except ValueError as e: - logg.error('handler fail value {}'.format(e)) - err = '400 Invalid data' - break - except RuntimeError as e: - logg.error('task fail value {}'.format(e)) - err = '500 Task failed, sorry I cannot tell you more' - break - if r != None: - (mime_type, content) = r - break - session.close() - - if err != None: - headers.append(('Content-Type', 'text/plain, charset=UTF-8',)) - start_response(err, headers) - session.close() - return [content] - - headers.append(('Content-Length', str(len(content))),) - headers.append(('Access-Control-Allow-Origin', '*',)); - - if len(content) == 0: - headers.append(('Content-Type', 'text/plain, charset=UTF-8',)) - start_response('404 Looked everywhere, sorry', headers) - else: - headers.append(('Content-Type', mime_type,)) - start_response('200 OK', headers) - - return [content] diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 6d5ba557..874c378b 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -194,6 +194,7 @@ def main(): except UnknownContractError as e: logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) sys.exit(1) + logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 10ef5154..0a977825 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -15,6 +15,7 @@ import cic_base.config import cic_base.log import cic_base.argparse import cic_base.rpc +from cic_base.eth.syncer import chain_interface from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec from chainlib.eth.constant import ZERO_ADDRESS @@ -26,10 +27,8 @@ from hexathon import ( strip_0x, ) from chainsyncer.backend.sql import SQLBackend -from chainsyncer.driver import ( - HeadSyncer, - HistorySyncer, - ) +from chainsyncer.driver.head import HeadSyncer +from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase # local imports @@ -80,6 +79,7 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) + def main(): # connect to celery celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) @@ -121,11 +121,11 @@ def main(): for syncer_backend in syncer_backends: try: - syncers.append(HistorySyncer(syncer_backend)) + syncers.append(HistorySyncer(syncer_backend, chain_interface)) logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend)) except AttributeError: logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend)) - syncers.append(HeadSyncer(syncer_backend)) + syncers.append(HeadSyncer(syncer_backend, chain_interface)) connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index fd65bab3..9a7a232e 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -9,8 +9,8 @@ import semver version = ( 0, 11, - 0, - 'beta.17', + 1, + 'alpha.2', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index ac143555..207045ed 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,25 +1,25 @@ -cic-base~=0.1.2b17 +cic-base==0.1.3a3+build.4aa03607 celery==4.4.7 -crypto-dev-signer~=0.4.14b3 +crypto-dev-signer~=0.4.14b6 confini~=0.3.6rc3 -cic-eth-registry~=0.5.5a7 +cic-eth-registry~=0.5.6a1 redis==3.5.3 alembic==1.4.2 websockets==8.1 requests~=2.24.0 -eth_accounts_index~=0.0.11a12 -erc20-transfer-authorization~=0.3.1a7 +eth_accounts_index~=0.0.12a1 +erc20-transfer-authorization~=0.3.2a1 uWSGI==2.0.19.1 semver==2.13.0 websocket-client==0.57.0 moolb~=0.1.1b2 -eth-address-index~=0.1.1a11 -chainlib~=0.0.3rc2 +eth-address-index~=0.1.2a1 +chainlib-eth~=0.0.5a1 hexathon~=0.0.1a7 -chainsyncer[sql]~=0.0.2a5 -chainqueue~=0.0.2b3 -sarafu-faucet~=0.0.3a4 -erc20-faucet~=0.2.1a4 +chainsyncer[sql]~=0.0.3a3 +chainqueue~=0.0.2b5 +sarafu-faucet~=0.0.4a1 +erc20-faucet~=0.2.2a1 coincurve==15.0.0 potaahto~=0.0.1a2 pycryptodome==3.10.1 diff --git a/apps/cic-eth/scripts/migrate.py b/apps/cic-eth/scripts/migrate.py index 4a2c0139..d3602989 100644 --- a/apps/cic-eth/scripts/migrate.py +++ b/apps/cic-eth/scripts/migrate.py @@ -2,6 +2,8 @@ import os import argparse import logging +import re +import sys import alembic from alembic.config import Config as AlembicConfig @@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser() argparser.add_argument('-c', type=str, default=config_dir, help='config file') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') +argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') +argparser.add_argument('-f', action='store_true', help='force action') argparser.add_argument('-v', action='store_true', help='be verbose') argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() @@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini')) ac.set_main_option('sqlalchemy.url', dsn) ac.set_main_option('script_location', migrations_dir) +if args.reset: + if not args.f: + if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')): + logg.error('user chickened out on requested reset, bailing') + sys.exit(1) + alembic.command.downgrade(ac, 'base') alembic.command.upgrade(ac, 'head') diff --git a/apps/cic-notify/cic_notify/api.py b/apps/cic-notify/cic_notify/api.py index f879a6ee..fb27f7bd 100644 --- a/apps/cic-notify/cic_notify/api.py +++ b/apps/cic-notify/cic_notify/api.py @@ -26,7 +26,7 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'): for q in qs[host]: if re.match(re_q, q['name']): host_queues.append((host, q['name'],)) - + task_prefix_len = len(task_prefix) queue_tasks = [] for (host, queue) in host_queues: @@ -35,17 +35,18 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'): for task in tasks: if len(task) >= task_prefix_len and task[:task_prefix_len] == task_prefix: queue_tasks.append((queue, task,)) - + return queue_tasks class Api: # TODO: Implement callback strategy - def __init__(self, queue='cic-notify'): + def __init__(self, queue=None): """ :param queue: The queue on which to execute notification tasks :type queue: str """ + self.queue = queue self.sms_tasks = get_sms_queue_tasks(app) logg.debug('sms tasks {}'.format(self.sms_tasks)) @@ -61,13 +62,19 @@ class Api: """ signatures = [] for q in self.sms_tasks: + + if not self.queue: + queue = q[0] + else: + queue = self.queue + signature = celery.signature( q[1], [ message, recipient, ], - queue=q[0], + queue=queue, ) signatures.append(signature) diff --git a/apps/cic-notify/cic_notify/runnable/tasker.py b/apps/cic-notify/cic_notify/runnable/tasker.py index 453044f0..90417c7e 100644 --- a/apps/cic-notify/cic_notify/runnable/tasker.py +++ b/apps/cic-notify/cic_notify/runnable/tasker.py @@ -87,10 +87,18 @@ for key in config.store.keys(): module = importlib.import_module(config.store[key]) if key == 'TASKS_AFRICASTALKING': africastalking_notifier = module.AfricasTalkingNotifier + + api_sender_id = config.get('AFRICASTALKING_API_SENDER_ID') + logg.debug(f'SENDER ID VALUE IS: {api_sender_id}') + + if not api_sender_id: + api_sender_id = None + logg.debug(f'SENDER ID RESOLVED TO NONE: {api_sender_id}') + africastalking_notifier.initialize( config.get('AFRICASTALKING_API_USERNAME'), config.get('AFRICASTALKING_API_KEY'), - config.get('AFRICASTALKING_API_SENDER_ID') + api_sender_id ) diff --git a/apps/cic-notify/cic_notify/version.py b/apps/cic-notify/cic_notify/version.py index e795aad8..4ba6055f 100644 --- a/apps/cic-notify/cic_notify/version.py +++ b/apps/cic-notify/cic_notify/version.py @@ -9,7 +9,7 @@ import semver logg = logging.getLogger() -version = (0, 4, 0, 'alpha.5') +version = (0, 4, 0, 'alpha.6') version_object = semver.VersionInfo( major=version[0], diff --git a/apps/cic-notify/requirements.txt b/apps/cic-notify/requirements.txt index 0b7de273..67db2dfb 100644 --- a/apps/cic-notify/requirements.txt +++ b/apps/cic-notify/requirements.txt @@ -1 +1 @@ -cic_base[full_graph]~=0.1.2a61 +cic_base[full_graph]==0.1.3a3+build.4aa03607 diff --git a/apps/cic-notify/test_requirements.txt b/apps/cic-notify/test_requirements.txt index fda03d35..41c5f099 100644 --- a/apps/cic-notify/test_requirements.txt +++ b/apps/cic-notify/test_requirements.txt @@ -2,4 +2,3 @@ pytest~=6.0.1 pytest-celery~=0.0.0a1 pytest-mock~=3.3.1 pysqlite3~=0.4.3 - diff --git a/apps/cic-ussd/cic_ussd/db/enum.py b/apps/cic-ussd/cic_ussd/db/enum.py new file mode 100644 index 00000000..cb9761d3 --- /dev/null +++ b/apps/cic-ussd/cic_ussd/db/enum.py @@ -0,0 +1,9 @@ +# standard import +from enum import IntEnum + + +class AccountStatus(IntEnum): + PENDING = 1 + ACTIVE = 2 + LOCKED = 3 + RESET = 4 diff --git a/apps/cic-ussd/cic_ussd/db/models/account.py b/apps/cic-ussd/cic_ussd/db/models/account.py index 18f5e370..1e073625 100644 --- a/apps/cic-ussd/cic_ussd/db/models/account.py +++ b/apps/cic-ussd/cic_ussd/db/models/account.py @@ -1,19 +1,13 @@ # standard imports -from enum import IntEnum - -# third party imports -from sqlalchemy import Column, Integer, String # local imports +from cic_ussd.db.enum import AccountStatus from cic_ussd.db.models.base import SessionBase from cic_ussd.encoder import check_password_hash, create_password_hash - -class AccountStatus(IntEnum): - PENDING = 1 - ACTIVE = 2 - LOCKED = 3 - RESET = 4 +# third party imports +from sqlalchemy import Column, Integer, String +from sqlalchemy.orm.session import Session class Account(SessionBase): @@ -30,6 +24,21 @@ class Account(SessionBase): account_status = Column(Integer) preferred_language = Column(String) + @staticmethod + def get_by_phone_number(phone_number: str, session: Session): + """Retrieves an account from a phone number. + :param phone_number: The E164 format of a phone number. + :type phone_number:str + :param session: + :type session: + :return: An account object. + :rtype: Account + """ + session = SessionBase.bind_session(session=session) + account = session.query(Account).filter_by(phone_number=phone_number).first() + SessionBase.release_session(session=session) + return account + def __init__(self, blockchain_address, phone_number): self.blockchain_address = blockchain_address self.phone_number = phone_number diff --git a/apps/cic-ussd/cic_ussd/db/ussd_menu.json b/apps/cic-ussd/cic_ussd/db/ussd_menu.json index ecc98173..bbb23d74 100644 --- a/apps/cic-ussd/cic_ussd/db/ussd_menu.json +++ b/apps/cic-ussd/cic_ussd/db/ussd_menu.json @@ -275,6 +275,18 @@ "display_key": "ussd.kenya.new_pin_confirmation", "name": "new_pin_confirmation", "parent": "metadata_management" + }, + "47": { + "description": "Year of birth entry menu.", + "display_key": "ussd.kenya.enter_date_of_birth", + "name": "enter_date_of_birth", + "parent": "metadata_management" + }, + "48": { + "description": "Pin entry menu for changing year of birth data.", + "display_key": "ussd.kenya.dob_edit_pin_authorization", + "name": "dob_edit_pin_authorization", + "parent": "metadata_management" } } diff --git a/apps/cic-ussd/cic_ussd/metadata/custom.py b/apps/cic-ussd/cic_ussd/metadata/custom.py new file mode 100644 index 00000000..6ab13d00 --- /dev/null +++ b/apps/cic-ussd/cic_ussd/metadata/custom.py @@ -0,0 +1,12 @@ +# standard imports + +# external imports + +# local imports +from .base import MetadataRequestsHandler + + +class CustomMetadata(MetadataRequestsHandler): + + def __init__(self, identifier: bytes): + super().__init__(cic_type=':cic.custom', identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/metadata/preferences.py b/apps/cic-ussd/cic_ussd/metadata/preferences.py new file mode 100644 index 00000000..88210d1a --- /dev/null +++ b/apps/cic-ussd/cic_ussd/metadata/preferences.py @@ -0,0 +1,12 @@ +# standard imports + +# external imports + +# local imports +from .base import MetadataRequestsHandler + + +class PreferencesMetadata(MetadataRequestsHandler): + + def __init__(self, identifier: bytes): + super().__init__(cic_type=':cic.preferences', identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/operations.py b/apps/cic-ussd/cic_ussd/operations.py index 5f2ca7a2..bd0e7476 100644 --- a/apps/cic-ussd/cic_ussd/operations.py +++ b/apps/cic-ussd/cic_ussd/operations.py @@ -6,11 +6,13 @@ import logging import celery import i18n from cic_eth.api.api_task import Api +from sqlalchemy.orm.session import Session from tinydb.table import Document from typing import Optional # local imports from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase from cic_ussd.db.models.ussd_session import UssdSession from cic_ussd.db.models.task_tracker import TaskTracker from cic_ussd.menu.ussd_menu import UssdMenu @@ -22,15 +24,18 @@ from cic_ussd.validator import check_known_user, validate_response_type logg = logging.getLogger() -def add_tasks_to_tracker(task_uuid): - """ - This function takes tasks spawned over api interfaces and records their creation time for tracking. +def add_tasks_to_tracker(session, task_uuid: str): + """This function takes tasks spawned over api interfaces and records their creation time for tracking. + :param session: + :type session: :param task_uuid: The uuid for an initiated task. :type task_uuid: str """ + session = SessionBase.bind_session(session=session) task_record = TaskTracker(task_uuid=task_uuid) - TaskTracker.session.add(task_record) - TaskTracker.session.commit() + session.add(task_record) + session.flush() + SessionBase.release_session(session=session) def define_response_with_content(headers: list, response: str) -> tuple: @@ -95,6 +100,7 @@ def create_or_update_session( service_code: str, user_input: str, current_menu: str, + session, session_data: Optional[dict] = None) -> InMemoryUssdSession: """ Handles the creation or updating of session as necessary. @@ -108,12 +114,15 @@ def create_or_update_session( :type user_input: str :param current_menu: Menu name that is currently being displayed on the ussd session :type current_menu: str + :param session: + :type session: :param session_data: Any additional data that was persisted during the user's interaction with the system. :type session_data: dict. :return: ussd session object :rtype: InMemoryUssdSession """ - existing_ussd_session = UssdSession.session.query(UssdSession).filter_by( + session = SessionBase.bind_session(session=session) + existing_ussd_session = session.query(UssdSession).filter_by( external_session_id=external_session_id).first() if existing_ussd_session: @@ -132,20 +141,25 @@ def create_or_update_session( current_menu=current_menu, session_data=session_data ) + SessionBase.release_session(session=session) return ussd_session -def get_account_status(phone_number) -> str: +def get_account_status(phone_number, session: Session) -> str: """Get the status of a user's account. :param phone_number: The phone number to be checked. :type phone_number: str + :param session: + :type session: :return: The user account status. :rtype: str """ - user = Account.session.query(Account).filter_by(phone_number=phone_number).first() - status = user.get_account_status() - Account.session.add(user) - Account.session.commit() + session = SessionBase.bind_session(session=session) + account = Account.get_by_phone_number(phone_number=phone_number, session=session) + status = account.get_account_status() + session.add(account) + session.flush() + SessionBase.release_session(session=session) return status @@ -165,6 +179,7 @@ def initiate_account_creation_request(chain_str: str, external_session_id: str, phone_number: str, service_code: str, + session, user_input: str) -> str: """This function issues a task to create a blockchain account on cic-eth. It then creates a record of the ussd session corresponding to the creation of the account and returns a response denoting that the user's account is @@ -177,6 +192,8 @@ def initiate_account_creation_request(chain_str: str, :type phone_number: str :param service_code: The service code dialed. :type service_code: str + :param session: + :type session: :param user_input: The input entered by the user. :type user_input: str :return: A response denoting that the account is being created. @@ -190,7 +207,7 @@ def initiate_account_creation_request(chain_str: str, creation_task_id = cic_eth_api.create_account().id # record task initiation time - add_tasks_to_tracker(task_uuid=creation_task_id) + add_tasks_to_tracker(task_uuid=creation_task_id, session=session) # cache account creation data cache_account_creation_task_id(phone_number=phone_number, task_id=creation_task_id) @@ -204,6 +221,7 @@ def initiate_account_creation_request(chain_str: str, phone=phone_number, service_code=service_code, current_menu=current_menu.get('name'), + session=session, user_input=user_input) # define response to relay to user @@ -268,12 +286,14 @@ def cache_account_creation_task_id(phone_number: str, task_id: str): redis_cache.persist(name=task_id) -def process_current_menu(ussd_session: Optional[dict], user: Account, user_input: str) -> Document: +def process_current_menu(account: Account, session: Session, ussd_session: Optional[dict], user_input: str) -> Document: """This function checks user input and returns a corresponding ussd menu :param ussd_session: An in db ussd session object. :type ussd_session: UssdSession - :param user: A user object. - :type user: Account + :param account: A account object. + :type account: Account + :param session: + :type session: :param user_input: The user's input. :type user_input: str :return: An in memory ussd menu object. @@ -285,7 +305,13 @@ def process_current_menu(ussd_session: Optional[dict], user: Account, user_input else: # get current state latest_input = get_latest_input(user_input=user_input) - current_menu = process_request(ussd_session=ussd_session, user_input=latest_input, user=user) + session = SessionBase.bind_session(session=session) + current_menu = process_request( + account=account, + session=session, + ussd_session=ussd_session, + user_input=latest_input) + SessionBase.release_session(session=session) return current_menu @@ -294,6 +320,7 @@ def process_menu_interaction_requests(chain_str: str, phone_number: str, queue: str, service_code: str, + session, user_input: str) -> str: """This function handles requests intended for interaction with ussd menu, it checks whether a user matching the provided phone number exists and in the absence of which it creates an account for the user. @@ -308,25 +335,29 @@ def process_menu_interaction_requests(chain_str: str, :type queue: str :param service_code: The service dialed by the user making the request. :type service_code: str + :param session: + :type session: :param user_input: The inputs entered by the user. :type user_input: str :return: A response based on the request received. :rtype: str """ # check whether the user exists - if not check_known_user(phone=phone_number): + if not check_known_user(phone_number=phone_number, session=session): response = initiate_account_creation_request(chain_str=chain_str, external_session_id=external_session_id, phone_number=phone_number, service_code=service_code, + session=session, user_input=user_input) else: - # get user - user = Account.session.query(Account).filter_by(phone_number=phone_number).first() + # get account + session = SessionBase.bind_session(session=session) + account = Account.get_by_phone_number(phone_number=phone_number, session=session) # retrieve and cache user's metadata - blockchain_address = user.blockchain_address + blockchain_address = account.blockchain_address s_query_person_metadata = celery.signature( 'cic_ussd.tasks.metadata.query_person_metadata', [blockchain_address] @@ -334,24 +365,25 @@ def process_menu_interaction_requests(chain_str: str, s_query_person_metadata.apply_async(queue='cic-ussd') # find any existing ussd session - existing_ussd_session = UssdSession.session.query(UssdSession).filter_by( - external_session_id=external_session_id).first() + existing_ussd_session = session.query(UssdSession).filter_by(external_session_id=external_session_id).first() # validate user inputs if existing_ussd_session: current_menu = process_current_menu( + account=account, + session=session, ussd_session=existing_ussd_session.to_json(), - user=user, user_input=user_input ) else: current_menu = process_current_menu( + account=account, + session=session, ussd_session=None, - user=user, user_input=user_input ) - last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number) + last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session) if last_ussd_session: # create or update the ussd session as appropriate @@ -361,6 +393,7 @@ def process_menu_interaction_requests(chain_str: str, service_code=service_code, user_input=user_input, current_menu=current_menu.get('name'), + session=session, session_data=last_ussd_session.session_data ) else: @@ -369,15 +402,17 @@ def process_menu_interaction_requests(chain_str: str, phone=phone_number, service_code=service_code, user_input=user_input, - current_menu=current_menu.get('name') + current_menu=current_menu.get('name'), + session=session ) # define appropriate response response = custom_display_text( + account=account, display_key=current_menu.get('display_key'), menu_name=current_menu.get('name'), + session=session, ussd_session=ussd_session.to_json(), - user=user ) # check that the response from the processor is valid @@ -386,21 +421,26 @@ def process_menu_interaction_requests(chain_str: str, # persist session to db persist_session_to_db_task(external_session_id=external_session_id, queue=queue) + SessionBase.release_session(session=session) return response -def reset_pin(phone_number: str) -> str: +def reset_pin(phone_number: str, session: Session) -> str: """Reset account status from Locked to Pending. :param phone_number: The phone number belonging to the account to be unlocked. :type phone_number: str + :param session: + :type session: :return: The status of the pin reset. :rtype: str """ - user = Account.session.query(Account).filter_by(phone_number=phone_number).first() - user.reset_account_pin() - Account.session.add(user) - Account.session.commit() + session = SessionBase.bind_session(session=session) + account = Account.get_by_phone_number(phone_number=phone_number, session=session) + account.reset_account_pin() + session.add(account) + session.flush() + SessionBase.release_session(session=session) response = f'Pin reset for user {phone_number} is successful!' return response @@ -438,11 +478,13 @@ def update_ussd_session( return session -def save_to_in_memory_ussd_session_data(queue: str, session_data: dict, ussd_session: dict): +def save_to_in_memory_ussd_session_data(queue: str, session: Session, session_data: dict, ussd_session: dict): """This function is used to save information to the session data attribute of a ussd session object in the redis cache. :param queue: The queue on which the celery task should run. :type queue: str + :param session: + :type session: :param session_data: A dictionary containing data for a specific ussd session in redis that needs to be saved temporarily. :type session_data: dict @@ -473,7 +515,7 @@ def save_to_in_memory_ussd_session_data(queue: str, session_data: dict, ussd_ses service_code=in_redis_ussd_session.get('service_code'), user_input=in_redis_ussd_session.get('user_input'), current_menu=in_redis_ussd_session.get('state'), + session=session, session_data=session_data ) persist_session_to_db_task(external_session_id=external_session_id, queue=queue) - diff --git a/apps/cic-ussd/cic_ussd/phone_number.py b/apps/cic-ussd/cic_ussd/phone_number.py index 2c476fa0..1fc94f6f 100644 --- a/apps/cic-ussd/cic_ussd/phone_number.py +++ b/apps/cic-ussd/cic_ussd/phone_number.py @@ -8,6 +8,10 @@ import phonenumbers from cic_ussd.db.models.account import Account +class E164Format: + region = None + + def process_phone_number(phone_number: str, region: str): """This function parses any phone number for the provided region :param phone_number: A string with a phone number. @@ -29,19 +33,5 @@ def process_phone_number(phone_number: str, region: str): return parsed_phone_number - -def get_user_by_phone_number(phone_number: str) -> Optional[Account]: - """This function queries the database for a user based on the provided phone number. - :param phone_number: A valid phone number. - :type phone_number: str - :return: A user object matching a given phone number - :rtype: Account|None - """ - # consider adding region to user's metadata - phone_number = process_phone_number(phone_number=phone_number, region='KE') - user = Account.session.query(Account).filter_by(phone_number=phone_number).first() - return user - - class Support: phone_number = None diff --git a/apps/cic-ussd/cic_ussd/processor.py b/apps/cic-ussd/cic_ussd/processor.py index a838d308..9f9bccd3 100644 --- a/apps/cic-ussd/cic_ussd/processor.py +++ b/apps/cic-ussd/cic_ussd/processor.py @@ -1,25 +1,27 @@ # standard imports +import datetime import logging import json -import re from typing import Optional # third party imports -import celery from sqlalchemy import desc from cic_eth.api import Api +from sqlalchemy.orm.session import Session from tinydb.table import Document # local imports from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance from cic_ussd.chain import Chain -from cic_ussd.db.models.account import AccountStatus, Account +from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase from cic_ussd.db.models.ussd_session import UssdSession -from cic_ussd.error import MetadataNotFoundError, SeppukuError +from cic_ussd.db.enum import AccountStatus +from cic_ussd.error import SeppukuError from cic_ussd.menu.ussd_menu import UssdMenu from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.phone_number import get_user_by_phone_number, Support +from cic_ussd.phone_number import Support from cic_ussd.redis import cache_data, create_cached_data_key, get_cached_data from cic_ussd.state_machine import UssdStateMachine from cic_ussd.conversions import to_wei, from_wei @@ -61,47 +63,48 @@ def retrieve_token_symbol(chain_str: str = Chain.spec.__str__()): raise SeppukuError(f'Could not retrieve default token for: {chain_str}') -def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str: - """ - This method provides translation for all ussd menu entries that follow the pin authorization pattern. +def process_pin_authorization(account: Account, display_key: str, **kwargs) -> str: + """This method provides translation for all ussd menu entries that follow the pin authorization pattern. + :param account: The account in a running USSD session. + :type account: Account :param display_key: The path in the translation files defining an appropriate ussd response :type display_key: str - :param user: The user in a running USSD session. - :type user: Account :param kwargs: Any additional information required by the text values in the internationalization files. :type kwargs :return: A string value corresponding the ussd menu's text value. :rtype: str """ remaining_attempts = 3 - if user.failed_pin_attempts > 0: + if account.failed_pin_attempts > 0: return translation_for( key=f'{display_key}.retry', - preferred_language=user.preferred_language, - remaining_attempts=(remaining_attempts - user.failed_pin_attempts) + preferred_language=account.preferred_language, + remaining_attempts=(remaining_attempts - account.failed_pin_attempts) ) else: return translation_for( key=f'{display_key}.first', - preferred_language=user.preferred_language, + preferred_language=account.preferred_language, **kwargs ) -def process_exit_insufficient_balance(display_key: str, user: Account, ussd_session: dict): +def process_exit_insufficient_balance(account: Account, display_key: str, session: Session, ussd_session: dict): """This function processes the exit menu letting users their account balance is insufficient to perform a specific transaction. + :param account: The account requesting access to the ussd menu. + :type account: Account :param display_key: The path in the translation files defining an appropriate ussd response :type display_key: str - :param user: The user requesting access to the ussd menu. - :type user: Account + :param session: + :type session: :param ussd_session: A JSON serialized in-memory ussd session object :type ussd_session: dict :return: Corresponding translation text response :rtype: str """ # get account balance - operational_balance = get_cached_operational_balance(blockchain_address=user.blockchain_address) + operational_balance = get_cached_operational_balance(blockchain_address=account.blockchain_address) # compile response data user_input = ussd_session.get('user_input').split('*')[-1] @@ -111,13 +114,13 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess token_symbol = retrieve_token_symbol() recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') - recipient = get_user_by_phone_number(phone_number=recipient_phone_number) + recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) tx_recipient_information = define_account_tx_metadata(user=recipient) return translation_for( key=display_key, - preferred_language=user.preferred_language, + preferred_language=account.preferred_language, amount=from_wei(transaction_amount), token_symbol=token_symbol, recipient_information=tx_recipient_information, @@ -125,12 +128,14 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess ) -def process_exit_successful_transaction(display_key: str, user: Account, ussd_session: dict): +def process_exit_successful_transaction(account: Account, display_key: str, session: Session, ussd_session: dict): """This function processes the exit menu after a successful initiation for a transfer of tokens. + :param account: The account requesting access to the ussd menu. + :type account: Account :param display_key: The path in the translation files defining an appropriate ussd response :type display_key: str - :param user: The user requesting access to the ussd menu. - :type user: Account + :param session: + :type session: :param ussd_session: A JSON serialized in-memory ussd session object :type ussd_session: dict :return: Corresponding translation text response @@ -139,13 +144,13 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se transaction_amount = to_wei(int(ussd_session.get('session_data').get('transaction_amount'))) token_symbol = retrieve_token_symbol() recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') - recipient = get_user_by_phone_number(phone_number=recipient_phone_number) + recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) tx_recipient_information = define_account_tx_metadata(user=recipient) - tx_sender_information = define_account_tx_metadata(user=user) + tx_sender_information = define_account_tx_metadata(user=account) return translation_for( key=display_key, - preferred_language=user.preferred_language, + preferred_language=account.preferred_language, transaction_amount=from_wei(transaction_amount), token_symbol=token_symbol, recipient_information=tx_recipient_information, @@ -153,13 +158,15 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se ) -def process_transaction_pin_authorization(user: Account, display_key: str, ussd_session: dict): +def process_transaction_pin_authorization(account: Account, display_key: str, session: Session, ussd_session: dict): """This function processes pin authorization where making a transaction is concerned. It constructs a pre-transaction response menu that shows the details of the transaction. - :param user: The user requesting access to the ussd menu. - :type user: Account + :param account: The account requesting access to the ussd menu. + :type account: Account :param display_key: The path in the translation files defining an appropriate ussd response :type display_key: str + :param session: + :type session: :param ussd_session: The USSD session determining what user data needs to be extracted and added to the menu's text values. :type ussd_session: UssdSession @@ -168,16 +175,16 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_ """ # compile response data recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') - recipient = get_user_by_phone_number(phone_number=recipient_phone_number) + recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) tx_recipient_information = define_account_tx_metadata(user=recipient) - tx_sender_information = define_account_tx_metadata(user=user) + tx_sender_information = define_account_tx_metadata(user=account) token_symbol = retrieve_token_symbol() user_input = ussd_session.get('session_data').get('transaction_amount') transaction_amount = to_wei(value=int(user_input)) logg.debug('Requires integration to determine user tokens.') return process_pin_authorization( - user=user, + account=account, display_key=display_key, recipient_information=tx_recipient_information, transaction_amount=from_wei(transaction_amount), @@ -186,14 +193,12 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_ ) -def process_account_balances(user: Account, display_key: str, ussd_session: dict): +def process_account_balances(user: Account, display_key: str): """ :param user: :type user: :param display_key: :type display_key: - :param ussd_session: - :type ussd_session: :return: :rtype: """ @@ -257,6 +262,10 @@ def process_display_user_metadata(user: Account, display_key: str): contact_data = get_contact_data_from_vcard(vcard=user_metadata.get('vcard')) logg.debug(f'{contact_data}') full_name = f'{contact_data.get("given")} {contact_data.get("family")}' + date_of_birth = user_metadata.get('date_of_birth') + year_of_birth = date_of_birth.get('year') + present_year = datetime.datetime.now().year + age = present_year - year_of_birth gender = user_metadata.get('gender') products = ', '.join(user_metadata.get('products')) location = user_metadata.get('location').get('area_name') @@ -265,6 +274,7 @@ def process_display_user_metadata(user: Account, display_key: str): key=display_key, preferred_language=user.preferred_language, full_name=full_name, + age=age, gender=gender, location=location, products=products @@ -284,20 +294,18 @@ def process_display_user_metadata(user: Account, display_key: str): preferred_language=user.preferred_language, full_name=absent, gender=absent, + age=absent, location=absent, products=absent ) - -def process_account_statement(user: Account, display_key: str, ussd_session: dict): +def process_account_statement(user: Account, display_key: str): """ :param user: :type user: :param display_key: :type display_key: - :param ussd_session: - :type ussd_session: :return: :rtype: """ @@ -399,23 +407,26 @@ def process_start_menu(display_key: str, user: Account): ) -def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession: +def retrieve_most_recent_ussd_session(phone_number: str, session: Session) -> UssdSession: # get last ussd session based on user phone number - last_ussd_session = UssdSession.session\ - .query(UssdSession)\ + session = SessionBase.bind_session(session=session) + last_ussd_session = session.query(UssdSession)\ .filter_by(msisdn=phone_number)\ .order_by(desc(UssdSession.created))\ .first() + SessionBase.release_session(session=session) return last_ussd_session -def process_request(user_input: str, user: Account, ussd_session: Optional[dict] = None) -> Document: +def process_request(account: Account, session, user_input: str, ussd_session: Optional[dict] = None) -> Document: """This function assesses a request based on the user from the request comes, the session_id and the user's input. It determines whether the request translates to a return to an existing session by checking whether the provided session id exists in the database or whether the creation of a new ussd session object is warranted. It then returns the appropriate ussd menu text values. - :param user: The user requesting access to the ussd menu. - :type user: Account + :param account: The account requesting access to the ussd menu. + :type account: Account + :param session: + :type session: :param user_input: The value a user enters in the ussd menu. :type user_input: str :param ussd_session: A JSON serialized in-memory ussd session object @@ -423,22 +434,20 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict] :return: A ussd menu's corresponding text value. :rtype: Document """ - # retrieve metadata before any transition - key = generate_metadata_pointer( - identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), - cic_type=':cic.person' - ) - person_metadata = get_cached_data(key=key) if ussd_session: if user_input == "0": return UssdMenu.parent_menu(menu_name=ussd_session.get('state')) else: - successive_state = next_state(ussd_session=ussd_session, user=user, user_input=user_input) + successive_state = next_state( + account=account, + session=session, + ussd_session=ussd_session, + user_input=user_input) return UssdMenu.find_by_name(name=successive_state) else: - if user.has_valid_pin(): - last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number) + if account.has_valid_pin(): + last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session) if last_ussd_session: # get last state @@ -452,33 +461,35 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict] 'exit_pin_mismatch', 'exit_invalid_request', 'exit_successful_transaction' - ] and person_metadata is not None: + ]: return UssdMenu.find_by_name(name='start') else: return UssdMenu.find_by_name(name=last_state) else: - if user.failed_pin_attempts >= 3 and user.get_account_status() == AccountStatus.LOCKED.name: + if account.failed_pin_attempts >= 3 and account.get_account_status() == AccountStatus.LOCKED.name: return UssdMenu.find_by_name(name='exit_pin_blocked') - elif user.preferred_language is None: + elif account.preferred_language is None: return UssdMenu.find_by_name(name='initial_language_selection') else: return UssdMenu.find_by_name(name='initial_pin_entry') -def next_state(ussd_session: dict, user: Account, user_input: str) -> str: +def next_state(account: Account, session, ussd_session: dict, user_input: str) -> str: """This function navigates the state machine based on the ussd session object and user inputs it receives. It checks the user input and provides the successive state in the state machine. It then updates the session's state attribute with the new state. + :param account: The account requesting access to the ussd menu. + :type account: Account + :param session: + :type session: :param ussd_session: A JSON serialized in-memory ussd session object :type ussd_session: dict - :param user: The user requesting access to the ussd menu. - :type user: Account :param user_input: The value a user enters in the ussd menu. :type user_input: str :return: A string value corresponding the successive give a specific state in the state machine. """ state_machine = UssdStateMachine(ussd_session=ussd_session) - state_machine.scan_data((user_input, ussd_session, user)) + state_machine.scan_data((user_input, ussd_session, account, session)) new_state = state_machine.state return new_state @@ -493,42 +504,63 @@ def process_exit_invalid_menu_option(display_key: str, preferred_language: str): def custom_display_text( + account: Account, display_key: str, menu_name: str, - ussd_session: dict, - user: Account) -> str: + session: Session, + ussd_session: dict) -> str: """This function extracts the appropriate session data based on the current menu name. It then inserts them as keywords in the i18n function. + :param account: The account in a running USSD session. + :type account: Account :param display_key: The path in the translation files defining an appropriate ussd response :type display_key: str :param menu_name: The name by which a specific menu can be identified. :type menu_name: str - :param user: The user in a running USSD session. - :type user: Account + :param session: + :type session: :param ussd_session: A JSON serialized in-memory ussd session object :type ussd_session: dict :return: A string value corresponding the ussd menu's text value. :rtype: str """ if menu_name == 'transaction_pin_authorization': - return process_transaction_pin_authorization(display_key=display_key, user=user, ussd_session=ussd_session) + return process_transaction_pin_authorization( + account=account, + display_key=display_key, + session=session, + ussd_session=ussd_session) elif menu_name == 'exit_insufficient_balance': - return process_exit_insufficient_balance(display_key=display_key, user=user, ussd_session=ussd_session) + return process_exit_insufficient_balance( + account=account, + display_key=display_key, + session=session, + ussd_session=ussd_session) elif menu_name == 'exit_successful_transaction': - return process_exit_successful_transaction(display_key=display_key, user=user, ussd_session=ussd_session) + return process_exit_successful_transaction( + account=account, + display_key=display_key, + session=session, + ussd_session=ussd_session) elif menu_name == 'start': - return process_start_menu(display_key=display_key, user=user) + return process_start_menu(display_key=display_key, user=account) elif 'pin_authorization' in menu_name: - return process_pin_authorization(display_key=display_key, user=user) + return process_pin_authorization( + account=account, + display_key=display_key, + session=session) elif 'enter_current_pin' in menu_name: - return process_pin_authorization(display_key=display_key, user=user) + return process_pin_authorization( + account=account, + display_key=display_key, + session=session) elif menu_name == 'account_balances': - return process_account_balances(display_key=display_key, user=user, ussd_session=ussd_session) + return process_account_balances(display_key=display_key, user=account) elif 'transaction_set' in menu_name: - return process_account_statement(display_key=display_key, user=user, ussd_session=ussd_session) + return process_account_statement(display_key=display_key, user=account) elif menu_name == 'display_user_metadata': - return process_display_user_metadata(display_key=display_key, user=user) + return process_display_user_metadata(display_key=display_key, user=account) elif menu_name == 'exit_invalid_menu_option': - return process_exit_invalid_menu_option(display_key=display_key, preferred_language=user.preferred_language) + return process_exit_invalid_menu_option(display_key=display_key, preferred_language=account.preferred_language) else: - return translation_for(key=display_key, preferred_language=user.preferred_language) + return translation_for(key=display_key, preferred_language=account.preferred_language) diff --git a/apps/cic-ussd/cic_ussd/requests.py b/apps/cic-ussd/cic_ussd/requests.py index 8d436f45..897d8539 100644 --- a/apps/cic-ussd/cic_ussd/requests.py +++ b/apps/cic-ussd/cic_ussd/requests.py @@ -8,9 +8,12 @@ from urllib.parse import urlparse, parse_qs # third-party imports from sqlalchemy import desc +from sqlalchemy.orm.session import Session # local imports -from cic_ussd.db.models.account import AccountStatus, Account +from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase +from cic_ussd.db.enum import AccountStatus from cic_ussd.operations import get_account_status, reset_pin from cic_ussd.validator import check_known_user @@ -72,24 +75,26 @@ def get_account_creation_callback_request_data(env: dict) -> tuple: return status, task_id, result -def process_pin_reset_requests(env: dict, phone_number: str): +def process_pin_reset_requests(env: dict, phone_number: str, session: Session): """This function processes requests that are responsible for the pin reset functionality. It processes GET and PUT requests responsible for returning an account's status and :param env: A dictionary of values representing data sent on the api. :type env: dict :param phone_number: The phone of the user whose pin is being reset. :type phone_number: str + :param session: + :type session: :return: A response denoting the result of the request to reset the user's pin. :rtype: str """ - if not check_known_user(phone=phone_number): + if not check_known_user(phone_number=phone_number, session=session): return f'No user matching {phone_number} was found.', '404 Not Found' if get_request_method(env) == 'PUT': - return reset_pin(phone_number=phone_number), '200 OK' + return reset_pin(phone_number=phone_number, session=session), '200 OK' if get_request_method(env) == 'GET': - status = get_account_status(phone_number=phone_number) + status = get_account_status(phone_number=phone_number, session=session) response = { 'status': f'{status}' } @@ -97,16 +102,18 @@ def process_pin_reset_requests(env: dict, phone_number: str): return response, '200 OK' -def process_locked_accounts_requests(env: dict) -> tuple: +def process_locked_accounts_requests(env: dict, session: Session) -> tuple: """This function authenticates staff requests and returns a serialized JSON formatted list of blockchain addresses of accounts for which the PIN has been locked due to too many failed attempts. :param env: A dictionary of values representing data sent on the api. :type env: dict + :param session: + :type session: :return: A tuple containing a serialized list of blockchain addresses for locked accounts and corresponding message for the response. :rtype: tuple """ - logg.debug('Authentication requires integration with cic-auth') + session = SessionBase.bind_session(session=session) response = '' if get_request_method(env) == 'GET': @@ -123,12 +130,14 @@ def process_locked_accounts_requests(env: dict) -> tuple: else: limit = r[1] - locked_accounts = Account.session.query(Account.blockchain_address).filter( + locked_accounts = session.query(Account.blockchain_address).filter( Account.account_status == AccountStatus.LOCKED.value, Account.failed_pin_attempts >= 3).order_by(desc(Account.updated)).offset(offset).limit(limit).all() # convert lists to scalar blockchain addresses locked_accounts = [blockchain_address for (blockchain_address, ) in locked_accounts] + + SessionBase.release_session(session=session) response = json.dumps(locked_accounts) return response, '200 OK' return response, '405 Play by the rules' diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py index 53f633f6..c8df1ccd 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py @@ -36,11 +36,8 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config)) # set up db data_source_name = dsn_from_config(config) SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG')) -# create session for the life time of http request -SessionBase.session = SessionBase.create_session() -# handle requests from CICADA def application(env, start_response): """Loads python code for application to be accessible over web server :param env: Object containing server and request information @@ -55,19 +52,24 @@ def application(env, start_response): errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')] headers = [('Content-Type', 'text/plain')] + # create session for the life time of http request + session = SessionBase.create_session() + if get_request_endpoint(env) == '/pin': phone_number = get_query_parameters(env=env, query_name='phoneNumber') phone_number = quote_plus(phone_number) - response, message = process_pin_reset_requests(env=env, phone_number=phone_number) + response, message = process_pin_reset_requests(env=env, phone_number=phone_number, session=session) response_bytes, headers = define_response_with_content(headers=errors_headers, response=response) - SessionBase.session.close() + session.commit() + session.close() start_response(message, headers) return [response_bytes] # handle requests for locked accounts - response, message = process_locked_accounts_requests(env=env) + response, message = process_locked_accounts_requests(env=env, session=session) response_bytes, headers = define_response_with_content(headers=headers, response=response) start_response(message, headers) - SessionBase.session.close() + session.commit() + session.close() return [response_bytes] diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py index 6aad181f..984fb680 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py @@ -26,7 +26,7 @@ from cic_ussd.metadata.base import Metadata from cic_ussd.operations import (define_response_with_content, process_menu_interaction_requests, define_multilingual_responses) -from cic_ussd.phone_number import process_phone_number, Support +from cic_ussd.phone_number import process_phone_number, Support, E164Format from cic_ussd.processor import get_default_token_data from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore from cic_ussd.requests import (get_request_endpoint, @@ -55,8 +55,6 @@ data_source_name = dsn_from_config(config) SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG')) -# create session for the life time of http request -SessionBase.session = SessionBase.create_session() # set up translations i18n.load_path.append(config.get('APP_LOCALE_PATH')) @@ -126,6 +124,7 @@ else: valid_service_codes = config.get('APP_SERVICE_CODE').split(",") +E164Format.region = config.get('PHONE_NUMBER_REGION') Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER') @@ -142,10 +141,13 @@ def application(env, start_response): errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')] headers = [('Content-Type', 'text/plain')] + # create session for the life time of http request + session = SessionBase.create_session() + if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/': if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded': - start_response('405 Play by the rules', errors_headers) + start_response('405 Urlencoded, please', errors_headers) return [] post_data = env.get('wsgi.input').read() @@ -168,7 +170,7 @@ def application(env, start_response): # add validation for phone number if phone_number: - phone_number = process_phone_number(phone_number=phone_number, region=config.get('PHONE_NUMBER_REGION')) + phone_number = process_phone_number(phone_number=phone_number, region=E164Format.region) # validate ip address if not check_ip(config=config, env=env): @@ -205,14 +207,20 @@ def application(env, start_response): phone_number=phone_number, queue=args.q, service_code=service_code, + session=session, user_input=user_input) response_bytes, headers = define_response_with_content(headers=headers, response=response) start_response('200 OK,', headers) - SessionBase.session.close() + session.commit() + session.close() return [response_bytes] else: + logg.error('invalid query {}'.format(env)) + for r in env: + logg.debug('{}: {}'.format(r, env)) + session.close() start_response('405 Play by the rules', errors_headers) return [] diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/balance.py b/apps/cic-ussd/cic_ussd/state_machine/logic/balance.py index 899ff346..64f1dd61 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/balance.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/balance.py @@ -3,6 +3,7 @@ import logging from typing import Tuple # third-party imports +from sqlalchemy.orm.session import Session # local imports from cic_ussd.db.models.account import Account @@ -10,11 +11,11 @@ from cic_ussd.db.models.account import Account logg = logging.getLogger(__file__) -def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account]): +def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account, Session]): """This function compiles a brief statement of a user's last three inbound and outbound transactions and send the same as a message on their selected avenue for notification. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data logg.debug('This section requires integration with cic-eth. (The last 6 transactions would be sent as an sms.)') diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py b/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py index c7239ab5..bac773b9 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py @@ -16,7 +16,7 @@ def menu_one_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '1' :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user_input == '1' @@ -27,7 +27,7 @@ def menu_two_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '2' :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user_input == '2' @@ -38,7 +38,7 @@ def menu_three_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '3' :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user_input == '3' @@ -50,7 +50,7 @@ def menu_four_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '4' :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user_input == '4' @@ -62,10 +62,22 @@ def menu_five_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '5' :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user_input == '5' +def menu_six_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: + """ + This function checks that user input matches a string with value '6' + :param state_machine_data: A tuple containing user input, a ussd session and user object. + :type state_machine_data: tuple + :return: A user input's match with '6' + :rtype: bool + """ + user_input, ussd_session, user, session = state_machine_data + return user_input == '6' + + def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: """ This function checks that user input matches a string with value '00' @@ -74,7 +86,7 @@ def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bo :return: A user input's match with '00' :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user_input == '00' @@ -86,5 +98,5 @@ def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account]) -> :return: A user input's match with '99' :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user_input == '99' diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py b/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py index 5f51e602..9419d248 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py @@ -9,11 +9,13 @@ import re from typing import Tuple # third party imports -import bcrypt +from sqlalchemy.orm.session import Session # local imports -from cic_ussd.db.models.account import AccountStatus, Account -from cic_ussd.encoder import PasswordEncoder, create_password_hash, check_password_hash +from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase +from cic_ussd.db.enum import AccountStatus +from cic_ussd.encoder import create_password_hash, check_password_hash from cic_ussd.operations import persist_session_to_db_task, create_or_update_session from cic_ussd.redis import InMemoryStore @@ -21,7 +23,7 @@ from cic_ussd.redis import InMemoryStore logg = logging.getLogger(__file__) -def is_valid_pin(state_machine_data: Tuple[str, dict, Account]) -> bool: +def is_valid_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks a pin's validity by ensuring it has a length of for characters and the characters are numeric. :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -29,7 +31,7 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A pin's validity :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data pin_is_valid = False matcher = r'^\d{4}$' if re.match(matcher, user_input): @@ -37,34 +39,34 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, Account]) -> bool: return pin_is_valid -def is_authorized_pin(state_machine_data: Tuple[str, dict, Account]) -> bool: +def is_authorized_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks whether the user input confirming a specific pin matches the initial pin entered. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user.verify_password(password=user_input) -def is_locked_account(state_machine_data: Tuple[str, dict, Account]) -> bool: +def is_locked_account(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks whether a user's account is locked due to too many failed attempts. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user.get_account_status() == AccountStatus.LOCKED.name -def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Account]): +def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): """This function hashes a pin and stores it in session data. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data # define redis cache entry point cache = InMemoryStore.cache @@ -93,54 +95,56 @@ def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Accoun service_code=in_redis_ussd_session.get('service_code'), user_input=user_input, current_menu=in_redis_ussd_session.get('state'), + session=session, session_data=session_data ) persist_session_to_db_task(external_session_id=external_session_id, queue='cic-ussd') -def pins_match(state_machine_data: Tuple[str, dict, Account]) -> bool: +def pins_match(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks whether the user input confirming a specific pin matches the initial pin entered. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data initial_pin = ussd_session.get('session_data').get('initial_pin') - logg.debug(f'USSD SESSION: {ussd_session}') return check_password_hash(user_input, initial_pin) -def complete_pin_change(state_machine_data: Tuple[str, dict, Account]): +def complete_pin_change(state_machine_data: Tuple[str, dict, Account, Session]): """This function persists the user's pin to the database :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data + session = SessionBase.bind_session(session=session) password_hash = ussd_session.get('session_data').get('initial_pin') user.password_hash = password_hash - Account.session.add(user) - Account.session.commit() + session.add(user) + session.flush() + SessionBase.release_session(session=session) -def is_blocked_pin(state_machine_data: Tuple[str, dict, Account]) -> bool: +def is_blocked_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks whether the user input confirming a specific pin matches the initial pin entered. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data return user.get_account_status() == AccountStatus.LOCKED.name -def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account]) -> bool: +def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks whether the user's new pin is a valid pin and that it isn't the same as the old one. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data is_old_pin = user.verify_password(password=user_input) return is_valid_pin(state_machine_data=state_machine_data) and not is_old_pin diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py b/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py index 517c09e9..6f33a555 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py @@ -9,15 +9,15 @@ logg = logging.getLogger() def send_terms_to_user_if_required(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data logg.debug('Requires integration to cic-notify.') def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data logg.debug('Requires integration to cic-notify.') def upsell_unregistered_recipient(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data logg.debug('Requires integration to cic-notify.') \ No newline at end of file diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py b/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py index 84c866ee..c2ac5e45 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py @@ -5,13 +5,16 @@ from typing import Tuple # third party imports import celery +from sqlalchemy.orm.session import Session # local imports -from cic_ussd.balance import BalanceManager, compute_operational_balance +from cic_ussd.balance import compute_operational_balance from cic_ussd.chain import Chain -from cic_ussd.db.models.account import AccountStatus, Account +from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase +from cic_ussd.db.enum import AccountStatus from cic_ussd.operations import save_to_in_memory_ussd_session_data -from cic_ussd.phone_number import get_user_by_phone_number, process_phone_number +from cic_ussd.phone_number import process_phone_number, E164Format from cic_ussd.processor import retrieve_token_symbol from cic_ussd.redis import create_cached_data_key, get_cached_data from cic_ussd.transactions import OutgoingTransactionProcessor @@ -20,7 +23,7 @@ from cic_ussd.transactions import OutgoingTransactionProcessor logg = logging.getLogger(__file__) -def is_valid_recipient(state_machine_data: Tuple[str, dict, Account]) -> bool: +def is_valid_recipient(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks that a user exists, is not the initiator of the transaction, has an active account status and is authorized to perform standard transactions. :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -28,14 +31,17 @@ def is_valid_recipient(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user's validity :rtype: bool """ - user_input, ussd_session, user = state_machine_data - recipient = get_user_by_phone_number(phone_number=user_input) - is_not_initiator = process_phone_number(user_input, 'KE') != user.phone_number + user_input, ussd_session, user, session = state_machine_data + phone_number = process_phone_number(user_input, E164Format.region) + session = SessionBase.bind_session(session=session) + recipient = Account.get_by_phone_number(phone_number=phone_number, session=session) + SessionBase.release_session(session=session) + is_not_initiator = phone_number != user.phone_number has_active_account_status = user.get_account_status() == AccountStatus.ACTIVE.name return is_not_initiator and has_active_account_status and recipient is not None -def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account]) -> bool: +def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks that the transaction amount provided is valid as per the criteria for the transaction being attempted. :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -43,14 +49,14 @@ def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account]) - :return: A transaction amount's validity :rtype: bool """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data try: return int(user_input) > 0 except ValueError: return False -def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> bool: +def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks that the transaction amount provided is valid as per the criteria for the transaction being attempted. :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -58,10 +64,7 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> boo :return: An account balance's validity :rtype: bool """ - user_input, ussd_session, user = state_machine_data - balance_manager = BalanceManager(address=user.blockchain_address, - chain_str=Chain.spec.__str__(), - token_symbol='SRF') + user_input, ussd_session, user, session = state_machine_data # get cached balance key = create_cached_data_key( identifier=bytes.fromhex(user.blockchain_address[2:]), @@ -73,30 +76,37 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> boo return int(user_input) <= operational_balance -def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Account]): +def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): """This function saves the phone number corresponding the intended recipients blockchain account. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data session_data = ussd_session.get('session_data') or {} - session_data['recipient_phone_number'] = user_input + recipient_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region) + session_data['recipient_phone_number'] = recipient_phone_number - save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session) + save_to_in_memory_ussd_session_data( + queue='cic-ussd', + session=session, + session_data=session_data, + ussd_session=ussd_session) -def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account]): +def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account, Session]): """ :param state_machine_data: :type state_machine_data: :return: :rtype: """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data - recipient = get_user_by_phone_number(phone_number=user_input) + recipient_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region) + recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) blockchain_address = recipient.blockchain_address + # retrieve and cache account's metadata s_query_person_metadata = celery.signature( 'cic_ussd.tasks.metadata.query_person_metadata', @@ -105,32 +115,36 @@ def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account]): s_query_person_metadata.apply_async(queue='cic-ussd') -def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, Account]): +def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): """This function saves the phone number corresponding the intended recipients blockchain account. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data session_data = ussd_session.get('session_data') or {} session_data['transaction_amount'] = user_input - save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session) + save_to_in_memory_ussd_session_data( + queue='cic-ussd', + session=session, + session_data=session_data, + ussd_session=ussd_session) -def process_transaction_request(state_machine_data: Tuple[str, dict, Account]): +def process_transaction_request(state_machine_data: Tuple[str, dict, Account, Session]): """This function saves the phone number corresponding the intended recipients blockchain account. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data # retrieve token symbol chain_str = Chain.spec.__str__() # get user from phone number recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') - recipient = get_user_by_phone_number(phone_number=recipient_phone_number) + recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) to_address = recipient.blockchain_address from_address = user.blockchain_address amount = int(ussd_session.get('session_data').get('transaction_amount')) diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/user.py b/apps/cic-ussd/cic_ussd/state_machine/logic/user.py index d2db5d29..7c01f971 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/user.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/user.py @@ -5,12 +5,14 @@ from typing import Tuple # third-party imports import celery -from cic_types.models.person import Person, generate_metadata_pointer +from cic_types.models.person import generate_metadata_pointer from cic_types.models.person import generate_vcard_from_contact_data, manage_identity_data +from sqlalchemy.orm.session import Session # local imports from cic_ussd.chain import Chain from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase from cic_ussd.error import MetadataNotFoundError from cic_ussd.metadata import blockchain_address_to_metadata_pointer from cic_ussd.operations import save_to_in_memory_ussd_session_data @@ -19,37 +21,63 @@ from cic_ussd.redis import get_cached_data logg = logging.getLogger(__file__) -def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account]): +def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account, Session]): """This function changes the user's preferred language to english. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data + session = SessionBase.bind_session(session=session) user.preferred_language = 'en' - Account.session.add(user) - Account.session.commit() + session.add(user) + session.flush() + SessionBase.release_session(session=session) + + preferences_data = { + 'preferred_language': 'en' + } + + s = celery.signature( + 'cic_ussd.tasks.metadata.add_preferences_metadata', + [user.blockchain_address, preferences_data] + ) + s.apply_async(queue='cic-ussd') -def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account]): +def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account, Session]): """This function changes the user's preferred language to swahili. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple """ - user_input, ussd_session, user = state_machine_data - user.preferred_language = 'sw' - Account.session.add(user) - Account.session.commit() + user_input, ussd_session, account, session = state_machine_data + session = SessionBase.bind_session(session=session) + account.preferred_language = 'sw' + session.add(account) + session.flush() + SessionBase.release_session(session=session) + + preferences_data = { + 'preferred_language': 'sw' + } + + s = celery.signature( + 'cic_ussd.tasks.metadata.add_preferences_metadata', + [account.blockchain_address, preferences_data] + ) + s.apply_async(queue='cic-ussd') -def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account]): +def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account, Session]): """This function sets user's account to active. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple """ - user_input, ussd_session, user = state_machine_data - user.activate_account() - Account.session.add(user) - Account.session.commit() + user_input, ussd_session, account, session = state_machine_data + session = SessionBase.bind_session(session=session) + account.activate_account() + session.add(account) + session.flush() + SessionBase.release_session(session=session) def process_gender_user_input(user: Account, user_input: str): @@ -61,6 +89,7 @@ def process_gender_user_input(user: Account, user_input: str): :return: :rtype: """ + gender = "" if user.preferred_language == 'en': if user_input == '1': gender = 'Male' @@ -78,13 +107,13 @@ def process_gender_user_input(user: Account, user_input: str): return gender -def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, Account]): +def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): """This function saves first name data to the ussd session in the redis cache. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple """ - user_input, ussd_session, user = state_machine_data - + user_input, ussd_session, user, session = state_machine_data + session = SessionBase.bind_session(session=session) # get current menu current_state = ussd_session.get('state') @@ -93,6 +122,9 @@ def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, if 'given_name' in current_state: key = 'given_name' + if 'date_of_birth' in current_state: + key = 'date_of_birth' + if 'family_name' in current_state: key = 'family_name' @@ -114,7 +146,11 @@ def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, session_data = { key: user_input } - save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session) + save_to_in_memory_ussd_session_data( + queue='cic-ussd', + session=session, + session_data=session_data, + ussd_session=ussd_session) def format_user_metadata(metadata: dict, user: Account): @@ -130,6 +166,13 @@ def format_user_metadata(metadata: dict, user: Account): given_name = metadata.get('given_name') family_name = metadata.get('family_name') + if isinstance(metadata.get('date_of_birth'), dict): + date_of_birth = metadata.get('date_of_birth') + else: + date_of_birth = { + "year": int(metadata.get('date_of_birth')[:4]) + } + # check whether there's existing location data if isinstance(metadata.get('location'), dict): location = metadata.get('location') @@ -154,6 +197,7 @@ def format_user_metadata(metadata: dict, user: Account): ) return { "date_registered": date_registered, + "date_of_birth": date_of_birth, "gender": gender, "identities": identities, "location": location, @@ -166,12 +210,12 @@ def format_user_metadata(metadata: dict, user: Account): } -def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account]): +def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account, Session]): """This function persists elements of the user metadata stored in session data :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data # get session data metadata = ussd_session.get('session_data') @@ -187,8 +231,8 @@ def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account]): s_create_person_metadata.apply_async(queue='cic-ussd') -def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user = state_machine_data +def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account, Session]): + user_input, ussd_session, user, session = state_machine_data blockchain_address = user.blockchain_address key = generate_metadata_pointer( identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), @@ -201,12 +245,12 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]): given_name = ussd_session.get('session_data').get('given_name') family_name = ussd_session.get('session_data').get('family_name') + date_of_birth = ussd_session.get('session_data').get('date_of_birth') gender = ussd_session.get('session_data').get('gender') location = ussd_session.get('session_data').get('location') products = ussd_session.get('session_data').get('products') # validate user metadata - person = Person() user_metadata = json.loads(user_metadata) # edit specific metadata attribute @@ -214,6 +258,11 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]): user_metadata['given_name'] = given_name if family_name: user_metadata['family_name'] = family_name + if date_of_birth and len(date_of_birth) == 4: + year = int(date_of_birth[:4]) + user_metadata['date_of_birth'] = { + 'year': year + } if gender: user_metadata['gender'] = gender if location: @@ -233,8 +282,8 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]): s_edit_person_metadata.apply_async(queue='cic-ussd') -def get_user_metadata(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user = state_machine_data +def get_user_metadata(state_machine_data: Tuple[str, dict, Account, Session]): + user_input, ussd_session, user, session = state_machine_data blockchain_address = user.blockchain_address s_get_user_metadata = celery.signature( 'cic_ussd.tasks.metadata.query_person_metadata', diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py b/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py index 2f6203d4..80747042 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py @@ -19,7 +19,7 @@ def has_cached_user_metadata(state_machine_data: Tuple[str, dict, Account]): :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data # check for user metadata in cache key = generate_metadata_pointer( identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), @@ -34,7 +34,7 @@ def is_valid_name(state_machine_data: Tuple[str, dict, Account]): :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data name_matcher = "^[a-zA-Z]+$" valid_name = re.match(name_matcher, user_input) if valid_name: @@ -50,9 +50,21 @@ def is_valid_gender_selection(state_machine_data: Tuple[str, dict, Account]): :return: :rtype: """ - user_input, ussd_session, user = state_machine_data + user_input, ussd_session, user, session = state_machine_data selection_matcher = "^[1-2]$" if re.match(selection_matcher, user_input): return True else: return False + + +def is_valid_date(state_machine_data: Tuple[str, dict, Account]): + """ + :param state_machine_data: + :type state_machine_data: + :return: + :rtype: + """ + user_input, ussd_session, user, session = state_machine_data + # For MVP this value is defaulting to year + return len(user_input) == 4 and int(user_input) >= 1900 diff --git a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py index ab860b94..aaf70745 100644 --- a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py +++ b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py @@ -53,13 +53,25 @@ def process_account_creation_callback(self, result: str, url: str, status_code: session.add(user) session.commit() session.close() - + queue = self.request.delivery_info.get('routing_key') - s = celery.signature( + + # add phone number metadata lookup + s_phone_pointer = celery.signature( 'cic_ussd.tasks.metadata.add_phone_pointer', [result, phone_number] ) - s.apply_async(queue=queue) + s_phone_pointer.apply_async(queue=queue) + + # add custom metadata tags + custom_metadata = { + "tags": ["ussd", "individual"] + } + s_custom_metadata = celery.signature( + 'cic_ussd.tasks.metadata.add_custom_metadata', + [result, custom_metadata] + ) + s_custom_metadata.apply_async(queue=queue) # expire cache cache.expire(task_id, timedelta(seconds=180)) diff --git a/apps/cic-ussd/cic_ussd/tasks/metadata.py b/apps/cic-ussd/cic_ussd/tasks/metadata.py index fa6caf86..37703a90 100644 --- a/apps/cic-ussd/cic_ussd/tasks/metadata.py +++ b/apps/cic-ussd/cic_ussd/tasks/metadata.py @@ -7,8 +7,10 @@ from hexathon import strip_0x # local imports from cic_ussd.metadata import blockchain_address_to_metadata_pointer +from cic_ussd.metadata.custom import CustomMetadata from cic_ussd.metadata.person import PersonMetadata from cic_ussd.metadata.phone import PhonePointerMetadata +from cic_ussd.metadata.preferences import PreferencesMetadata from cic_ussd.tasks.base import CriticalMetadataTask celery_app = celery.current_app @@ -44,7 +46,7 @@ def create_person_metadata(blockchain_address: str, data: dict): @celery_app.task -def edit_person_metadata(blockchain_address: str, data: bytes): +def edit_person_metadata(blockchain_address: str, data: dict): identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client.edit(data=data) @@ -56,3 +58,17 @@ def add_phone_pointer(self, blockchain_address: str, phone_number: str): stripped_address = strip_0x(blockchain_address) phone_metadata_client = PhonePointerMetadata(identifier=identifier) phone_metadata_client.create(data=stripped_address) + + +@celery_app.task() +def add_custom_metadata(blockchain_address: str, data: dict): + identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) + custom_metadata_client = CustomMetadata(identifier=identifier) + custom_metadata_client.create(data=data) + + +@celery_app.task() +def add_preferences_metadata(blockchain_address: str, data: dict): + identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) + custom_metadata_client = PreferencesMetadata(identifier=identifier) + custom_metadata_client.create(data=data) diff --git a/apps/cic-ussd/cic_ussd/validator.py b/apps/cic-ussd/cic_ussd/validator.py index 660f2783..a1706f9a 100644 --- a/apps/cic-ussd/cic_ussd/validator.py +++ b/apps/cic-ussd/cic_ussd/validator.py @@ -9,6 +9,7 @@ from confini import Config # local imports from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase logg = logging.getLogger(__file__) @@ -45,29 +46,21 @@ def check_request_content_length(config: Config, env: dict): config.get('APP_MAX_BODY_LENGTH')) -def check_known_user(phone: str): - """ - This method attempts to ascertain whether the user already exists and is known to the system. +def check_known_user(phone_number: str, session): + """This method attempts to ascertain whether the user already exists and is known to the system. It sends a get request to the platform application and attempts to retrieve the user's data which it persists in memory. - :param phone: A valid phone number - :type phone: str + :param phone_number: A valid phone number + :type phone_number: str + :param session: + :type session: :return: Is known phone number :rtype: boolean """ - user = Account.session.query(Account).filter_by(phone_number=phone).first() - return user is not None - - -def check_phone_number(number: str): - """ - Checks whether phone number is present - :param number: A valid phone number - :type number: str - :return: Phone number presence - :rtype: boolean - """ - return number is not None + session = SessionBase.bind_session(session=session) + account = session.query(Account).filter_by(phone_number=phone_number).first() + SessionBase.release_session(session=session) + return account is not None def check_request_method(env: dict): diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index ddaa7abf..f527c4ef 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,4 +1,4 @@ -cic_base[full_graph]~=0.1.2b15 -cic-eth~=0.11.0b16 -cic-notify~=0.4.0a5 -cic-types~=0.1.0a10 +cic_base[full_graph]==0.1.3a3+build.4aa03607 +cic-eth~=0.11.1a2 +cic-notify~=0.4.0a6 +cic-types~=0.1.0a11 diff --git a/apps/cic-ussd/states/account_management_states.json b/apps/cic-ussd/states/account_management_states.json index 456edd92..ea9027a2 100644 --- a/apps/cic-ussd/states/account_management_states.json +++ b/apps/cic-ussd/states/account_management_states.json @@ -7,6 +7,7 @@ "new_pin_confirmation", "display_user_metadata", "name_edit_pin_authorization", + "dob_edit_pin_authorization", "gender_edit_pin_authorization", "location_edit_pin_authorization", "products_edit_pin_authorization", diff --git a/apps/cic-ussd/states/user_metadata_states.json b/apps/cic-ussd/states/user_metadata_states.json index 59bc1fd7..21ef82df 100644 --- a/apps/cic-ussd/states/user_metadata_states.json +++ b/apps/cic-ussd/states/user_metadata_states.json @@ -5,5 +5,6 @@ "enter_age", "enter_location", "enter_products", + "enter_date_of_birth", "display_metadata_pin_authorization" ] \ No newline at end of file diff --git a/apps/cic-ussd/test_requirements.txt b/apps/cic-ussd/test_requirements.txt index 76aa808a..59af6564 100644 --- a/apps/cic-ussd/test_requirements.txt +++ b/apps/cic-ussd/test_requirements.txt @@ -8,4 +8,4 @@ pytest-mock==3.3.1 pytest-ordering==0.6 pytest-redis==2.0.0 requests-mock==1.8.0 -tavern==1.14.2 \ No newline at end of file +tavern==1.14.2 diff --git a/apps/cic-ussd/tests/fixtures/integration.py b/apps/cic-ussd/tests/fixtures/integration.py index b4488dcc..c8c349d9 100644 --- a/apps/cic-ussd/tests/fixtures/integration.py +++ b/apps/cic-ussd/tests/fixtures/integration.py @@ -124,46 +124,6 @@ def second_profile_management_session_id() -> str: return session_id() -@pytest.fixture(scope='session') -def first_account_change_given_name() -> str: - return fake.first_name() - - -@pytest.fixture(scope='session') -def second_account_change_given_name() -> str: - return fake.first_name() - - -@pytest.fixture(scope='session') -def first_account_change_family_name() -> str: - return fake.last_name() - - -@pytest.fixture(scope='session') -def second_account_change_family_name() -> str: - return fake.last_name() - - -@pytest.fixture(scope='session') -def first_account_change_location() -> str: - return fake.city() - - -@pytest.fixture(scope='session') -def second_account_change_location() -> str: - return fake.city() - - -@pytest.fixture(scope='session') -def first_account_change_product() -> str: - return fake.color_name() - - -@pytest.fixture(scope='session') -def second_account_change_product() -> str: - return fake.color_name() - - @pytest.fixture(scope='session') def first_profile_management_session_id_1() -> str: return session_id() diff --git a/apps/cic-ussd/tests/integration/README.md b/apps/cic-ussd/tests/integration/README.md new file mode 100644 index 00000000..dd567c53 --- /dev/null +++ b/apps/cic-ussd/tests/integration/README.md @@ -0,0 +1,25 @@ +# INTEGRATION TESTING + +This folder contains integration tests. + +## OVERVIEW + +There are four files defining the integration tests. + +* **test_account_creation**: Tests account sign up process. +* **test_transactions**: Tests transactions between two accounts. +* **test_profile_management**: Tests that account metadata can be edited. +* **test_account_management**: Tests that account management functionalities are intact. + +## REQUIREMENTS + +In order to run the transaction tests, please ensure that the faucet amount is set to a non-zero value, ideally `50000000` +which is the value set in the config file `.config/test/integration.ini`. + +This implies setting the `DEV_FAUCET_AMOUNT` to a non-zero value before bringing up the contract-migration image: + +```shell +export DEV_FAUCET_AMOUNT=50000000 +RUN_MASK=1 docker-compose up contract-migration +RUN_MASK=2 docker-compose up contract-migration +``` diff --git a/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml b/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml index 4fa83dd5..d602583c 100644 --- a/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml +++ b/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml @@ -214,12 +214,13 @@ stages: status_code: - 200 headers: - Content-Length: '28' + Content-Length: '51' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Enter first name\n0. Back" + expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help" + delay_before: 10 - name: Pin number confirmation [{second_account_pin_number} - second account] request: @@ -232,227 +233,6 @@ stages: headers: content-type: "application/x-www-form-urlencoded" method: POST - response: - status_code: - - 200 - headers: - Content-Length: '37' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka jina lako la kwanza\n0. Nyuma" - - - name: Enter first name [first_account_given_name - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_metadata_entry_session_id}" - phoneNumber: "{first_account_phone_number}" - text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '29' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Enter family name\n0. Back" - - - name: Enter first name [second_account_given_name - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_metadata_entry_session_id}" - phoneNumber: "{second_account_phone_number}" - text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '37' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka jina lako la mwisho\n0. Nyuma" - - - name: Enter last name [first_account_family_name - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_metadata_entry_session_id}" - phoneNumber: "{first_account_phone_number}" - text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '51' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Enter gender\n1. Male\n2. Female\n3. Other\n0. Back" - - - name: Enter last name [second_account_family_name - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_metadata_entry_session_id}" - phoneNumber: "{second_account_phone_number}" - text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '64' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n3. Nyngine\n0. Nyuma" - - - name: Select gender [Male - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_metadata_entry_session_id}" - phoneNumber: "{first_account_phone_number}" - text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '31' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Enter your location\n0. Back" - - - name: Select gender [Female - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_metadata_entry_session_id}" - phoneNumber: "{second_account_phone_number}" - text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '27' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka eneo lako\n0. Nyuma" - - - name: Enter location [first_account_location - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_metadata_entry_session_id}" - phoneNumber: "{first_account_phone_number}" - text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '55' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Please enter a product or service you offer\n0. Back" - - - name: Enter location [second_account_location - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_metadata_entry_session_id}" - phoneNumber: "{second_account_phone_number}" - text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '42' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka bidhaa ama huduma unauza\n0. Nyuma" - - - name: Enter product [first_account_product - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_metadata_entry_session_id}" - phoneNumber: "{first_account_phone_number}" - text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}*{first_account_product}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '51' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help" - delay_before: 10 - - - name: Enter product [second_account_product - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_metadata_entry_session_id}" - phoneNumber: "{second_account_phone_number}" - text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}*{second_account_product}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST response: status_code: - 200 diff --git a/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml b/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml index 03a5a4df..1d9ea975 100644 --- a/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml +++ b/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml @@ -31,7 +31,6 @@ stages: status_code: - 200 headers: - Content-Length: '51' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response diff --git a/apps/cic-ussd/tests/integration/test_profile_management.tavern.yaml b/apps/cic-ussd/tests/integration/test_profile_management.tavern.yaml index 274f0265..7e19a772 100644 --- a/apps/cic-ussd/tests/integration/test_profile_management.tavern.yaml +++ b/apps/cic-ussd/tests/integration/test_profile_management.tavern.yaml @@ -17,22 +17,6 @@ marks: - second_account_product - first_profile_management_session_id - second_profile_management_session_id - - first_account_change_family_name - - second_account_change_family_name - - first_account_change_given_name - - second_account_change_given_name - - first_account_change_location - - second_account_change_location - - first_account_change_product - - second_account_change_product - - first_profile_management_session_id_1 - - second_profile_management_session_id_1 - - first_profile_management_session_id_2 - - second_profile_management_session_id_2 - - first_profile_management_session_id_3 - - second_profile_management_session_id_3 - - first_profile_management_session_id_4 - - second_profile_management_session_id_4 - third stages: @@ -139,12 +123,12 @@ stages: status_code: - 200 headers: - Content-Length: '103' + Content-Length: '115' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit location\n4. Edit products\n5. View my profile\n0. Back" + expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit age\n4. Edit location\n5. Edit products\n6. View my profile\n0. Back" - name: Profile management menu [second account] request: @@ -161,12 +145,12 @@ stages: status_code: - 200 headers: - Content-Length: '104' + Content-Length: '117' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka eneo\n4. Weka bidhaa\n5. Angalia wasifu wako\n0. Nyuma" + expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka umri\n4. Weka eneo\n5. Weka bidhaa\n6. Angalia wasifu wako\n0. Nyuma" - name: Enter pin to view profile [first account] request: @@ -175,7 +159,7 @@ stages: serviceCode: "*483*46#" sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5" + text: "2*1*6" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -197,7 +181,7 @@ stages: serviceCode: "*483*46#" sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5" + text: "2*1*6" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -219,7 +203,7 @@ stages: serviceCode: "*483*46#" sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{first_account_pin_number}" + text: "2*1*6*{first_account_pin_number}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -231,7 +215,7 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Your details are:\n Name: {first_account_given_name} {first_account_family_name}\n Gender: Male\n Location: {first_account_location}\n You sell: {first_account_product}\n0. Back" + expected_response: "CON Your details are:\n Name: Not provided\n Gender: Not provided\n Age: Not provided\n Location: Not provided\n You sell: Not provided\n0. Back" - name: Display profile [second account] request: @@ -240,7 +224,7 @@ stages: serviceCode: "*483*46#" sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}" + text: "2*1*6*{second_account_pin_number}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -252,7 +236,7 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Wasifu wako una maelezo yafuatayo:\n Jina: {second_account_given_name} {second_account_family_name}\n Jinsia: Mwanamke\n Eneo: {second_account_location}\n Unauza: {second_account_product}\n0. Nyuma" + expected_response: "CON Wasifu wako una maelezo yafuatayo:\n Jina: Haijawekwa\n Jinsia: Haijawekwa\n Umri: Haijawekwa\n Eneo: Haijawekwa\n Unauza: Haijawekwa\n0. Nyuma" - name: Second profile management menu [first account] request: @@ -261,7 +245,7 @@ stages: serviceCode: "*483*46#" sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{first_account_pin_number}*0" + text: "2*1*6*{first_account_pin_number}*0" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -269,12 +253,12 @@ stages: status_code: - 200 headers: - Content-Length: '103' + Content-Length: '115' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit location\n4. Edit products\n5. View my profile\n0. Back" + expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit age\n4. Edit location\n5. Edit products\n6. View my profile\n0. Back" - name: Second profile management menu [second account] request: @@ -283,7 +267,7 @@ stages: serviceCode: "*483*46#" sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0" + text: "2*1*6*{second_account_pin_number}*0" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -291,12 +275,12 @@ stages: status_code: - 200 headers: - Content-Length: '104' + Content-Length: '117' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka eneo\n4. Weka bidhaa\n5. Angalia wasifu wako\n0. Nyuma" + expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka umri\n4. Weka eneo\n5. Weka bidhaa\n6. Angalia wasifu wako\n0. Nyuma" - name: Edit name [first account] request: @@ -305,7 +289,7 @@ stages: serviceCode: "*483*46#" sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1" + text: "2*1*6*{first_account_pin_number}*0*1" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -327,7 +311,7 @@ stages: serviceCode: "*483*46#" sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1" + text: "2*1*6*{second_account_pin_number}*0*1" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -349,7 +333,7 @@ stages: serviceCode: "*483*46#" sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1*{first_account_change_given_name}" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -371,7 +355,7 @@ stages: serviceCode: "*483*46#" sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1*{second_account_change_given_name}" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -393,7 +377,7 @@ stages: serviceCode: "*483*46#" sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1*{first_account_change_given_name}*{first_account_change_family_name}" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -401,12 +385,12 @@ stages: status_code: - 200 headers: - Content-Length: '33' + Content-Length: '51' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Please enter your PIN\n0. Back" + expected_response: "CON Enter gender\n1. Male\n2. Female\n3. Other\n0. Back" - name: Enter family name [second account] request: @@ -415,7 +399,7 @@ stages: serviceCode: "*483*46#" sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1*{second_account_change_given_name}*{second_account_change_family_name}" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -423,21 +407,21 @@ stages: status_code: - 200 headers: - Content-Length: '36' + Content-Length: '64' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Tafadhali weka PIN yako\n0. Nyuma" + expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n3. Nyngine\n0. Nyuma" - - name: Enter name change pin [first account] + - name: Select gender [Male - first account] request: url: "{server_url}" data: serviceCode: "*483*46#" sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1*{first_account_change_given_name}*{first_account_change_family_name}*{first_account_pin_number}" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -445,21 +429,21 @@ stages: status_code: - 200 headers: - Content-Length: '36' + Content-Length: '31' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "END Thank you for using the service." + expected_response: "CON Enter year of birth\n0. Back" - - name: Enter name change pin [second account] + - name: Select gender [Female - second account] request: url: "{server_url}" data: serviceCode: "*483*46#" sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*1*{second_account_change_given_name}*{second_account_change_family_name}*{second_account_pin_number}" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -467,21 +451,109 @@ stages: status_code: - 200 headers: - Content-Length: '30' + Content-Length: '35' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "END Asante kwa kutumia huduma." + expected_response: "CON Weka mwaka wa kuzaliwa\n0. Nyuma" - - name: Second profile management start menu [first account] + - name: Enter age [1993 - first account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_1}" + sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993" + headers: + content-type: "application/x-www-form-urlencoded" + method: POST + response: + status_code: + - 200 + headers: + Content-Length: '31' + Content-Type: "text/plain" + verify_response_with: + function: ext.validator:validate_response + extra_kwargs: + expected_response: "CON Enter your location\n0. Back" + + - name: Enter age [1974 - second account] + request: + url: "{server_url}" + data: + serviceCode: "*483*46#" + sessionId: "{second_profile_management_session_id}" + phoneNumber: "{second_account_phone_number}" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974" + headers: + content-type: "application/x-www-form-urlencoded" + method: POST + response: + status_code: + - 200 + headers: + Content-Length: '27' + Content-Type: "text/plain" + verify_response_with: + function: ext.validator:validate_response + extra_kwargs: + expected_response: "CON Weka eneo lako\n0. Nyuma" + + - name: Enter location [first_account_location - first account] + request: + url: "{server_url}" + data: + serviceCode: "*483*46#" + sessionId: "{first_profile_management_session_id}" + phoneNumber: "{first_account_phone_number}" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}" + headers: + content-type: "application/x-www-form-urlencoded" + method: POST + response: + status_code: + - 200 + headers: + Content-Length: '55' + Content-Type: "text/plain" + verify_response_with: + function: ext.validator:validate_response + extra_kwargs: + expected_response: "CON Please enter a product or service you offer\n0. Back" + + - name: Enter location [second_account_location - second account] + request: + url: "{server_url}" + data: + serviceCode: "*483*46#" + sessionId: "{second_profile_management_session_id}" + phoneNumber: "{second_account_phone_number}" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}" + headers: + content-type: "application/x-www-form-urlencoded" + method: POST + response: + status_code: + - 200 + headers: + Content-Length: '42' + Content-Type: "text/plain" + verify_response_with: + function: ext.validator:validate_response + extra_kwargs: + expected_response: "CON Weka bidhaa ama huduma unauza\n0. Nyuma" + + - name: Enter product [first_account_product - first account] + request: + url: "{server_url}" + data: + serviceCode: "*483*46#" + sessionId: "{first_profile_management_session_id}" + phoneNumber: "{first_account_phone_number}" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}*{first_account_product}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -496,14 +568,14 @@ stages: extra_kwargs: expected_response: "CON Balance 58.00 {token_symbol}\n1. Send\n2. My Account\n3. Help" - - name: Second profile management start menu [second account] + - name: Enter product [second_account_product - second account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_1}" + sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}*{second_account_product}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -523,9 +595,9 @@ stages: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_1}" + sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}*{first_account_product}*2" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -540,14 +612,14 @@ stages: extra_kwargs: expected_response: "CON My account\n1. My profile\n2. Change language\n3. Check balance\n4. Check statement\n5. Change PIN\n0. Back" - - name: Second account management [second account] + - name: Second account management menu [second account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_1}" + sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}*{second_account_product}*2" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -567,9 +639,9 @@ stages: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_1}" + sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}*{first_account_product}*2*1" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -577,21 +649,21 @@ stages: status_code: - 200 headers: - Content-Length: '103' + Content-Length: '115' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit location\n4. Edit products\n5. View my profile\n0. Back" + expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit age\n4. Edit location\n5. Edit products\n6. View my profile\n0. Back" - name: Second profile management menu [second account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_1}" + sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}*{second_account_product}*2*1" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -599,813 +671,21 @@ stages: status_code: - 200 headers: - Content-Length: '104' + Content-Length: '117' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka eneo\n4. Weka bidhaa\n5. Angalia wasifu wako\n0. Nyuma" - - - name: Gender change [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_1}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '51' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Enter gender\n1. Male\n2. Female\n3. Other\n0. Back" - - - name: Gender change [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_1}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '64' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n3. Nyngine\n0. Nyuma" - - - name: Select gender [female - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_1}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*2*2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '33' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Please enter your PIN\n0. Back" - - - name: Select gender [male - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_1}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*2*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '36' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Tafadhali weka PIN yako\n0. Nyuma" - - - name: Enter gender change pin [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_1}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*2*2*{first_account_pin_number}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '36' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "END Thank you for using the service." - - - name: Enter gender change pin [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_1}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*2*1*{second_account_pin_number}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '30' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "END Asante kwa kutumia huduma." - - - name: Third profile management start menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_2}" - phoneNumber: "{first_account_phone_number}" - text: "" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '51' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Balance 58.00 {token_symbol}\n1. Send\n2. My Account\n3. Help" - - - name: Third profile management start menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_2}" - phoneNumber: "{second_account_phone_number}" - text: "" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '56' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Salio 42.00 {token_symbol}\n1. Tuma\n2. Akaunti yangu\n3. Usaidizi" - - - name: Third account management menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_2}" - phoneNumber: "{first_account_phone_number}" - text: "2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '105' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON My account\n1. My profile\n2. Change language\n3. Check balance\n4. Check statement\n5. Change PIN\n0. Back" - - - name: Third account management [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_2}" - phoneNumber: "{second_account_phone_number}" - text: "2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '148' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Akaunti yangu\n1. Wasifu wangu\n2. Chagua lugha utakayotumia\n3. Angalia salio\n4. Angalia taarifa ya matumizi\n5. Badilisha nambari ya siri\n0. Nyuma" - - - name: Third profile management menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_2}" - phoneNumber: "{first_account_phone_number}" - text: "2*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '103' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit location\n4. Edit products\n5. View my profile\n0. Back" - - - name: Third profile management menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_2}" - phoneNumber: "{second_account_phone_number}" - text: "2*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '104' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka eneo\n4. Weka bidhaa\n5. Angalia wasifu wako\n0. Nyuma" - - - name: Location change [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_2}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*3" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '31' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Enter your location\n0. Back" - - - name: Location change [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_2}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*3" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '27' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka eneo lako\n0. Nyuma" - - - name: Enter location change [first_account_change_location - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_2}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*3*{first_account_change_location}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '33' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Please enter your PIN\n0. Back" - - - name: Enter location change [second_account_change_location - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_2}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*3*{second_account_change_location}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '36' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Tafadhali weka PIN yako\n0. Nyuma" - - - name: Enter location change pin [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_2}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*3*{first_account_change_location}*{first_account_pin_number}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '36' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "END Thank you for using the service." - - - name: Enter location change pin [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_2}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*3*{second_account_change_location}*{second_account_pin_number}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '30' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "END Asante kwa kutumia huduma." - - - name: Fourth profile management start menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_3}" - phoneNumber: "{first_account_phone_number}" - text: "" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '51' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Balance 58.00 {token_symbol}\n1. Send\n2. My Account\n3. Help" - - - name: Fourth profile management start menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_3}" - phoneNumber: "{second_account_phone_number}" - text: "" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '56' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Salio 42.00 {token_symbol}\n1. Tuma\n2. Akaunti yangu\n3. Usaidizi" - - - name: Fourth account management menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_3}" - phoneNumber: "{first_account_phone_number}" - text: "2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '105' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON My account\n1. My profile\n2. Change language\n3. Check balance\n4. Check statement\n5. Change PIN\n0. Back" - - - name: Fourth account management menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_3}" - phoneNumber: "{second_account_phone_number}" - text: "2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '148' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Akaunti yangu\n1. Wasifu wangu\n2. Chagua lugha utakayotumia\n3. Angalia salio\n4. Angalia taarifa ya matumizi\n5. Badilisha nambari ya siri\n0. Nyuma" - - - name: Fourth profile management menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_3}" - phoneNumber: "{first_account_phone_number}" - text: "2*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '103' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit location\n4. Edit products\n5. View my profile\n0. Back" - - - name: Fourth profile management menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_3}" - phoneNumber: "{second_account_phone_number}" - text: "2*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '104' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka eneo\n4. Weka bidhaa\n5. Angalia wasifu wako\n0. Nyuma" - - - name: Product change [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_3}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*4" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '55' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Please enter a product or service you offer\n0. Back" - - - name: Product change [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_3}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*4" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '42' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Weka bidhaa ama huduma unauza\n0. Nyuma" - - - name: Enter product change [first_account_change_product - first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_3}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*4*{first_account_change_product}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '33' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Please enter your PIN\n0. Back" - - - name: Enter product change [second_account_change_product - second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_3}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*4*{second_account_change_product}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '36' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Tafadhali weka PIN yako\n0. Nyuma" - - - name: Enter product change pin [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_3}" - phoneNumber: "{first_account_phone_number}" - text: "2*1*4*{first_account_change_product}*{first_account_pin_number}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '36' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "END Thank you for using the service." - - - name: Enter product change pin [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_3}" - phoneNumber: "{second_account_phone_number}" - text: "2*1*4*{second_account_change_product}*{second_account_pin_number}" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '30' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "END Asante kwa kutumia huduma." - - - name: Fifth profile managment start menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_4}" - phoneNumber: "{first_account_phone_number}" - text: "" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '51' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Balance 58.00 {token_symbol}\n1. Send\n2. My Account\n3. Help" - - - name: Fifth profile managment start menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_4}" - phoneNumber: "{second_account_phone_number}" - text: "" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '56' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Salio 42.00 {token_symbol}\n1. Tuma\n2. Akaunti yangu\n3. Usaidizi" - - - name: Fifth account management menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_4}" - phoneNumber: "{first_account_phone_number}" - text: "2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '105' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON My account\n1. My profile\n2. Change language\n3. Check balance\n4. Check statement\n5. Change PIN\n0. Back" - - - name: Fifth account management menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_4}" - phoneNumber: "{second_account_phone_number}" - text: "2" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '148' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Akaunti yangu\n1. Wasifu wangu\n2. Chagua lugha utakayotumia\n3. Angalia salio\n4. Angalia taarifa ya matumizi\n5. Badilisha nambari ya siri\n0. Nyuma" - - - name: Fifth profile management menu [first account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_4}" - phoneNumber: "{first_account_phone_number}" - text: "2*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '103' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit location\n4. Edit products\n5. View my profile\n0. Back" - - - name: Fifth profile management menu [second account] - request: - url: "{server_url}" - data: - serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_4}" - phoneNumber: "{second_account_phone_number}" - text: "2*1" - headers: - content-type: "application/x-www-form-urlencoded" - method: POST - response: - status_code: - - 200 - headers: - Content-Length: '104' - Content-Type: "text/plain" - verify_response_with: - function: ext.validator:validate_response - extra_kwargs: - expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka eneo\n4. Weka bidhaa\n5. Angalia wasifu wako\n0. Nyuma" + expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka umri\n4. Weka eneo\n5. Weka bidhaa\n6. Angalia wasifu wako\n0. Nyuma" - name: Second enter pin to view profile [first account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_4}" + sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}*{first_account_product}*2*1*6" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -1425,9 +705,9 @@ stages: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_4}" + sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}*{second_account_product}*2*1*6" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -1447,9 +727,9 @@ stages: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_4}" + sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{first_account_pin_number}" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}*{first_account_product}*2*1*6*{first_account_pin_number}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -1461,16 +741,16 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Your details are:\n Name: {first_account_change_given_name} {first_account_change_family_name}\n Gender: Female\n Location: {first_account_change_location}\n You sell: {first_account_change_product}\n0. Back" + expected_response: "CON Your details are:\n Name: {first_account_given_name} {first_account_family_name}\n Gender: Male\n Age: 28\n Location: {first_account_location}\n You sell: {first_account_product}\n0. Back" - name: Second display profile [second account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_4}" + sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}*{second_account_product}*2*1*6*{second_account_pin_number}" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -1482,16 +762,16 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Wasifu wako una maelezo yafuatayo:\n Jina: {second_account_change_given_name} {second_account_change_family_name}\n Jinsia: Mwanaume\n Eneo: {second_account_change_location}\n Unauza: {second_account_change_product}\n0. Nyuma" + expected_response: "CON Wasifu wako una maelezo yafuatayo:\n Jina: {second_account_given_name} {second_account_family_name}\n Jinsia: Mwanamke\n Umri: 47\n Eneo: {second_account_location}\n Unauza: {second_account_product}\n0. Nyuma" - name: Return to profile management menu [first account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_4}" + sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{first_account_pin_number}*0" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}*{first_account_product}*2*1*6*{first_account_pin_number}*0" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -1499,21 +779,21 @@ stages: status_code: - 200 headers: - Content-Length: '103' + Content-Length: '115' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit location\n4. Edit products\n5. View my profile\n0. Back" + expected_response: "CON My profile\n1. Edit name\n2. Edit gender\n3. Edit age\n4. Edit location\n5. Edit products\n6. View my profile\n0. Back" - name: Return to profile management menu [second account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_4}" + sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}*{second_account_product}*2*1*6*{second_account_pin_number}*0" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -1521,21 +801,21 @@ stages: status_code: - 200 headers: - Content-Length: '104' + Content-Length: '117' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka eneo\n4. Weka bidhaa\n5. Angalia wasifu wako\n0. Nyuma" + expected_response: "CON Wasifu wangu\n1. Weka jina\n2. Weka jinsia\n3. Weka umri\n4. Weka eneo\n5. Weka bidhaa\n6. Angalia wasifu wako\n0. Nyuma" - name: Resume start menu [first account] request: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{first_profile_management_session_id_4}" + sessionId: "{first_profile_management_session_id}" phoneNumber: "{first_account_phone_number}" - text: "2*1*5*{first_account_pin_number}*0*0" + text: "2*1*6*{first_account_pin_number}*0*1*{first_account_given_name}*{first_account_family_name}*1*1993*{first_account_location}*{first_account_product}*2*1*6*{first_account_pin_number}*0*0" headers: content-type: "application/x-www-form-urlencoded" method: POST @@ -1555,9 +835,9 @@ stages: url: "{server_url}" data: serviceCode: "*483*46#" - sessionId: "{second_profile_management_session_id_4}" + sessionId: "{second_profile_management_session_id}" phoneNumber: "{second_account_phone_number}" - text: "2*1*5*{second_account_pin_number}*0*0" + text: "2*1*6*{second_account_pin_number}*0*1*{second_account_given_name}*{second_account_family_name}*2*1974*{second_account_location}*{second_account_product}*2*1*6*{second_account_pin_number}*0*0" headers: content-type: "application/x-www-form-urlencoded" method: POST diff --git a/apps/cic-ussd/tests/integration/test_transactions.tavern.yaml b/apps/cic-ussd/tests/integration/test_transactions.tavern.yaml index 9a50f012..f71e7f70 100644 --- a/apps/cic-ussd/tests/integration/test_transactions.tavern.yaml +++ b/apps/cic-ussd/tests/integration/test_transactions.tavern.yaml @@ -170,7 +170,7 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back" + expected_response: "CON {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back" - name: Enter transcation amount [second account] request: @@ -191,7 +191,7 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma" + expected_response: "CON {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma" - name: Pin to authorize transaction [first account] request: @@ -212,7 +212,7 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Your request has been sent. {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\n00. Back\n99. Exit" + expected_response: "CON Your request has been sent. {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\n00. Back\n99. Exit" - name: Pin to authorize transaction [second account] request: @@ -233,7 +233,7 @@ stages: verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Ombi lako limetumwa. {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\n00. Nyuma\n99. Ondoka" + expected_response: "CON Ombi lako limetumwa. {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\n00. Nyuma\n99. Ondoka" - name: Verify balance changes [first account] delay_before: 10 diff --git a/apps/cic-ussd/transitions/age_setting_transitions.json b/apps/cic-ussd/transitions/age_setting_transitions.json new file mode 100644 index 00000000..957c6ae8 --- /dev/null +++ b/apps/cic-ussd/transitions/age_setting_transitions.json @@ -0,0 +1,39 @@ +[ + { + "trigger": "scan_data", + "source": "enter_date_of_birth", + "dest": "dob_edit_pin_authorization", + "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", + "conditions": [ + "cic_ussd.state_machine.logic.validator.has_cached_user_metadata", + "cic_ussd.state_machine.logic.validator.is_valid_date" + ] + }, + { + "trigger": "scan_data", + "source": "enter_date_of_birth", + "dest": "enter_location", + "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date", + "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", + "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + }, + { + "trigger": "scan_data", + "source": "enter_date_of_birth", + "dest": "exit_invalid_input", + "unless": "cic_ussd.state_machine.logic.validator.is_valid_date" + }, + { + "trigger": "scan_data", + "source": "dob_edit_pin_authorization", + "dest": "exit", + "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin", + "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute" + }, + { + "trigger": "scan_data", + "source": "dob_edit_pin_authorization", + "dest": "exit_pin_blocked", + "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account" + } +] \ No newline at end of file diff --git a/apps/cic-ussd/transitions/gender_setting_transitions.json b/apps/cic-ussd/transitions/gender_setting_transitions.json index 7881cdd7..e0d2b346 100644 --- a/apps/cic-ussd/transitions/gender_setting_transitions.json +++ b/apps/cic-ussd/transitions/gender_setting_transitions.json @@ -2,7 +2,7 @@ { "trigger": "scan_data", "source": "enter_gender", - "dest": "enter_location", + "dest": "enter_date_of_birth", "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", "conditions": "cic_ussd.state_machine.logic.validator.is_valid_gender_selection", "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" diff --git a/apps/cic-ussd/transitions/user_metadata_transitions.json b/apps/cic-ussd/transitions/user_metadata_transitions.json index dff068f6..f0f428e3 100644 --- a/apps/cic-ussd/transitions/user_metadata_transitions.json +++ b/apps/cic-ussd/transitions/user_metadata_transitions.json @@ -14,21 +14,27 @@ { "trigger": "scan_data", "source": "metadata_management", - "dest": "enter_location", + "dest": "enter_date_of_birth", "conditions": "cic_ussd.state_machine.logic.menu.menu_three_selected" }, { "trigger": "scan_data", "source": "metadata_management", - "dest": "enter_products", + "dest": "enter_location", "conditions": "cic_ussd.state_machine.logic.menu.menu_four_selected" }, { "trigger": "scan_data", "source": "metadata_management", - "dest": "display_metadata_pin_authorization", + "dest": "enter_products", "conditions": "cic_ussd.state_machine.logic.menu.menu_five_selected" }, + { + "trigger": "scan_data", + "source": "metadata_management", + "dest": "display_metadata_pin_authorization", + "conditions": "cic_ussd.state_machine.logic.menu.menu_six_selected" + }, { "trigger": "scan_data", "source": "display_metadata_pin_authorization", diff --git a/apps/cic-ussd/var/lib/locale/ussd.en.yml b/apps/cic-ussd/var/lib/locale/ussd.en.yml index d1f0653b..bfb6545a 100644 --- a/apps/cic-ussd/var/lib/locale/ussd.en.yml +++ b/apps/cic-ussd/var/lib/locale/ussd.en.yml @@ -17,6 +17,9 @@ en: enter_family_name: |- CON Enter family name 0. Back + enter_date_of_birth: |- + CON Enter year of birth + 0. Back enter_gender: |- CON Enter gender 1. Male @@ -52,14 +55,16 @@ en: CON My profile 1. Edit name 2. Edit gender - 3. Edit location - 4. Edit products - 5. View my profile + 3. Edit age + 4. Edit location + 5. Edit products + 6. View my profile 0. Back display_user_metadata: |- CON Your details are: Name: %{full_name} Gender: %{gender} + Age: %{age} Location: %{location} You sell: %{products} 0. Back @@ -117,6 +122,13 @@ en: retry: |- CON Please enter your PIN. You have %{remaining_attempts} attempts remaining 0. Back + dob_edit_pin_authorization: + first: |- + CON Please enter your PIN + 0. Back + retry: |- + CON Please enter your PIN. You have %{remaining_attempts} attempts remaining + 0. Back gender_edit_pin_authorization: first: |- CON Please enter your PIN diff --git a/apps/cic-ussd/var/lib/locale/ussd.sw.yml b/apps/cic-ussd/var/lib/locale/ussd.sw.yml index a40db2b7..81690287 100644 --- a/apps/cic-ussd/var/lib/locale/ussd.sw.yml +++ b/apps/cic-ussd/var/lib/locale/ussd.sw.yml @@ -17,6 +17,9 @@ sw: enter_family_name: |- CON Weka jina lako la mwisho 0. Nyuma + enter_date_of_birth: |- + CON Weka mwaka wa kuzaliwa + 0. Nyuma enter_gender: |- CON Weka jinsia yako 1. Mwanaume @@ -52,14 +55,16 @@ sw: CON Wasifu wangu 1. Weka jina 2. Weka jinsia - 3. Weka eneo - 4. Weka bidhaa - 5. Angalia wasifu wako + 3. Weka umri + 4. Weka eneo + 5. Weka bidhaa + 6. Angalia wasifu wako 0. Nyuma display_user_metadata: |- CON Wasifu wako una maelezo yafuatayo: Jina: %{full_name} Jinsia: %{gender} + Umri: %{age} Eneo: %{location} Unauza: %{products} 0. Nyuma @@ -117,6 +122,13 @@ sw: retry: |- CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. 0. Nyuma + dob_edit_pin_authorization: + first: |- + CON Tafadhali weka PIN yako + 0. Nyuma + retry: |- + CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. + 0. Nyuma gender_edit_pin_authorization: first: |- CON Tafadhali weka PIN yako diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index 5f5ec43b..708f3809 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -84,6 +84,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then >&2 echo "add deployer address as account index writer" eth-accounts-index-writer $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER + >&2 echo "deploy contract registry contract" CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w` eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS @@ -95,7 +96,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS # Deploy transfer authorization contact - >&2 echo "deploy address declarator contract" + >&2 echo "deploy transfer auth contract" DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv` eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS diff --git a/apps/data-seeding/README.md b/apps/data-seeding/README.md index cb33352a..1b9e645b 100644 --- a/apps/data-seeding/README.md +++ b/apps/data-seeding/README.md @@ -136,7 +136,7 @@ First, make a note of the **block height** before running anything: To import, run to _completion_: -`python eth/import_users.py -v -c config -p -r -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` +`python eth/import_users.py -v -c config -p -r -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` After the script completes, keystore files for all generated accouts will be found in `/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one). @@ -150,7 +150,7 @@ Then run: Run in sequence, in first terminal: -`python cic_eth/import_balance.py -v -c config -p -r --token-symbol -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out` +`python cic_eth/import_balance.py -v -c config -p -r --token-symbol -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out` In another terminal: @@ -171,7 +171,11 @@ Then, in sequence, run in first terminal: In second terminal: -`python cic_ussd/import_users.py -v -c config out` +`python cic_ussd/import_users.py -v --ussd-host --ussd-port -c config out` + +In the event that you are running the command in a local environment you may want to consider passing the `--ussd-no-ssl` flag i.e: + +`python cic_ussd/import_users.py -v --ussd-host --ussd-port --ussd-no-ssl -c config out` @@ -199,6 +203,13 @@ If _number of users_ is omitted the script will run until manually interrupted. If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing. +### Importing preferences metadata + +`node cic_meta/import_meta_preferences.js ` + +If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported. + + ##### Importing pins and ussd data (optional) Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps: @@ -226,7 +237,7 @@ The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in ### Step 5 - Verify -`python verify.py -v -c config -r -p ` +`python verify.py -v -c config -r -p --token-symbol ` Included checks: * Private key is in cic-eth keystore @@ -262,3 +273,5 @@ Should exit with code 0 if all input data is found in the respective services. - MacOS BigSur issue when installing psycopg2: ld: library not found for -lssl -> https://github.com/psycopg/psycopg2/issues/1115#issuecomment-831498953 - `cic_ussd` imports is poorly implemented, and consumes a lot of resources. Therefore it takes a long time to complete. Reducing the amount of polls for the phone pointer would go a long way to improve it. + +- A strict constraint is maintained insistin the use of postgresql-12. diff --git a/apps/data-seeding/cic_eth/import_balance.py b/apps/data-seeding/cic_eth/import_balance.py index 97c75d60..99e8eef4 100644 --- a/apps/data-seeding/cic_eth/import_balance.py +++ b/apps/data-seeding/cic_eth/import_balance.py @@ -18,19 +18,17 @@ from hexathon import ( add_0x, ) from chainsyncer.backend.memory import MemBackend -from chainsyncer.driver import HeadSyncer +from chainsyncer.driver.head import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( block_latest, - block_by_number, - Block, ) from chainlib.hash import keccak256_string_to_hex from chainlib.eth.address import to_checksum_address from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.tx import TxFactory -from chainlib.jsonrpc import jsonrpc_template +from chainlib.jsonrpc import JSONRPCRequest from chainlib.eth.error import EthException from chainlib.chain import ChainSpec from chainlib.eth.constant import ZERO_ADDRESS @@ -38,6 +36,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from crypto_dev_signer.keystore.dict import DictKeystore from cic_types.models.person import Person from eth_erc20 import ERC20 +from cic_base.eth.syncer import chain_interface logging.basicConfig(level=logging.WARNING) @@ -70,12 +69,14 @@ elif args.vv == True: config_dir = os.path.join(args.c) os.makedirs(config_dir, 0o777, True) config = confini.Config(config_dir, args.env_prefix) -config.process() # override args +config.process() +logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) args_override = { 'CIC_CHAIN_SPEC': getattr(args, 'i'), 'ETH_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), + 'KEYSTORE_FILE_PATH': getattr(args, 'y'), } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') @@ -184,27 +185,6 @@ class Handler: # logg.error('key record not found in imports: {}'.format(e).ljust(200)) -#class BlockGetter: -# -# def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec): -# self.conn = conn -# self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id) -# -# -# def get(self, n): -# o = block_by_number(n) -# r = self.conn.do(o) -# b = None -# try: -# b = Block(r) -# except TypeError as e: -# if r == None: -# logg.debug('block not found {}'.format(n)) -# else: -# logg.error('block retrieve error {}'.format(e)) -# return b - - def progress_callback(block_number, tx_index): sys.stdout.write(str(block_number).ljust(200) + "\n") @@ -225,11 +205,13 @@ def main(): data = add_0x(registry_addressof_method) data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex() txf.set_code(tx, data) - - o = jsonrpc_template() + + j = JSONRPCRequest() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) logg.info('found token index address {}'.format(token_index_address)) @@ -243,10 +225,11 @@ def main(): z = h.digest() data += eth_abi.encode_single('bytes32', z).hex() txf.set_code(tx, data) - o = jsonrpc_template() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) try: sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) @@ -304,7 +287,7 @@ def main(): f.close() syncer_backend.set(block_offset, 0) - syncer = HeadSyncer(syncer_backend, block_callback=progress_callback) + syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback) handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/data-seeding/cic_eth/import_users.py b/apps/data-seeding/cic_eth/import_users.py index f803913b..8faa152c 100644 --- a/apps/data-seeding/cic_eth/import_users.py +++ b/apps/data-seeding/cic_eth/import_users.py @@ -194,8 +194,7 @@ if __name__ == '__main__': f.write(json.dumps(o)) f.close() - #fi.write('{},{}\n'.format(new_address, old_address)) - meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person') + meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person') meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper())) os.symlink(os.path.realpath(filepath), meta_filepath) @@ -221,7 +220,7 @@ if __name__ == '__main__': # custom data - custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom') + custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.custom') custom_filepath = os.path.join(custom_dir, 'meta', custom_key) filepath = os.path.join( diff --git a/apps/data-seeding/cic_meta/import_meta.js b/apps/data-seeding/cic_meta/import_meta.js index 2884fa95..7bdda53a 100644 --- a/apps/data-seeding/cic_meta/import_meta.js +++ b/apps/data-seeding/cic_meta/import_meta.js @@ -31,13 +31,16 @@ function sendit(uid, envelope) { const req = http.request(url + uid, opts, (res) => { res.on('data', process.stdout.write); res.on('end', () => { + if (!res.complete) { + console.log('The connection was terminated while the message was being sent.') + } console.log('result', res.statusCode, res.headers); }); }); - if (!req.write(d)) { - console.error('foo', d); - process.exit(1); - } + req.on('error', (err) => { + console.log('ERROR when talking to meta', err) + }) + req.write(d) req.end(); } @@ -55,6 +58,7 @@ function doOne(keystore, filePath) { const s = new crdt.Syncable(uid, o); s.setSigner(signer); s.onwrap = (env) => { + console.log(`Sending uid: ${uid} and env: ${env} to meta`) sendit(uid, env); }; s.sign(); @@ -84,6 +88,7 @@ let batchCount = 0; function importMeta(keystore) { + console.log('Running importMeta....') let err; let files; @@ -94,6 +99,11 @@ function importMeta(keystore) { setTimeout(importMeta, batchDelay, keystore); return; } + console.log(`Trying to read ${files.length} files`) + if (files === 0) { + console.log(`ERROR did not find any files under ${workDir}. \nLooks like there is no work for me, bailing!`) + process.exit(1) + } let limit = batchSize; if (files.length < limit) { limit = files.length; @@ -108,6 +118,7 @@ function importMeta(keystore) { doOne(keystore, filePath); count++; batchCount++; + //console.log('done one', count, batchCount) if (batchCount == batchSize) { console.debug('reached batch size, breathing'); batchCount=0; diff --git a/apps/data-seeding/cic_meta/import_meta_preferences.js b/apps/data-seeding/cic_meta/import_meta_preferences.js new file mode 100644 index 00000000..be9be563 --- /dev/null +++ b/apps/data-seeding/cic_meta/import_meta_preferences.js @@ -0,0 +1,133 @@ +const fs = require('fs'); +const path = require('path'); +const http = require('http'); + +const cic = require('@cicnet/cic-client-meta'); +const crdt = require('@cicnet/crdt-meta'); + +//const conf = JSON.parse(fs.readFileSync('./cic.conf')); + +const config = new crdt.Config('./config'); +config.process(); +console.log(config); + + +function sendit(uid, envelope) { + const d = envelope.toJSON(); + + const contentLength = (new TextEncoder().encode(d)).length; + const opts = { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': contentLength, + 'X-CIC-AUTOMERGE': 'client', + + }, + }; + let url = config.get('META_URL'); + url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/'); + console.log('posting to url: ' + url + uid); + const req = http.request(url + uid, opts, (res) => { + res.on('data', process.stdout.write); + res.on('end', () => { + console.log('result', res.statusCode, res.headers); + }); + }); + if (!req.write(d)) { + console.error('foo', d); + process.exit(1); + } + req.end(); +} + +function doOne(keystore, filePath, identifier) { + const signer = new crdt.PGPSigner(keystore); + + const o = JSON.parse(fs.readFileSync(filePath).toString()); + + cic.Custom.toKey(identifier).then((uid) => { + const s = new crdt.Syncable(uid, o); + s.setSigner(signer); + s.onwrap = (env) => { + sendit(identifier, env); + }; + s.sign(); + }); +} + +const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE')); +const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE')); +pk = fs.readFileSync(privateKeyPath); +pubk = fs.readFileSync(publicKeyPath); + +new crdt.PGPKeyStore( + config.get('PGP_PASSPHRASE'), + pk, + pubk, + undefined, + undefined, + importMetaCustom, +); + +const batchSize = 16; +const batchDelay = 1000; +const total = parseInt(process.argv[3]); +const dataDir = process.argv[2]; +const workDir = path.join(dataDir, 'preferences/meta'); +const userDir = path.join(dataDir, 'preferences/new'); +let count = 0; +let batchCount = 0; + + +function importMetaCustom(keystore) { + let err; + let files; + + try { + err, files = fs.readdirSync(workDir); + } catch { + console.error('source directory not yet ready', workDir); + setTimeout(importMetaCustom, batchDelay, keystore); + return; + } + let limit = batchSize; + if (files.length < limit) { + limit = files.length; + } + for (let i = 0; i < limit; i++) { + const file = files[i]; + if (file.length < 3) { + console.debug('skipping file', file); + continue; + } + //const identifier = file.substr(0,file.length-5); + const identifier = file; + const filePath = path.join(workDir, file); + console.log(filePath); + + //const address = fs.readFileSync(filePath).toString().substring(2).toUpperCase(); + const custom = JSON.parse(fs.readFileSync(filePath).toString()); + const customFilePath = path.join( + userDir, + identifier.substring(0, 2), + identifier.substring(2, 4), + identifier + '.json', + ); + + doOne(keystore, filePath, identifier); + fs.unlinkSync(filePath); + count++; + batchCount++; + if (batchCount == batchSize) { + console.debug('reached batch size, breathing'); + batchCount=0; + setTimeout(importMetaCustom, batchDelay, keystore); + return; + } + } + if (count == total) { + return; + } + setTimeout(importMetaCustom, 100, keystore); +} diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 35430e95..56435608 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -70,6 +70,7 @@ args_override = { 'REDIS_DB': getattr(args, 'redis_db'), 'META_HOST': getattr(args, 'meta_host'), 'META_PORT': getattr(args, 'meta_port'), + 'KEYSTORE_FILE_PATH': getattr(args, 'y') } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') @@ -114,7 +115,7 @@ def main(): conn = EthHTTPConnection(config.get('ETH_PROVIDER')) ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer) - ImportTask.balance_processor.init() + ImportTask.balance_processor.init(token_symbol) # TODO get decimals from token balances = {} @@ -139,6 +140,7 @@ def main(): ImportTask.balances = balances ImportTask.count = i + ImportTask.import_dir = user_dir s = celery.signature( 'import_task.send_txs', diff --git a/apps/data-seeding/cic_ussd/import_pins.py b/apps/data-seeding/cic_ussd/import_pins.py index 37bc5eb4..119f8d07 100644 --- a/apps/data-seeding/cic_ussd/import_pins.py +++ b/apps/data-seeding/cic_ussd/import_pins.py @@ -39,6 +39,7 @@ elif args.vv: config_dir = args.c config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) config.process() +logg.debug('config loaded from {}:\n{}'.format(args.c, config)) celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) @@ -62,9 +63,6 @@ def main(): ) s_import_pins.apply_async() - argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG'] - celery_app.worker_main(argv) - if __name__ == '__main__': main() diff --git a/apps/data-seeding/cic_ussd/import_task.py b/apps/data-seeding/cic_ussd/import_task.py index bc06b4e8..c5917688 100644 --- a/apps/data-seeding/cic_ussd/import_task.py +++ b/apps/data-seeding/cic_ussd/import_task.py @@ -1,6 +1,7 @@ # standard imports import os import logging +import random import urllib.parse import urllib.error import urllib.request @@ -136,6 +137,42 @@ def generate_metadata(self, address, phone): ) os.symlink(os.path.realpath(filepath), meta_filepath) + # write ussd data + ussd_data = { + 'phone': phone, + 'is_activated': 1, + 'preferred_language': random.sample(['en', 'sw'], 1)[0], + 'is_disabled': False + } + ussd_data_dir = os.path.join(self.import_dir, 'ussd') + ussd_data_file_path = os.path.join(ussd_data_dir, f'{old_address}.json') + f = open(ussd_data_file_path, 'w') + f.write(json.dumps(ussd_data)) + f.close() + + # write preferences data + preferences_dir = os.path.join(self.import_dir, 'preferences') + preferences_data = { + 'preferred_language': ussd_data['preferred_language'] + } + + preferences_key = generate_metadata_pointer(bytes.fromhex(new_address_clean[2:]), ':cic.preferences') + preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key) + + filepath = os.path.join( + preferences_dir, + 'new', + preferences_key[:2].upper(), + preferences_key[2:4].upper(), + preferences_key.upper() + '.json' + ) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + + f = open(filepath, 'w') + f.write(json.dumps(preferences_data)) + f.close() + os.symlink(os.path.realpath(filepath), preferences_filepath) + logg.debug('found metadata {} for phone {}'.format(o, phone)) return address diff --git a/apps/data-seeding/cic_ussd/import_users.py b/apps/data-seeding/cic_ussd/import_users.py index 1898c17c..e44b92fb 100644 --- a/apps/data-seeding/cic_ussd/import_users.py +++ b/apps/data-seeding/cic_ussd/import_users.py @@ -1,29 +1,21 @@ # standard imports -import os -import sys +import argparse import json import logging -import argparse -import uuid -import datetime +import os +import sys import time import urllib.request -from glob import glob +import uuid +from urllib.parse import urlencode -# third-party imports -import redis -import confini +# external imports import celery -from hexathon import ( - add_0x, - strip_0x, - ) -from chainlib.eth.address import to_checksum -from cic_types.models.person import Person -from cic_eth.api.api_task import Api -from chainlib.chain import ChainSpec -from cic_types.processor import generate_metadata_pointer +import confini import phonenumbers +import redis +from chainlib.chain import ChainSpec +from cic_types.models.person import Person logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -39,6 +31,9 @@ argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db t argparser.add_argument('--batch-size', dest='batch_size', default=100, type=int, help='burst size of sending transactions to node') # batch size should be slightly below cumulative gas limit worth, eg 80000 gas txs with 8000000 limit is a bit less than 100 batch size argparser.add_argument('--batch-delay', dest='batch_delay', default=3, type=int, help='seconds delay between batches') argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout') +argparser.add_argument('--ussd-host', dest='ussd_host', type=str, help="host to ussd app responsible for processing ussd requests.") +argparser.add_argument('--ussd-port', dest='ussd_port', type=str, help="port to ussd app responsible for processing ussd requests.") +argparser.add_argument('--ussd-no-ssl', dest='ussd_no_ssl', help='do not use ssl (careful)', action='store_true') argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') argparser.add_argument('-v', action='store_true', help='Be verbose') argparser.add_argument('-vv', action='store_true', help='Be more verbose') @@ -72,6 +67,12 @@ ps = r.pubsub() user_new_dir = os.path.join(args.user_dir, 'new') os.makedirs(user_new_dir) +ussd_data_dir = os.path.join(args.user_dir, 'ussd') +os.makedirs(ussd_data_dir) + +preferences_dir = os.path.join(args.user_dir, 'preferences') +os.makedirs(os.path.join(preferences_dir, 'meta')) + meta_dir = os.path.join(args.user_dir, 'meta') os.makedirs(meta_dir) @@ -86,22 +87,22 @@ chain_str = str(chain_spec) batch_size = args.batch_size batch_delay = args.batch_delay - -db_configs = { - 'database': config.get('DATABASE_NAME'), - 'host': config.get('DATABASE_HOST'), - 'port': config.get('DATABASE_PORT'), - 'user': config.get('DATABASE_USER'), - 'password': config.get('DATABASE_PASSWORD') -} - +ussd_port = args.ussd_port +ussd_host = args.ussd_host +ussd_no_ssl = args.ussd_no_ssl +if ussd_no_ssl is True: + ussd_ssl = False +else: + ussd_ssl = True def build_ussd_request(phone, host, port, service_code, username, password, ssl=False): url = 'http' if ssl: url += 's' - url += '://{}:{}'.format(host, port) - url += '/?username={}&password={}'.format(username, password) #config.get('USSD_USER'), config.get('USSD_PASS')) + url += '://{}'.format(host) + if port: + url += ':{}'.format(port) + url += '/?username={}&password={}'.format(username, password) logg.info('ussd service url {}'.format(url)) logg.info('ussd phone {}'.format(phone)) @@ -114,9 +115,10 @@ def build_ussd_request(phone, host, port, service_code, username, password, ssl= 'text': service_code, } req = urllib.request.Request(url) - data_str = json.dumps(data) + req.method=('POST') + data_str = urlencode(data) data_bytes = data_str.encode('utf-8') - req.add_header('Content-Type', 'application/json') + req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.data = data_bytes return req @@ -126,7 +128,15 @@ def register_ussd(i, u): phone_object = phonenumbers.parse(u.tel) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) logg.debug('tel {} {}'.format(u.tel, phone)) - req = build_ussd_request(phone, 'localhost', 63315, '*483*46#', '', '') + req = build_ussd_request( + phone, + ussd_host, + ussd_port, + config.get('APP_SERVICE_CODE'), + '', + '', + ussd_ssl + ) response = urllib.request.urlopen(req) response_data = response.read().decode('utf-8') state = response_data[:3] @@ -143,59 +153,57 @@ if __name__ == '__main__': if y[len(y)-5:] != '.json': continue # handle json containing person object - filepath = None - if y[:15] != '_ussd_data.json': - filepath = os.path.join(x[0], y) - f = open(filepath, 'r') - try: - o = json.load(f) - except json.decoder.JSONDecodeError as e: - f.close() - logg.error('load error for {}: {}'.format(y, e)) - continue + filepath = os.path.join(x[0], y) + f = open(filepath, 'r') + try: + o = json.load(f) + except json.decoder.JSONDecodeError as e: f.close() - u = Person.deserialize(o) + logg.error('load error for {}: {}'.format(y, e)) + continue + f.close() + u = Person.deserialize(o) - new_address = register_ussd(i, u) + new_address = register_ussd(i, u) - phone_object = phonenumbers.parse(u.tel) - phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) + phone_object = phonenumbers.parse(u.tel) + phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) - s_phone = celery.signature( - 'import_task.resolve_phone', - [ - phone, - ], - queue='cic-import-ussd', - ) + s_phone = celery.signature( + 'import_task.resolve_phone', + [ + phone, + ], + queue='cic-import-ussd', + ) - s_meta = celery.signature( - 'import_task.generate_metadata', - [ - phone, - ], - queue='cic-import-ussd', - ) + s_meta = celery.signature( + 'import_task.generate_metadata', + [ + phone, + ], + queue='cic-import-ussd', + ) - s_balance = celery.signature( - 'import_task.opening_balance_tx', - [ - phone, - i, - ], - queue='cic-import-ussd', - ) + s_balance = celery.signature( + 'import_task.opening_balance_tx', + [ + phone, + i, + ], + queue='cic-import-ussd', + ) - s_meta.link(s_balance) - s_phone.link(s_meta) - # block time plus a bit of time for ussd processing - s_phone.apply_async(countdown=7) + s_meta.link(s_balance) + s_phone.link(s_meta) + # block time plus a bit of time for ussd processing + s_phone.apply_async(countdown=7) - i += 1 - sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r") + i += 1 + sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r") - j += 1 - if j == batch_size: - time.sleep(batch_delay) - j = 0 + j += 1 + if j == batch_size: + time.sleep(batch_delay) + j = 0 diff --git a/apps/data-seeding/cic_ussd/import_ussd_data.py b/apps/data-seeding/cic_ussd/import_ussd_data.py index d2283000..fddd8bb6 100644 --- a/apps/data-seeding/cic_ussd/import_ussd_data.py +++ b/apps/data-seeding/cic_ussd/import_ussd_data.py @@ -31,9 +31,9 @@ elif args.vv: config_dir = args.c config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) config.process() +logg.debug('config loaded from {}:\n{}'.format(args.c, config)) -user_old_dir = os.path.join(args.user_dir, 'old') -os.stat(user_old_dir) +ussd_data_dir = os.path.join(args.user_dir, 'ussd') db_configs = { 'database': config.get('DATABASE_NAME'), @@ -45,18 +45,15 @@ db_configs = { celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) if __name__ == '__main__': - for x in os.walk(user_old_dir): + for x in os.walk(ussd_data_dir): for y in x[2]: - if y[len(y) - 5:] != '.json': - continue - - # handle ussd_data json object - if y[:15] == '_ussd_data.json': + if y[len(y) - 5:] == '.json': filepath = os.path.join(x[0], y) f = open(filepath, 'r') try: ussd_data = json.load(f) + logg.debug(f'LOADING USSD DATA: {ussd_data}') except json.decoder.JSONDecodeError as e: f.close() logg.error('load error for {}: {}'.format(y, e)) diff --git a/apps/data-seeding/cic_ussd/import_util.py b/apps/data-seeding/cic_ussd/import_util.py index ebe706bb..d88b354a 100644 --- a/apps/data-seeding/cic_ussd/import_util.py +++ b/apps/data-seeding/cic_ussd/import_util.py @@ -6,7 +6,7 @@ from eth_contract_registry import Registry from eth_token_index import TokenUniqueSymbolIndex from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import OverrideNonceOracle -from chainlib.eth.erc20 import ERC20 +from eth_erc20 import ERC20 from chainlib.eth.tx import ( count, TxFormat, @@ -37,7 +37,7 @@ class BalanceProcessor: self.value_multiplier = 1 - def init(self): + def init(self, token_symbol): # Get Token registry address registry = Registry(self.chain_spec) o = registry.address_of(self.registry_address, 'TokenRegistry') @@ -46,10 +46,10 @@ class BalanceProcessor: logg.info('found token index address {}'.format(self.token_index_address)) token_registry = TokenUniqueSymbolIndex(self.chain_spec) - o = token_registry.address_of(self.token_index_address, 'SRF') + o = token_registry.address_of(self.token_index_address, token_symbol) r = self.conn.do(o) self.token_address = token_registry.parse_address_of(r) - logg.info('found SRF token address {}'.format(self.token_address)) + logg.info('found {} token address {}'.format(token_symbol, self.token_address)) tx_factory = ERC20(self.chain_spec) o = tx_factory.decimals(self.token_address) diff --git a/apps/data-seeding/config/app.ini b/apps/data-seeding/config/app.ini index 6d37c421..82cd4f1d 100644 --- a/apps/data-seeding/config/app.ini +++ b/apps/data-seeding/config/app.ini @@ -22,3 +22,6 @@ TRANSITIONS=/usr/src/cic-ussd/transitions/ host = port = ssl = + +[keystore] +file_path = keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c diff --git a/apps/data-seeding/config/cic.ini b/apps/data-seeding/config/cic.ini index b2e0b80c..5064df15 100644 --- a/apps/data-seeding/config/cic.ini +++ b/apps/data-seeding/config/cic.ini @@ -7,3 +7,5 @@ approval_escrow_address = chain_spec = evm:bloxberg:8996 tx_retry_delay = trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C +user_ussd_svc_service_port = + diff --git a/apps/data-seeding/config/database.ini b/apps/data-seeding/config/database.ini index e5b89263..f464cf44 100644 --- a/apps/data-seeding/config/database.ini +++ b/apps/data-seeding/config/database.ini @@ -1,5 +1,10 @@ [database] -name = sempo -host = localhost -port = 5432 -user = postgres +NAME=sempo +USER=postgres +PASSWORD= +HOST=localhost +PORT=5432 +ENGINE=postgresql +DRIVER=psycopg2 +DEBUG=0 +POOL_SIZE=1 diff --git a/apps/data-seeding/config/eth.ini b/apps/data-seeding/config/eth.ini index 54eed6ab..321384f8 100644 --- a/apps/data-seeding/config/eth.ini +++ b/apps/data-seeding/config/eth.ini @@ -1,8 +1,2 @@ [eth] -#ws_provider = ws://localhost:8546 -#ttp_provider = http://localhost:8545 provider = http://localhost:63545 -gas_provider_address = -#chain_id = -abi_dir = /usr/local/share/cic/solidity/abi -account_accounts_index_writer = diff --git a/apps/data-seeding/create_import_pins.py b/apps/data-seeding/create_import_pins.py index daebe3ba..d4dd9716 100644 --- a/apps/data-seeding/create_import_pins.py +++ b/apps/data-seeding/create_import_pins.py @@ -3,6 +3,7 @@ import argparse import json import logging import os +import uuid # third-party imports import bcrypt @@ -83,7 +84,7 @@ if __name__ == '__main__': phone_object = phonenumbers.parse(u.tel) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) - password_hash = generate_password_hash() + password_hash = uuid.uuid4().hex pins_file.write(f'{phone},{password_hash}\n') logg.info(f'Writing phone: {phone}, password_hash: {password_hash}') diff --git a/apps/data-seeding/create_import_users.py b/apps/data-seeding/create_import_users.py index 58e0f137..dc252fce 100644 --- a/apps/data-seeding/create_import_users.py +++ b/apps/data-seeding/create_import_users.py @@ -204,9 +204,9 @@ def gen(): ])) if random.randint(0, 1): # fake.local_latitude() - p.location['latitude'] = (random.random() + 180) - 90 + p.location['latitude'] = (random.random() * 180) - 90 # fake.local_latitude() - p.location['longitude'] = (random.random() + 360) - 180 + p.location['longitude'] = (random.random() * 360) - 179 return (old_blockchain_checksum_address, phone, p) @@ -228,7 +228,6 @@ def prepareLocalFilePath(datadir, address): if __name__ == '__main__': base_dir = os.path.join(user_dir, 'old') - ussd_dir = os.path.join(user_dir, 'ussd') os.makedirs(base_dir, exist_ok=True) fa = open(os.path.join(user_dir, 'balances.csv'), 'w') @@ -248,23 +247,11 @@ if __name__ == '__main__': print(o) - ussd_data = { - 'phone': phone, - 'is_activated': 1, - 'preferred_language': random.sample(['en', 'sw'], 1)[0], - 'is_disabled': False - } - d = prepareLocalFilePath(base_dir, uid) f = open('{}/{}'.format(d, uid + '.json'), 'w') json.dump(o.serialize(), f) f.close() - d = prepareLocalFilePath(ussd_dir, uid) - x = open('{}/{}'.format(d, uid + '_ussd_data.json'), 'w') - json.dump(ussd_data, x) - x.close() - pidx = genPhoneIndex(phone) d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), pidx) f = open('{}/{}'.format(d, pidx), 'w') diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index 79f9d6a1..3b37085a 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -1,43 +1,21 @@ # syntax = docker/dockerfile:1.2 -FROM python:3.8.6-slim-buster as compile-image +#FROM python:3.8.6-slim-buster as compile-image +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45 -RUN apt-get update -RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo - -WORKDIR /root RUN mkdir -vp /usr/local/etc/cic +COPY data-seeding/package.json \ + data-seeding/package-lock.json \ + . + +RUN npm install + COPY data-seeding/requirements.txt . ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt - -# -------------- begin runtime container ---------------- -FROM python:3.8.6-slim-buster as runtime-image - -RUN apt-get update -RUN apt-get install -y --no-install-recommends gnupg libpq-dev -RUN apt-get install -y jq bash iputils-ping socat telnet dnsutils - -COPY --from=compile-image /usr/local/bin/ /usr/local/bin/ -COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/ -COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \ - /usr/local/lib/python3.8/site-packages/ - -WORKDIR root/ - -ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433 -# RUN useradd -u 1001 --create-home grassroots -# RUN adduser grassroots sudo && \ -# echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -# WORKDIR /home/grassroots +ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" +RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL -r requirements.txt COPY data-seeding/ . -# we copied these from the root build container. -# this is dumb though...I guess the compile image should have the same user -# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ - -# USER grassroots - ENTRYPOINT [ ] diff --git a/apps/data-seeding/eth/import_balance.py b/apps/data-seeding/eth/import_balance.py index b3fb1112..2d436a66 100644 --- a/apps/data-seeding/eth/import_balance.py +++ b/apps/data-seeding/eth/import_balance.py @@ -18,25 +18,24 @@ from hexathon import ( add_0x, ) from chainsyncer.backend.memory import MemBackend -from chainsyncer.driver import HeadSyncer +from chainsyncer.driver.head import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( block_latest, - block_by_number, - Block, ) from chainlib.hash import keccak256_string_to_hex from chainlib.eth.address import to_checksum_address from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.tx import TxFactory -from chainlib.jsonrpc import jsonrpc_template +from chainlib.jsonrpc import JSONRPCRequest from chainlib.eth.error import EthException from chainlib.chain import ChainSpec from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from crypto_dev_signer.keystore.dict import DictKeystore from cic_types.models.person import Person from eth_erc20 import ERC20 +from cic_base.eth.syncer import chain_interface logging.basicConfig(level=logging.WARNING) @@ -75,6 +74,7 @@ args_override = { 'CIC_CHAIN_SPEC': getattr(args, 'i'), 'ETH_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), + 'KEYSTORE_FILE_PATH': getattr(args, 'y') } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') @@ -183,27 +183,6 @@ class Handler: # logg.error('key record not found in imports: {}'.format(e).ljust(200)) -#class BlockGetter: -# -# def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec): -# self.conn = conn -# self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id) -# -# -# def get(self, n): -# o = block_by_number(n) -# r = self.conn.do(o) -# b = None -# try: -# b = Block(r) -# except TypeError as e: -# if r == None: -# logg.debug('block not found {}'.format(n)) -# else: -# logg.error('block retrieve error {}'.format(e)) -# return b - - def progress_callback(block_number, tx_index): sys.stdout.write(str(block_number).ljust(200) + "\n") @@ -224,11 +203,13 @@ def main(): data = add_0x(registry_addressof_method) data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex() txf.set_code(tx, data) - - o = jsonrpc_template() + + j = JSONRPCRequest() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) logg.info('found token index address {}'.format(token_index_address)) @@ -242,10 +223,11 @@ def main(): z = h.digest() data += eth_abi.encode_single('bytes32', z).hex() txf.set_code(tx, data) - o = jsonrpc_template() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) try: sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) @@ -299,7 +281,7 @@ def main(): f.close() syncer_backend.set(block_offset, 0) - syncer = HeadSyncer(syncer_backend, block_callback=progress_callback) + syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback) handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/data-seeding/eth/import_users.py b/apps/data-seeding/eth/import_users.py index 3a88b86d..be04b2f2 100644 --- a/apps/data-seeding/eth/import_users.py +++ b/apps/data-seeding/eth/import_users.py @@ -59,6 +59,7 @@ config.process() args_override = { 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'CIC_CHAIN_SPEC': getattr(args, 'i'), + 'KEYSTORE_FILE_PATH': getattr(args, 'y') } config.dict_override(args_override, 'cli') config.add(args.user_dir, '_USERDIR', True) diff --git a/apps/data-seeding/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c b/apps/data-seeding/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c new file mode 100644 index 00000000..2b843ec6 --- /dev/null +++ b/apps/data-seeding/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c @@ -0,0 +1 @@ +{"address":"eb3907ecad74a0013c259d5874ae7f22dcbcc95c","crypto":{"cipher":"aes-128-ctr","ciphertext":"b0f70a8af4071faff2267374e2423cbc7a71012096fd2215866d8de7445cc215","cipherparams":{"iv":"9ac89383a7793226446dcb7e1b45cdf3"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"299f7b5df1d08a0a7b7f9c9eb44fe4798683b78da3513fcf9603fd913ab3336f"},"mac":"6f4ed36c11345a9a48353cd2f93f1f92958c96df15f3112a192bc994250e8d03"},"id":"61a9dd88-24a9-495c-9a51-152bd1bfaa5b","version":3} \ No newline at end of file diff --git a/apps/data-seeding/package-lock.json b/apps/data-seeding/package-lock.json index 336fa188..e2da1566 100644 --- a/apps/data-seeding/package-lock.json +++ b/apps/data-seeding/package-lock.json @@ -1,5 +1,5 @@ { - "name": "scripts", + "name": "data-seeding", "lockfileVersion": 2, "requires": true, "packages": { @@ -49,20 +49,20 @@ } }, "node_modules/@ethereumjs/common": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.2.0.tgz", - "integrity": "sha512-PyQiTG00MJtBRkJmv46ChZL8u2XWxNBeAthznAUIUiefxPAXjbkuiCZOuncgJS34/XkMbNc9zMt/PlgKRBElig==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.3.0.tgz", + "integrity": "sha512-Fmi15MdVptsC85n6NcUXIFiiXCXWEfZNgPWP+OGAQOC6ZtdzoNawtxH/cYpIgEgSuIzfOeX3VKQP/qVI1wISHg==", "dependencies": { "crc-32": "^1.2.0", - "ethereumjs-util": "^7.0.9" + "ethereumjs-util": "^7.0.10" } }, "node_modules/@ethereumjs/tx": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.1.3.tgz", - "integrity": "sha512-DJBu6cbwYtiPTFeCUR8DF5p+PF0jxs+0rALJZiEcTz2tiRPIEkM72GEbrkGuqzENLCzBrJHT43O0DxSYTqeo+g==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.2.0.tgz", + "integrity": "sha512-D3X/XtZ3ldUg34hr99Jvj7NxW3NxVKdUKrwQnEWlAp4CmCQpvYoyn7NF4lk34rHEt7ScS+Agu01pcDHoOcd19A==", "dependencies": { - "@ethereumjs/common": "^2.2.0", + "@ethereumjs/common": "^2.3.0", "ethereumjs-util": "^7.0.10" } }, @@ -75,9 +75,9 @@ } }, "node_modules/@types/node": { - "version": "14.14.41", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.41.tgz", - "integrity": "sha512-dueRKfaJL4RTtSa7bWeTK1M+VH+Gns73oCgzvYfHZywRCoPSd8EkXBL0mZ9unPTveBn+D9phZBaxuzpwjWkW0g==" + "version": "15.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz", + "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw==" }, "node_modules/@types/pbkdf2": { "version": "3.1.0", @@ -115,6 +115,10 @@ "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" } }, "node_modules/ansi-regex": { @@ -134,6 +138,9 @@ }, "engines": { "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/aproba": { @@ -803,9 +810,9 @@ } }, "node_modules/glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -816,6 +823,9 @@ }, "engines": { "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/graceful-fs": { @@ -836,6 +846,7 @@ "version": "5.1.5", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "deprecated": "this library is no longer supported", "optional": true, "dependencies": { "ajv": "^6.12.3", @@ -909,9 +920,9 @@ } }, "node_modules/ignore-walk": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", - "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.4.tgz", + "integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==", "dependencies": { "minimatch": "^3.0.4" } @@ -1037,6 +1048,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/keccak/-/keccak-3.0.1.tgz", "integrity": "sha512-epq90L9jlFWCW7+pQa6JOnKn2Xgl2mtI664seYR6MHskvI9agt7AnDqmAlp9TqU4/caMYbA08Hi5DMZAl5zdkA==", + "hasInstallScript": true, "dependencies": { "node-addon-api": "^2.0.0", "node-gyp-build": "^4.2.0" @@ -1056,21 +1068,21 @@ } }, "node_modules/mime-db": { - "version": "1.47.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", + "version": "1.48.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz", + "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==", "optional": true, "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.30", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "version": "2.1.31", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz", + "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==", "optional": true, "dependencies": { - "mime-db": "1.47.0" + "mime-db": "1.48.0" }, "engines": { "node": ">= 0.6" @@ -1215,6 +1227,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", + "deprecated": "Please upgrade to @mapbox/node-pre-gyp: the non-scoped node-pre-gyp package is deprecated and only the @mapbox scoped package will recieve updates in the future", "dependencies": { "detect-libc": "^1.0.2", "mkdirp": "^0.5.1", @@ -1273,9 +1286,9 @@ } }, "node_modules/npm-bundled": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", - "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz", + "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==", "dependencies": { "npm-normalize-package-bin": "^1.0.1" } @@ -1426,6 +1439,14 @@ }, "engines": { "node": ">= 8.0.0" + }, + "peerDependencies": { + "pg-native": ">=2.0.0" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } } }, "node_modules/pg-connection-string": { @@ -1444,7 +1465,10 @@ "node_modules/pg-pool": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.3.0.tgz", - "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg==" + "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg==", + "peerDependencies": { + "pg": ">=8.0" + } }, "node_modules/pg-protocol": { "version": "1.5.0", @@ -1596,6 +1620,7 @@ "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", "optional": true, "dependencies": { "aws-sign2": "~0.7.0", @@ -1670,7 +1695,21 @@ "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, "node_modules/safer-buffer": { "version": "2.1.2", @@ -1691,6 +1730,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-4.0.2.tgz", "integrity": "sha512-UDar4sKvWAksIlfX3xIaQReADn+WFnHvbVujpcbr+9Sf/69odMwy2MUsz5CKLQgX9nsIyrjuxL2imVyoNHa3fg==", + "hasInstallScript": true, "dependencies": { "elliptic": "^6.5.2", "node-addon-api": "^2.0.0", @@ -1755,18 +1795,27 @@ "version": "5.0.2", "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.0.2.tgz", "integrity": "sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA==", + "hasInstallScript": true, "dependencies": { "node-addon-api": "^3.0.0", "node-pre-gyp": "^0.11.0" }, "optionalDependencies": { "node-gyp": "3.x" + }, + "peerDependencies": { + "node-gyp": "3.x" + }, + "peerDependenciesMeta": { + "node-gyp": { + "optional": true + } } }, "node_modules/sqlite3/node_modules/node-addon-api": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz", - "integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw==" + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", + "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==" }, "node_modules/sshpk": { "version": "1.16.1", @@ -1784,11 +1833,6 @@ "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" }, - "bin": { - "sshpk-conv": "bin/sshpk-conv", - "sshpk-sign": "bin/sshpk-sign", - "sshpk-verify": "bin/sshpk-verify" - }, "engines": { "node": ">=0.10.0" } @@ -1872,12 +1916,16 @@ "node_modules/transit-immutable-js": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/transit-immutable-js/-/transit-immutable-js-0.7.0.tgz", - "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk=" + "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk=", + "peerDependencies": { + "immutable": ">= 3", + "transit-js": ">= 0.8" + } }, "node_modules/transit-js": { - "version": "0.8.867", - "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.867.tgz", - "integrity": "sha512-rOwB4K0z/WZ+E2bV42iN9UV3mvGzmwSv/IpMOKdnFpawPAZT0d1L7f91Y+tZQF7lXSDGk+oln4XyIQXo+pyTGA==", + "version": "0.8.874", + "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.874.tgz", + "integrity": "sha512-IDJJGKRzUbJHmN0P15HBBa05nbKor3r2MmG6aSt0UxXIlJZZKcddTk67/U7WyAeW9Hv/VYI02IqLzolsC4sbPA==", "engines": { "node": ">= 0.10.0" } @@ -1923,6 +1971,7 @@ "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", "bin": { "uuid": "bin/uuid" } @@ -1977,6 +2026,9 @@ }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/wrap-ansi/node_modules/ansi-regex": { @@ -2151,20 +2203,20 @@ } }, "@ethereumjs/common": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.2.0.tgz", - "integrity": "sha512-PyQiTG00MJtBRkJmv46ChZL8u2XWxNBeAthznAUIUiefxPAXjbkuiCZOuncgJS34/XkMbNc9zMt/PlgKRBElig==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.3.0.tgz", + "integrity": "sha512-Fmi15MdVptsC85n6NcUXIFiiXCXWEfZNgPWP+OGAQOC6ZtdzoNawtxH/cYpIgEgSuIzfOeX3VKQP/qVI1wISHg==", "requires": { "crc-32": "^1.2.0", - "ethereumjs-util": "^7.0.9" + "ethereumjs-util": "^7.0.10" } }, "@ethereumjs/tx": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.1.3.tgz", - "integrity": "sha512-DJBu6cbwYtiPTFeCUR8DF5p+PF0jxs+0rALJZiEcTz2tiRPIEkM72GEbrkGuqzENLCzBrJHT43O0DxSYTqeo+g==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.2.0.tgz", + "integrity": "sha512-D3X/XtZ3ldUg34hr99Jvj7NxW3NxVKdUKrwQnEWlAp4CmCQpvYoyn7NF4lk34rHEt7ScS+Agu01pcDHoOcd19A==", "requires": { - "@ethereumjs/common": "^2.2.0", + "@ethereumjs/common": "^2.3.0", "ethereumjs-util": "^7.0.10" } }, @@ -2177,9 +2229,9 @@ } }, "@types/node": { - "version": "14.14.41", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.41.tgz", - "integrity": "sha512-dueRKfaJL4RTtSa7bWeTK1M+VH+Gns73oCgzvYfHZywRCoPSd8EkXBL0mZ9unPTveBn+D9phZBaxuzpwjWkW0g==" + "version": "15.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz", + "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw==" }, "@types/pbkdf2": { "version": "3.1.0", @@ -2822,9 +2874,9 @@ } }, "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -2909,9 +2961,9 @@ } }, "ignore-walk": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", - "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.4.tgz", + "integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==", "requires": { "minimatch": "^3.0.4" } @@ -3037,18 +3089,18 @@ } }, "mime-db": { - "version": "1.47.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", + "version": "1.48.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz", + "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==", "optional": true }, "mime-types": { - "version": "2.1.30", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "version": "2.1.31", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz", + "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==", "optional": true, "requires": { - "mime-db": "1.47.0" + "mime-db": "1.48.0" } }, "minimalistic-assert": { @@ -3209,9 +3261,9 @@ } }, "npm-bundled": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", - "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz", + "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==", "requires": { "npm-normalize-package-bin": "^1.0.1" } @@ -3350,7 +3402,8 @@ "pg-pool": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.3.0.tgz", - "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg==" + "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg==", + "requires": {} }, "pg-protocol": { "version": "1.5.0", @@ -3610,9 +3663,9 @@ }, "dependencies": { "node-addon-api": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz", - "integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw==" + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", + "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==" } } }, @@ -3696,12 +3749,13 @@ "transit-immutable-js": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/transit-immutable-js/-/transit-immutable-js-0.7.0.tgz", - "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk=" + "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk=", + "requires": {} }, "transit-js": { - "version": "0.8.867", - "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.867.tgz", - "integrity": "sha512-rOwB4K0z/WZ+E2bV42iN9UV3mvGzmwSv/IpMOKdnFpawPAZT0d1L7f91Y+tZQF7lXSDGk+oln4XyIQXo+pyTGA==" + "version": "0.8.874", + "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.874.tgz", + "integrity": "sha512-IDJJGKRzUbJHmN0P15HBBa05nbKor3r2MmG6aSt0UxXIlJZZKcddTk67/U7WyAeW9Hv/VYI02IqLzolsC4sbPA==" }, "tunnel-agent": { "version": "0.6.0", diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index 4f2cb559..c178b15f 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,5 +1,5 @@ -cic-base[full_graph]==0.1.2b15 -sarafu-faucet==0.0.3a3 -cic-eth==0.11.0b16 -cic-types==0.1.0a11 -crypto-dev-signer==0.4.14b3 +cic_base[full_graph]==0.1.3a3+build.4aa03607 +sarafu-faucet==0.0.4a1 +cic-eth==0.11.1a1 +cic-types==0.1.0a13 +crypto-dev-signer==0.4.14b6 diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 2e265264..bca6ce95 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -9,6 +9,7 @@ import sys import urllib import urllib.request import uuid +import urllib.parse # external imports import celery @@ -24,7 +25,7 @@ from chainlib.eth.gas import ( ) from chainlib.eth.tx import TxFactory from chainlib.hash import keccak256_string_to_hex -from chainlib.jsonrpc import jsonrpc_template +from chainlib.jsonrpc import JSONRPCRequest from cic_types.models.person import ( Person, generate_metadata_pointer, @@ -72,7 +73,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification') -argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') +argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error') @@ -185,9 +186,9 @@ def send_ussd_request(address, data_dir): } req = urllib.request.Request(config.get('_USSD_PROVIDER')) - data_str = json.dumps(data) - data_bytes = data_str.encode('utf-8') - req.add_header('Content-Type', 'application/json') + urlencoded_data = urllib.parse.urlencode(data) + data_bytes = urlencoded_data.encode('utf-8') + req.add_header('Content-Type', 'application/x-www-form-urlencoded') req.data = data_bytes response = urllib.request.urlopen(req) return response.read().decode('utf-8') @@ -263,9 +264,11 @@ class Verifier: data += eth_abi.encode_single('address', address).hex() tx = self.tx_factory.set_code(tx, data) tx = self.tx_factory.normalize(tx) - o = jsonrpc_template() + j = JSONRPCRequest() + o = j.template() o['method'] = 'eth_call' o['params'].append(tx) + o = j.finalize(o) r = self.conn.do(o) logg.debug('index check for {}: {}'.format(address, r)) n = eth_abi.decode_single('uint256', bytes.fromhex(strip_0x(r))) @@ -388,10 +391,9 @@ class Verifier: def verify_ussd_pins(self, address, balance): response_data = send_ussd_request(address, self.data_dir) - if response_data[:11] != 'CON Balance': + if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio': raise VerifierError(response_data, 'pins') - def verify(self, address, balance, debug_stem=None): for k in active_tests: @@ -429,10 +431,12 @@ def main(): data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex() txf.set_code(tx, data) - o = jsonrpc_template() + j = JSONRPCRequest() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) logg.info('found token index address {}'.format(token_index_address)) @@ -441,10 +445,11 @@ def main(): data += eth_abi.encode_single('bytes32', b'AccountRegistry').hex() txf.set_code(tx, data) - o = jsonrpc_template() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) account_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) logg.info('found account index address {}'.format(account_index_address)) @@ -453,10 +458,11 @@ def main(): data += eth_abi.encode_single('bytes32', b'Faucet').hex() txf.set_code(tx, data) - o = jsonrpc_template() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) faucet_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) logg.info('found faucet {}'.format(faucet_address)) @@ -471,10 +477,11 @@ def main(): z = h.digest() data += eth_abi.encode_single('bytes32', z).hex() txf.set_code(tx, data) - o = jsonrpc_template() + o = j.template() o['method'] = 'eth_call' o['params'].append(txf.normalize(tx)) o['params'].append('latest') + o = j.finalize(o) r = conn.do(o) sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) logg.info('found token address {}'.format(sarafu_token_address)) diff --git a/apps/util/requirements/base_requirement.txt b/apps/util/requirements/base_requirement.txt new file mode 100644 index 00000000..ea67cea4 --- /dev/null +++ b/apps/util/requirements/base_requirement.txt @@ -0,0 +1 @@ +cic-base==0.1.3a3+build.4aa03607 diff --git a/apps/util/requirements/requirements.txt b/apps/util/requirements/requirements.txt new file mode 100644 index 00000000..c5b56441 --- /dev/null +++ b/apps/util/requirements/requirements.txt @@ -0,0 +1 @@ +requirements-magic~=0.0.2 diff --git a/apps/util/requirements/update_base.sh b/apps/util/requirements/update_base.sh new file mode 100644 index 00000000..938867be --- /dev/null +++ b/apps/util/requirements/update_base.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +which pyreq-merge &> /dev/null +if [ $? -gt 0 ]; then + >&2 echo pyreq-merge missing, please install requirements + exit 1 +fi + +t=$(mktemp) +>&2 echo using tmp $t + +repos=(../../cic-cache ../../cic-eth ../../cic-ussd ../../data-seeding ../../cic-notify) + +for r in ${repos[@]}; do + f="$r/requirements.txt" + >&2 echo updating $f + f="$r/test_requirements.txt" + >&2 echo updating $f + pyreq-update $f base_requirement.txt > $t + cp $t $f +done diff --git a/docker-compose.yml b/docker-compose.yml index 71e4e593..8d65dd9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -395,44 +395,6 @@ services: # command: "/root/start_retry.sh -q cic-eth -vv" -# cic-eth-server: -# build: -# context: apps/ -# dockerfile: cic-eth/docker/Dockerfile -# environment: -# CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC -# CELERY_BROKER_URL: $CELERY_BROKER_URL -# CELERY_RESULT_URL: $CELERY_RESULT_URL -# SERVER_PORT: 8000 -# depends_on: -# - eth -# - postgres -# - redis -# ports: -# - ${HTTP_PORT_CIC_ETH:-63314}:8000 -# deploy: -# restart_policy: -# condition: on-failure -# volumes: -# - contract-config:/tmp/cic/config/:ro -# command: -# - /bin/bash -# - -c -# - | -# if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi -# "/usr/local/bin/uwsgi" \ -# --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \ -# --http :80 \ -# --pyargv -vv -## entrypoint: -## - "/usr/local/bin/uwsgi" -## - "--wsgi-file" -## - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py" -## - "--http" -## - ":80" -# # command: "--pyargv -vv" - - cic-notify-tasker: build: @@ -492,7 +454,7 @@ services: restart_policy: condition: on-failure volumes: - - ${LOCAL_VOLUME_DIR:-/tmp/cic}/pgp:/tmp/cic/pgp + - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp # command: "/root/start_server.sh -vv" cic-user-ussd-server: