diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index c7572135..3fa1d970 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 11, 0, - 'beta.1', + 'beta.3', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 73ed26e3..92ce8c4f 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,8 +1,8 @@ -cic-base~=0.1.2a62 +cic-base~=0.1.2a67 celery==4.4.7 crypto-dev-signer~=0.4.14a17 confini~=0.3.6rc3 -cic-eth-registry~=0.5.4a12 +cic-eth-registry~=0.5.4a13 #cic-bancor~=0.0.6 redis==3.5.3 alembic==1.4.2 @@ -18,7 +18,7 @@ moolb~=0.1.1b2 eth-address-index~=0.1.1a7 chainlib~=0.0.2a5 hexathon~=0.0.1a7 -chainsyncer~=0.0.1a21 +chainsyncer[sql]~=0.0.2a1 chainqueue~=0.0.1a7 pysha3==1.0.2 coincurve==15.0.0 diff --git a/apps/cic-ussd/.config/app.ini b/apps/cic-ussd/.config/app.ini index 5df11153..6d37c421 100644 --- a/apps/cic-ussd/.config/app.ini +++ b/apps/cic-ussd/.config/app.ini @@ -11,7 +11,14 @@ REGION=KE [ussd] MENU_FILE=/usr/src/data/ussd_menu.json +user = +pass = [statemachine] STATES=/usr/src/cic-ussd/states/ TRANSITIONS=/usr/src/cic-ussd/transitions/ + +[client] +host = +port = +ssl = diff --git a/apps/cic-ussd/cic_ussd/metadata/base.py b/apps/cic-ussd/cic_ussd/metadata/base.py index 8f163d78..6f2d1ffc 100644 --- a/apps/cic-ussd/cic_ussd/metadata/base.py +++ b/apps/cic-ussd/cic_ussd/metadata/base.py @@ -1,7 +1,126 @@ +# standard imports +import json +import logging +import os +from typing import Dict, Union + +# third-part imports +import requests +from cic_types.models.person import generate_metadata_pointer, Person + +# local imports +from cic_ussd.metadata import make_request +from cic_ussd.metadata.signer import Signer +from cic_ussd.redis import cache_data +from cic_ussd.error import MetadataStoreError + + +logg = logging.getLogger().getChild(__name__) + + class Metadata: """ - :cvar base_url: - :type base_url: + :cvar base_url: The base url or the metadata server. + :type base_url: str """ base_url = None + + +def metadata_http_error_handler(result: requests.Response): + """ This function handles and appropriately raises errors from http requests interacting with the metadata server. + :param result: The response object from a http request. + :type result: requests.Response + """ + status_code = result.status_code + + if 100 <= status_code < 200: + raise MetadataStoreError(f'Informational errors: {status_code}, reason: {result.reason}') + + elif 300 <= status_code < 400: + raise MetadataStoreError(f'Redirect Issues: {status_code}, reason: {result.reason}') + + elif 400 <= status_code < 500: + raise MetadataStoreError(f'Client Error: {status_code}, reason: {result.reason}') + + elif 500 <= status_code < 600: + raise MetadataStoreError(f'Server Error: {status_code}, reason: {result.reason}') + + +class MetadataRequestsHandler(Metadata): + + def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'): + """ + :param cic_type: The salt value with which to hash a specific metadata identifier. + :type cic_type: str + :param engine: Encryption used for sending data to the metadata server. + :type engine: str + :param identifier: A unique element of data in bytes necessary for creating a metadata pointer. + :type identifier: bytes + """ + self.cic_type = cic_type + self.engine = engine + self.headers = { + 'X-CIC-AUTOMERGE': 'server', + 'Content-Type': 'application/json' + } + self.identifier = identifier + self.metadata_pointer = generate_metadata_pointer( + identifier=self.identifier, + cic_type=self.cic_type + ) + if self.base_url: + self.url = os.path.join(self.base_url, self.metadata_pointer) + + def create(self, data: Union[Dict, str]): + """ This function is responsible for posting data to the metadata server with a corresponding metadata pointer + for storage. + :param data: The data to be stored in the metadata server. + :type data: dict|str + """ + data = json.dumps(data).encode('utf-8') + result = make_request(method='POST', url=self.url, data=data, headers=self.headers) + metadata_http_error_handler(result=result) + metadata = result.content + self.edit(data=metadata) + + def edit(self, data: bytes): + """ This function is responsible for editing data in the metadata server corresponding to a unique pointer. + :param data: The data to be edited in the metadata server. + :type data: bytes + """ + cic_meta_signer = Signer() + signature = cic_meta_signer.sign_digest(data=data) + algorithm = cic_meta_signer.get_operational_key().get('algo') + decoded_data = data.decode('utf-8') + formatted_data = { + 'm': data.decode('utf-8'), + 's': { + 'engine': self.engine, + 'algo': algorithm, + 'data': signature, + 'digest': json.loads(data).get('digest'), + } + } + formatted_data = json.dumps(formatted_data).encode('utf-8') + result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers) + logg.info(f'signed metadata submission status: {result.status_code}.') + metadata_http_error_handler(result=result) + try: + decoded_identifier = self.identifier.decode("utf-8") + except UnicodeDecodeError: + decoded_identifier = self.identifier.hex() + logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {decoded_data}.') + + def query(self): + """This function is responsible for querying the metadata server for data corresponding to a unique pointer.""" + result = make_request(method='GET', url=self.url) + metadata_http_error_handler(result=result) + response_data = result.content + data = json.loads(response_data.decode('utf-8')) + if result.status_code == 200 and self.cic_type == 'cic.person': + person = Person() + deserialized_person = person.deserialize(person_data=json.loads(data)) + data = json.dumps(deserialized_person.serialize()) + cache_data(self.metadata_pointer, data=data) + logg.debug(f'caching: {data} with key: {self.metadata_pointer}') diff --git a/apps/cic-ussd/cic_ussd/metadata/person.py b/apps/cic-ussd/cic_ussd/metadata/person.py new file mode 100644 index 00000000..57dec98c --- /dev/null +++ b/apps/cic-ussd/cic_ussd/metadata/person.py @@ -0,0 +1,12 @@ +# standard imports + +# third-party imports + +# local imports +from .base import MetadataRequestsHandler + + +class PersonMetadata(MetadataRequestsHandler): + + def __init__(self, identifier: bytes): + super().__init__(cic_type='cic.person', identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/metadata/phone.py b/apps/cic-ussd/cic_ussd/metadata/phone.py index 4f1d8d51..46d508cb 100644 --- a/apps/cic-ussd/cic_ussd/metadata/phone.py +++ b/apps/cic-ussd/cic_ussd/metadata/phone.py @@ -1,85 +1,13 @@ # standard imports -import json import logging -import os # external imports -import requests -from cic_types.models.person import generate_metadata_pointer -from cic_ussd.metadata import make_request -from cic_ussd.metadata.signer import Signer # local imports -from cic_ussd.error import MetadataStoreError -from .base import Metadata - -logg = logging.getLogger().getChild(__name__) +from .base import MetadataRequestsHandler -class PhonePointerMetadata(Metadata): - - def __init__(self, identifier: bytes, engine: str): - """ - :param identifier: - :type identifier: - """ - - self.headers = { - 'X-CIC-AUTOMERGE': 'server', - 'Content-Type': 'application/json' - } - self.identifier = identifier - self.metadata_pointer = generate_metadata_pointer( - identifier=self.identifier, - cic_type=':cic.phone' - ) - if self.base_url: - self.url = os.path.join(self.base_url, self.metadata_pointer) - - self.engine = engine - - - def create(self, data: str): - try: - data = json.dumps(data).encode('utf-8') - result = make_request(method='POST', url=self.url, data=data, headers=self.headers) - metadata = result.content - logg.debug('data {} meta {} resp {} stats {}'.format(data, metadata, result.reason, result.status_code)) - self.edit(data=metadata, engine=self.engine) - result.raise_for_status() - except requests.exceptions.HTTPError as error: - raise MetadataStoreError(error) - - - def edit(self, data: bytes, engine: str): - """ - :param data: - :type data: - :param engine: - :type engine: - :return: - :rtype: - """ - cic_meta_signer = Signer() - signature = cic_meta_signer.sign_digest(data=data) - algorithm = cic_meta_signer.get_operational_key().get('algo') - decoded_data = data.decode('utf-8') - formatted_data = { - 'm': decoded_data, - 's': { - 'engine': engine, - 'algo': algorithm, - 'data': signature, - 'digest': json.loads(data).get('digest'), - } - } - formatted_data = json.dumps(formatted_data).encode('utf-8') - - try: - result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers) - logg.debug(f'signed phone pointer metadata submission status: {result.status_code}.') - result.raise_for_status() - logg.info('phone {} metadata pointer {} set to {}'.format(self.identifier.decode('utf-8'), self.metadata_pointer, decoded_data)) - except requests.exceptions.HTTPError as error: - raise MetadataStoreError(error) +class PhonePointerMetadata(MetadataRequestsHandler): + def __init__(self, identifier: bytes): + super().__init__(cic_type='cic.msisdn', identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/metadata/user.py b/apps/cic-ussd/cic_ussd/metadata/user.py deleted file mode 100644 index 7565350b..00000000 --- a/apps/cic-ussd/cic_ussd/metadata/user.py +++ /dev/null @@ -1,100 +0,0 @@ -# standard imports -import json -import logging -import os - -# third-party imports -import requests -from cic_types.models.person import generate_metadata_pointer, Person - -# local imports -from cic_ussd.chain import Chain -from cic_ussd.metadata import make_request -from cic_ussd.metadata.signer import Signer -from cic_ussd.redis import cache_data -from cic_ussd.error import MetadataStoreError -from .base import Metadata - -logg = logging.getLogger() - - -class UserMetadata(Metadata): - - - def __init__(self, identifier: bytes): - """ - :param identifier: - :type identifier: - """ - self.headers = { - 'X-CIC-AUTOMERGE': 'server', - 'Content-Type': 'application/json' - } - self.identifier = identifier - self.metadata_pointer = generate_metadata_pointer( - identifier=self.identifier, - cic_type=':cic.person' - ) - if self.base_url: - self.url = os.path.join(self.base_url, self.metadata_pointer) - - def create(self, data: dict): - try: - data = json.dumps(data).encode('utf-8') - result = make_request(method='POST', url=self.url, data=data, headers=self.headers) - metadata = result.content - self.edit(data=metadata, engine='pgp') - logg.info(f'Get sign material response status: {result.status_code}') - result.raise_for_status() - except requests.exceptions.HTTPError as error: - raise MetadataStoreError(error) - - def edit(self, data: bytes, engine: str): - """ - :param data: - :type data: - :param engine: - :type engine: - :return: - :rtype: - """ - cic_meta_signer = Signer() - signature = cic_meta_signer.sign_digest(data=data) - algorithm = cic_meta_signer.get_operational_key().get('algo') - formatted_data = { - 'm': data.decode('utf-8'), - 's': { - 'engine': engine, - 'algo': algorithm, - 'data': signature, - 'digest': json.loads(data).get('digest'), - } - } - formatted_data = json.dumps(formatted_data).encode('utf-8') - - try: - result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers) - logg.debug(f'signed user metadata submission status: {result.status_code}.') - result.raise_for_status() - except requests.exceptions.HTTPError as error: - raise MetadataStoreError(error) - - def query(self): - result = make_request(method='GET', url=self.url) - status = result.status_code - logg.info(f'Get latest data status: {status}') - try: - if status == 200: - response_data = result.content - data = json.loads(response_data.decode()) - - # validate data - person = Person() - deserialized_person = person.deserialize(person_data=json.loads(data)) - - cache_data(key=self.metadata_pointer, data=json.dumps(deserialized_person.serialize())) - elif status == 404: - logg.info('The data is not available and might need to be added.') - result.raise_for_status() - except requests.exceptions.HTTPError as error: - raise MetadataNotFoundError(error) diff --git a/apps/cic-ussd/cic_ussd/processor.py b/apps/cic-ussd/cic_ussd/processor.py index 413ee0e0..061e9fbb 100644 --- a/apps/cic-ussd/cic_ussd/processor.py +++ b/apps/cic-ussd/cic_ussd/processor.py @@ -235,7 +235,7 @@ def process_display_user_metadata(user: User, display_key: str): products=products ) else: - raise MetadataNotFoundError(f'Expected user metadata but found none in cache for key: {user.blockchain_address}') + raise MetadataNotFoundError(f'Expected person metadata but found none in cache for key: {key}') def process_account_statement(user: User, display_key: str, ussd_session: dict): @@ -331,11 +331,11 @@ def process_start_menu(display_key: str, user: User): operational_balance = compute_operational_balance(balances=balances_data) # retrieve and cache account's metadata - s_query_user_metadata = celery.signature( - 'cic_ussd.tasks.metadata.query_user_metadata', + s_query_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.query_person_metadata', [blockchain_address] ) - s_query_user_metadata.apply_async(queue='cic-ussd') + s_query_person_metadata.apply_async(queue='cic-ussd') # retrieve and cache account's statement retrieve_account_statement(blockchain_address=blockchain_address) @@ -389,14 +389,11 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] = identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), cic_type='cic.person' ) - logg.debug(f'METADATA POINTER: {key}') - user_metadata = get_cached_data(key=key) - logg.debug(f'METADATA: {user_metadata}') + person_metadata = get_cached_data(key=key) if last_ussd_session: # get last state last_state = last_ussd_session.state - logg.debug(f'LAST USSD SESSION STATE: {last_state}') # if last state is account_creation_prompt and metadata exists, show start menu if last_state in [ 'account_creation_prompt', @@ -405,7 +402,7 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] = 'exit_invalid_new_pin', 'exit_pin_mismatch', 'exit_invalid_request' - ] and user_metadata is not None: + ] and person_metadata is not None: return UssdMenu.find_by_name(name='start') else: return UssdMenu.find_by_name(name=last_state) diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py b/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py index 74653fd8..7a006a78 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py @@ -97,11 +97,11 @@ def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, User]): recipient = get_user_by_phone_number(phone_number=user_input) blockchain_address = recipient.blockchain_address # retrieve and cache account's metadata - s_query_user_metadata = celery.signature( - 'cic_ussd.tasks.metadata.query_user_metadata', + s_query_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.query_person_metadata', [blockchain_address] ) - s_query_user_metadata.apply_async(queue='cic-ussd') + s_query_person_metadata.apply_async(queue='cic-ussd') def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, User]): diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/user.py b/apps/cic-ussd/cic_ussd/state_machine/logic/user.py index 5849bd77..fd53904d 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/user.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/user.py @@ -164,11 +164,11 @@ def save_complete_user_metadata(state_machine_data: Tuple[str, dict, User]): user_metadata = format_user_metadata(metadata=metadata, user=user) blockchain_address = user.blockchain_address - s_create_user_metadata = celery.signature( - 'cic_ussd.tasks.metadata.create_user_metadata', + s_create_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.create_person_metadata', [blockchain_address, user_metadata] ) - s_create_user_metadata.apply_async(queue='cic-ussd') + s_create_person_metadata.apply_async(queue='cic-ussd') def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, User]): @@ -211,18 +211,18 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, User]): edited_metadata = deserialized_person.serialize() - s_edit_user_metadata = celery.signature( - 'cic_ussd.tasks.metadata.edit_user_metadata', - [blockchain_address, edited_metadata, 'pgp'] + s_edit_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.edit_person_metadata', + [blockchain_address, edited_metadata] ) - s_edit_user_metadata.apply_async(queue='cic-ussd') + s_edit_person_metadata.apply_async(queue='cic-ussd') def get_user_metadata(state_machine_data: Tuple[str, dict, User]): user_input, ussd_session, user = state_machine_data blockchain_address = user.blockchain_address s_get_user_metadata = celery.signature( - 'cic_ussd.tasks.metadata.query_user_metadata', + 'cic_ussd.tasks.metadata.query_person_metadata', [blockchain_address] ) s_get_user_metadata.apply_async(queue='cic-ussd') diff --git a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py index 2c247339..286840b8 100644 --- a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py +++ b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py @@ -57,14 +57,9 @@ def process_account_creation_callback(self, result: str, url: str, status_code: queue = self.request.delivery_info.get('routing_key') s = celery.signature( 'cic_ussd.tasks.metadata.add_phone_pointer', - [ - result, - phone_number, - 'pgp', - ], - queue=queue, + [result, phone_number] ) - s.apply_async() + s.apply_async(queue=queue) # expire cache cache.expire(task_id, timedelta(seconds=180)) diff --git a/apps/cic-ussd/cic_ussd/tasks/metadata.py b/apps/cic-ussd/cic_ussd/tasks/metadata.py index 095613fc..fa6caf86 100644 --- a/apps/cic-ussd/cic_ussd/tasks/metadata.py +++ b/apps/cic-ussd/cic_ussd/tasks/metadata.py @@ -1,14 +1,13 @@ # standard imports -import json import logging -# external imports +# third-party imports import celery from hexathon import strip_0x # local imports from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.metadata.user import UserMetadata +from cic_ussd.metadata.person import PersonMetadata from cic_ussd.metadata.phone import PhonePointerMetadata from cic_ussd.tasks.base import CriticalMetadataTask @@ -17,7 +16,7 @@ logg = logging.getLogger().getChild(__name__) @celery_app.task -def query_user_metadata(blockchain_address: str): +def query_person_metadata(blockchain_address: str): """ :param blockchain_address: :type blockchain_address: @@ -25,12 +24,12 @@ def query_user_metadata(blockchain_address: str): :rtype: """ identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) - user_metadata_client = UserMetadata(identifier=identifier) - user_metadata_client.query() + person_metadata_client = PersonMetadata(identifier=identifier) + person_metadata_client.query() @celery_app.task -def create_user_metadata(blockchain_address: str, data: dict): +def create_person_metadata(blockchain_address: str, data: dict): """ :param blockchain_address: :type blockchain_address: @@ -40,19 +39,20 @@ def create_user_metadata(blockchain_address: str, data: dict): :rtype: """ identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) - user_metadata_client = UserMetadata(identifier=identifier) - user_metadata_client.create(data=data) + person_metadata_client = PersonMetadata(identifier=identifier) + person_metadata_client.create(data=data) @celery_app.task -def edit_user_metadata(blockchain_address: str, data: bytes, engine: str): +def edit_person_metadata(blockchain_address: str, data: bytes): identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) - user_metadata_client = UserMetadata(identifier=identifier) - user_metadata_client.edit(data=data, engine=engine) + person_metadata_client = PersonMetadata(identifier=identifier) + person_metadata_client.edit(data=data) @celery_app.task(bind=True, base=CriticalMetadataTask) -def add_phone_pointer(self, blockchain_address: str, phone: str, engine: str): +def add_phone_pointer(self, blockchain_address: str, phone_number: str): + identifier = phone_number.encode('utf-8') stripped_address = strip_0x(blockchain_address) - phone_metadata_client = PhonePointerMetadata(identifier=phone.encode('utf-8'), engine=engine) + phone_metadata_client = PhonePointerMetadata(identifier=identifier) phone_metadata_client.create(data=stripped_address) diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 7738c9b6..fe3e79e2 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,4 +1,4 @@ -cic_base[full_graph]~=0.1.2a61 -cic-eth~=0.11.0b1 +cic_base[full_graph]~=0.1.2a68 +cic-eth~=0.11.0b3 cic-notify~=0.4.0a3 cic-types~=0.1.0a10 diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py index dc04951e..6c932ce0 100644 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py @@ -9,26 +9,26 @@ from cic_types.models.person import generate_metadata_pointer # local imports from cic_ussd.metadata import blockchain_address_to_metadata_pointer from cic_ussd.metadata.signer import Signer -from cic_ussd.metadata.user import UserMetadata +from cic_ussd.metadata.person import PersonMetadata from cic_ussd.redis import get_cached_data def test_user_metadata(create_activated_user, define_metadata_pointer_url, load_config): - UserMetadata.base_url = load_config.get('CIC_META_URL') + PersonMetadata.base_url = load_config.get('CIC_META_URL') identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - user_metadata_client = UserMetadata(identifier=identifier) + person_metadata_client = PersonMetadata(identifier=identifier) - assert user_metadata_client.url == define_metadata_pointer_url + assert person_metadata_client.url == define_metadata_pointer_url -def test_create_user_metadata(caplog, - create_activated_user, - define_metadata_pointer_url, - load_config, - mock_meta_post_response, - person_metadata): +def test_create_person_metadata(caplog, + create_activated_user, + define_metadata_pointer_url, + load_config, + mock_meta_post_response, + person_metadata): identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - user_metadata_client = UserMetadata(identifier=identifier) + person_metadata_client = PersonMetadata(identifier=identifier) with requests_mock.Mocker(real_http=False) as request_mocker: request_mocker.register_uri( @@ -38,7 +38,7 @@ def test_create_user_metadata(caplog, reason='CREATED', content=json.dumps(mock_meta_post_response).encode('utf-8') ) - user_metadata_client.create(data=person_metadata) + person_metadata_client.create(data=person_metadata) assert 'Get signed material response status: 201' in caplog.text with pytest.raises(RuntimeError) as error: @@ -49,19 +49,19 @@ def test_create_user_metadata(caplog, status_code=400, reason='BAD REQUEST' ) - user_metadata_client.create(data=person_metadata) + person_metadata_client.create(data=person_metadata) assert str(error.value) == f'400 Client Error: BAD REQUEST for url: {define_metadata_pointer_url}' -def test_edit_user_metadata(caplog, - create_activated_user, - define_metadata_pointer_url, - load_config, - person_metadata, - setup_metadata_signer): +def test_edit_person_metadata(caplog, + create_activated_user, + define_metadata_pointer_url, + load_config, + person_metadata, + setup_metadata_signer): Signer.gpg_passphrase = load_config.get('KEYS_PASSPHRASE') identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - user_metadata_client = UserMetadata(identifier=identifier) + person_metadata_client = PersonMetadata(identifier=identifier) with requests_mock.Mocker(real_http=False) as request_mocker: request_mocker.register_uri( 'PUT', @@ -69,7 +69,7 @@ def test_edit_user_metadata(caplog, status_code=200, reason='OK' ) - user_metadata_client.edit(data=person_metadata, engine='pgp') + person_metadata_client.edit(data=person_metadata) assert 'Signed content submission status: 200' in caplog.text with pytest.raises(RuntimeError) as error: @@ -80,7 +80,7 @@ def test_edit_user_metadata(caplog, status_code=400, reason='BAD REQUEST' ) - user_metadata_client.edit(data=person_metadata, engine='pgp') + person_metadata_client.edit(data=person_metadata) assert str(error.value) == f'400 Client Error: BAD REQUEST for url: {define_metadata_pointer_url}' @@ -92,7 +92,7 @@ def test_get_user_metadata(caplog, person_metadata, setup_metadata_signer): identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - user_metadata_client = UserMetadata(identifier=identifier) + person_metadata_client = PersonMetadata(identifier=identifier) with requests_mock.Mocker(real_http=False) as request_mocker: request_mocker.register_uri( 'GET', @@ -101,7 +101,7 @@ def test_get_user_metadata(caplog, content=json.dumps(person_metadata).encode('utf-8'), reason='OK' ) - user_metadata_client.query() + person_metadata_client.query() assert 'Get latest data status: 200' in caplog.text key = generate_metadata_pointer( identifier=identifier, @@ -118,6 +118,6 @@ def test_get_user_metadata(caplog, status_code=404, reason='NOT FOUND' ) - user_metadata_client.query() + person_metadata_client.query() assert 'The data is not available and might need to be added.' in caplog.text assert str(error.value) == f'400 Client Error: NOT FOUND for url: {define_metadata_pointer_url}' diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py index 6ac9f39b..97f1327f 100644 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py @@ -15,7 +15,7 @@ from cic_ussd.state_machine.logic.user import ( get_user_metadata, save_complete_user_metadata, process_gender_user_input, - save_profile_attribute_to_session_data, + save_metadata_attribute_to_session_data, update_account_status_to_active) @@ -41,14 +41,14 @@ def test_update_account_status_to_active(create_pending_user, create_in_db_ussd_ ("enter_location", "location", "Kangemi", "Kangemi"), ("enter_products", "products", "Mandazi", "Mandazi"), ]) -def test_save_save_profile_attribute_to_session_data(current_state, - expected_key, - expected_result, - user_input, - celery_session_worker, - create_activated_user, - create_in_db_ussd_session, - create_in_redis_ussd_session): +def test_save_metadata_attribute_to_session_data(current_state, + expected_key, + expected_result, + user_input, + celery_session_worker, + create_activated_user, + create_in_db_ussd_session, + create_in_redis_ussd_session): create_in_db_ussd_session.state = current_state serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() state_machine_data = (user_input, serialized_in_db_ussd_session, create_activated_user) @@ -56,7 +56,7 @@ def test_save_save_profile_attribute_to_session_data(current_state, in_memory_ussd_session = json.loads(in_memory_ussd_session) assert in_memory_ussd_session.get('session_data') == {} serialized_in_db_ussd_session['state'] = current_state - save_profile_attribute_to_session_data(state_machine_data=state_machine_data) + save_metadata_attribute_to_session_data(state_machine_data=state_machine_data) in_memory_ussd_session = InMemoryStore.cache.get('AT974186') in_memory_ussd_session = json.loads(in_memory_ussd_session) @@ -87,18 +87,18 @@ def test_format_user_metadata(create_activated_user, def test_save_complete_user_metadata(celery_session_worker, - complete_user_metadata, - create_activated_user, - create_in_redis_ussd_session, - mocker, - setup_chain_spec, - ussd_session_data): + complete_user_metadata, + create_activated_user, + create_in_redis_ussd_session, + mocker, + setup_chain_spec, + ussd_session_data): ussd_session = create_in_redis_ussd_session.get(ussd_session_data.get('external_session_id')) ussd_session = json.loads(ussd_session) ussd_session['session_data'] = complete_user_metadata user_metadata = format_user_metadata(metadata=ussd_session.get('session_data'), user=create_activated_user) state_machine_data = ('', ussd_session, create_activated_user) - mocked_create_metadata_task = mocker.patch('cic_ussd.tasks.metadata.create_user_metadata.apply_async') + mocked_create_metadata_task = mocker.patch('cic_ussd.tasks.metadata.create_person_metadata.apply_async') save_complete_user_metadata(state_machine_data=state_machine_data) mocked_create_metadata_task.assert_called_with( (user_metadata, create_activated_user.blockchain_address), @@ -127,7 +127,7 @@ def test_edit_user_metadata_attribute(celery_session_worker, } state_machine_data = ('', ussd_session, create_activated_user) - mocked_edit_metadata = mocker.patch('cic_ussd.tasks.metadata.edit_user_metadata.apply_async') + mocked_edit_metadata = mocker.patch('cic_ussd.tasks.metadata.edit_person_metadata.apply_async') edit_user_metadata_attribute(state_machine_data=state_machine_data) person_metadata['location']['area_name'] = 'nairobi' mocked_edit_metadata.assert_called_with( @@ -146,7 +146,7 @@ def test_get_user_metadata_attribute(celery_session_worker, ussd_session = json.loads(ussd_session) state_machine_data = ('', ussd_session, create_activated_user) - mocked_get_metadata = mocker.patch('cic_ussd.tasks.metadata.query_user_metadata.apply_async') + mocked_get_metadata = mocker.patch('cic_ussd.tasks.metadata.query_person_metadata.apply_async') get_user_metadata(state_machine_data=state_machine_data) mocked_get_metadata.assert_called_with( (create_activated_user.blockchain_address,), diff --git a/apps/cic-ussd/tests/fixtures/config.py b/apps/cic-ussd/tests/fixtures/config.py index ba03f4d5..9dd18c37 100644 --- a/apps/cic-ussd/tests/fixtures/config.py +++ b/apps/cic-ussd/tests/fixtures/config.py @@ -18,7 +18,7 @@ from cic_ussd.files.local_files import create_local_file_data_stores, json_file_ from cic_ussd.menu.ussd_menu import UssdMenu from cic_ussd.metadata import blockchain_address_to_metadata_pointer from cic_ussd.metadata.signer import Signer -from cic_ussd.metadata.user import UserMetadata +from cic_ussd.metadata.person import PersonMetadata from cic_ussd.state_machine import UssdStateMachine @@ -121,9 +121,9 @@ def setup_metadata_signer(load_config): @pytest.fixture(scope='function') def define_metadata_pointer_url(load_config, create_activated_user): identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - UserMetadata.base_url = load_config.get('CIC_META_URL') - user_metadata_client = UserMetadata(identifier=identifier) - return user_metadata_client.url + PersonMetadata.base_url = load_config.get('CIC_META_URL') + person_metadata_client = PersonMetadata(identifier=identifier) + return person_metadata_client.url @pytest.fixture(scope='function') diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 8a583ea1..dd2cf26c 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -57,7 +57,7 @@ WORKDIR /home/grassroots USER grassroots ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 -ARG cic_base_version=0.1.2a62 +ARG cic_base_version=0.1.2a67 ARG cic_eth_version=0.11.0b1 ARG sarafu_faucet_version=0.0.2a19 ARG cic_contracts_version=0.0.2a2 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index 0ef9da0e..2d81ae64 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -72,12 +72,13 @@ if [[ -n "${ETH_PROVIDER}" ]]; then # Sarafu faucet contract >&2 echo "deploy token faucet contract" DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -v --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS` - >&2 echo "set token faucet amount" - sarafu-faucet-set -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_FAUCET_ADDRESS $faucet_amount eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS >&2 echo "set faucet as token minter" giftable-token-minter -w -y $keystore_file -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS + >&2 echo "set token faucet amount" + sarafu-faucet-set -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_FAUCET_ADDRESS $faucet_amount + else echo "\$ETH_PROVIDER not set!" diff --git a/apps/contract-migration/scripts/cic_eth/import_balance.py b/apps/contract-migration/scripts/cic_eth/import_balance.py index 618f2421..2067d197 100644 --- a/apps/contract-migration/scripts/cic_eth/import_balance.py +++ b/apps/contract-migration/scripts/cic_eth/import_balance.py @@ -204,8 +204,8 @@ class Handler: # return b -def progress_callback(block_number, tx_index, s): - sys.stdout.write(str(s).ljust(200) + "\n") +def progress_callback(block_number, tx_index): + sys.stdout.write(str(block_number).ljust(200) + "\n") @@ -299,7 +299,7 @@ def main(): f.close() syncer_backend.set(block_offset, 0) - syncer = HeadSyncer(syncer_backend, progress_callback=progress_callback) + syncer = HeadSyncer(syncer_backend, block_callback=progress_callback) handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/contract-migration/scripts/config/app.ini b/apps/contract-migration/scripts/config/app.ini new file mode 100644 index 00000000..6d37c421 --- /dev/null +++ b/apps/contract-migration/scripts/config/app.ini @@ -0,0 +1,24 @@ +[app] +ALLOWED_IP=0.0.0.0/0 +LOCALE_FALLBACK=en +LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/ +MAX_BODY_LENGTH=1024 +PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I= +SERVICE_CODE=*483*46# + +[phone_number] +REGION=KE + +[ussd] +MENU_FILE=/usr/src/data/ussd_menu.json +user = +pass = + +[statemachine] +STATES=/usr/src/cic-ussd/states/ +TRANSITIONS=/usr/src/cic-ussd/transitions/ + +[client] +host = +port = +ssl = diff --git a/apps/contract-migration/scripts/eth/import_balance.py b/apps/contract-migration/scripts/eth/import_balance.py index 618f2421..2067d197 100644 --- a/apps/contract-migration/scripts/eth/import_balance.py +++ b/apps/contract-migration/scripts/eth/import_balance.py @@ -204,8 +204,8 @@ class Handler: # return b -def progress_callback(block_number, tx_index, s): - sys.stdout.write(str(s).ljust(200) + "\n") +def progress_callback(block_number, tx_index): + sys.stdout.write(str(block_number).ljust(200) + "\n") @@ -299,7 +299,7 @@ def main(): f.close() syncer_backend.set(block_offset, 0) - syncer = HeadSyncer(syncer_backend, progress_callback=progress_callback) + syncer = HeadSyncer(syncer_backend, block_callback=progress_callback) handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/contract-migration/scripts/package.json b/apps/contract-migration/scripts/package.json new file mode 100644 index 00000000..b5f273d2 --- /dev/null +++ b/apps/contract-migration/scripts/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "cic-client-meta": "^0.0.7-alpha.6", + "vcard-parser": "^1.0.0" + } +} diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index 5a62a439..d0ddd1e6 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,5 +1,5 @@ -cic-base[full_graph]==0.1.2a61 -sarafu-faucet==0.0.2a17 -cic-eth==0.11.0b1 +cic-base[full_graph]==0.1.2a67 +sarafu-faucet==0.0.2a20 +cic-eth==0.11.0b3 cic-types==0.1.0a10 -crypto-dev-signer==0.4.14a17 +crypto-dev-signer==0.4.14b1 diff --git a/apps/contract-migration/scripts/verify.py b/apps/contract-migration/scripts/verify.py index e4f51b11..14272ddd 100644 --- a/apps/contract-migration/scripts/verify.py +++ b/apps/contract-migration/scripts/verify.py @@ -343,7 +343,13 @@ class Verifier: address_recovered = json.loads(b.decode('utf-8')) address_recovered = address_recovered.replace('"', '') - if strip_0x(address) != strip_0x(address_recovered): + try: + address = strip_0x(address) + address_recovered = strip_0x(address_recovered) + except ValueError: + raise VerifierError(address_recovered, 'metadata (phone) address {} address recovered {}'.format(address, address_recovered)) + + if address != address_recovered: raise VerifierError(address_recovered, 'metadata (phone)')