Merge branch 'philip/ussd-cic-types-meta-tool' into 'master'

Philip/ussd cic types meta tool

See merge request grassrootseconomics/cic-internal-integration!294

apps/cic-ussd/cic_ussd/metadata/__init__.py

@@ -3,7 +3,6 @@
 # external imports
 
 # local imports
-from .base import Metadata
 from .custom import CustomMetadata
 from .person import PersonMetadata
 from .phone import PhonePointerMetadata

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -1,99 +1,30 @@
 # standard imports
-import json
 import logging
-import os
-from typing import Dict, Union
 
-# third-part imports
+# external imports
-from cic_types.models.person import generate_metadata_pointer, Person
+from cic_types.condiments import MetadataPointer
+from cic_types.ext.metadata import MetadataRequestsHandler
+from cic_types.processor import generate_metadata_pointer
 
 # local imports
 from cic_ussd.cache import cache_data, get_cached_data
-from cic_ussd.http.requests import error_handler, make_request
-from cic_ussd.metadata.signer import Signer
 
 logg = logging.getLogger(__file__)
 
 
-class Metadata:
+class UssdMetadataHandler(MetadataRequestsHandler):
-    """
+    def __init__(self, cic_type: MetadataPointer, identifier: bytes):
-    :cvar base_url: The base url or the metadata server.
+        super().__init__(cic_type, identifier)
-    :type base_url: str
-    """
 
-    base_url = None
+    def cache_metadata(self, data: str):
-
+        """
-
+        :param data:
-class MetadataRequestsHandler(Metadata):
+        :type data:
-
+        :return:
-    def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'):
+        :rtype:
-        """"""
+        """
-        self.cic_type = cic_type
+        cache_data(self.metadata_pointer, data)
-        self.engine = engine
+        logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
-        self.headers = {
-            'X-CIC-AUTOMERGE': 'server',
-            'Content-Type': 'application/json'
-        }
-        self.identifier = identifier
-        self.metadata_pointer = generate_metadata_pointer(
-            identifier=self.identifier,
-            cic_type=self.cic_type
-        )
-        if self.base_url:
-            self.url = os.path.join(self.base_url, self.metadata_pointer)
-
-    def create(self, data: Union[Dict, str]):
-        """"""
-        data = json.dumps(data).encode('utf-8')
-        result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
-
-        error_handler(result=result)
-        metadata = result.json()
-        return self.edit(data=metadata)
-
-    def edit(self, data: Union[Dict, str]):
-        """"""
-        cic_meta_signer = Signer()
-        signature = cic_meta_signer.sign_digest(data=data)
-        algorithm = cic_meta_signer.get_operational_key().get('algo')
-        formatted_data = {
-            'm': json.dumps(data),
-            's': {
-                'engine': self.engine,
-                'algo': algorithm,
-                'data': signature,
-                'digest': data.get('digest'),
-            }
-        }
-        formatted_data = json.dumps(formatted_data)
-        result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
-        logg.info(f'signed metadata submission status: {result.status_code}.')
-        error_handler(result=result)
-        try:
-            decoded_identifier = self.identifier.decode("utf-8")
-        except UnicodeDecodeError:
-            decoded_identifier = self.identifier.hex()
-        logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.')
-        return result
-
-    def query(self):
-        """"""
-        result = make_request(method='GET', url=self.url)
-        error_handler(result=result)
-        result_data = result.json()
-        if not isinstance(result_data, dict):
-            raise ValueError(f'Invalid result data object: {result_data}.')
-        if result.status_code == 200:
-            if self.cic_type == ':cic.person':
-                person = Person()
-                person_data = person.deserialize(person_data=result_data)
-                serialized_person_data = person_data.serialize()
-                data = json.dumps(serialized_person_data)
-            else:
-                data = json.dumps(result_data)
-            cache_data(key=self.metadata_pointer, data=data)
-            logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
-        return result_data
 
     def get_cached_metadata(self):
         """"""

apps/cic-ussd/cic_ussd/metadata/custom.py

@@ -1,12 +1,13 @@
 # standard imports
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class CustomMetadata(MetadataRequestsHandler):
+class CustomMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.custom', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.CUSTOM, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -1,12 +1,13 @@
 # standard imports
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PersonMetadata(MetadataRequestsHandler):
+class PersonMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.person', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PERSON, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -2,12 +2,13 @@
 import logging
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PhonePointerMetadata(MetadataRequestsHandler):
+class PhonePointerMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.phone', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PHONE, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -1,13 +1,13 @@
 # standard imports
 
 # external imports
-import celery
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PreferencesMetadata(MetadataRequestsHandler):
+class PreferencesMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.preferences', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PREFERENCES, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/signer.py

@@ -1,60 +0,0 @@
-# standard imports
-import json
-import logging
-from typing import Optional
-from urllib.request import Request, urlopen
-
-# third-party imports
-import gnupg
-
-# local imports
-
-logg = logging.getLogger()
-
-
-class Signer:
-    """
-    :cvar gpg_path:
-    :type gpg_path:
-    :cvar gpg_passphrase:
-    :type gpg_passphrase:
-    :cvar key_file_path:
-    :type key_file_path:
-
-    """
-    gpg_path: str = None
-    gpg_passphrase: str = None
-    key_file_path: str = None
-
-    def __init__(self):
-        self.gpg = gnupg.GPG(gnupghome=self.gpg_path)
-
-        with open(self.key_file_path, 'r') as key_file:
-            self.key_data = key_file.read()
-
-    def get_operational_key(self):
-        """
-        :return:
-        :rtype:
-        """
-        # import key data into keyring
-        self.gpg.import_keys(key_data=self.key_data)
-        gpg_keys = self.gpg.list_keys()
-        key_algorithm = gpg_keys[0].get('algo')
-        key_id = gpg_keys[0].get("keyid")
-        logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}')
-        return gpg_keys[0]
-
-    def sign_digest(self, data: dict):
-        """
-        :param data:
-        :type data:
-        :return:
-        :rtype:
-        """
-        digest = data['digest']
-        key_id = self.get_operational_key().get('keyid')
-        signature = self.gpg.sign(digest, passphrase=self.gpg_passphrase, keyid=key_id)
-        return str(signature)
-
-

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -1,15 +1,17 @@
 # standard imports
+import json
 import logging
 
 # third-party imports
 import celery
+from cic_types.models.person import Person
 
 # local imports
 from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
 from cic_ussd.tasks.base import CriticalMetadataTask
 
 celery_app = celery.current_app
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__file__)
 
 
 @celery_app.task
@@ -22,7 +24,13 @@ def query_person_metadata(blockchain_address: str):
     """
     identifier = bytes.fromhex(blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
-    person_metadata_client.query()
+    response = person_metadata_client.query()
+    data = response.json()
+    person = Person()
+    person_data = person.deserialize(person_data=data)
+    serialized_person_data = person_data.serialize()
+    data = json.dumps(serialized_person_data)
+    person_metadata_client.cache_metadata(data=data)
 
 
 @celery_app.task
@@ -76,6 +84,9 @@ def query_preferences_metadata(blockchain_address: str):
     :type blockchain_address: str | Ox-hex
     """
     identifier = bytes.fromhex(blockchain_address)
-    logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
+    logg.debug(f'retrieving preferences metadata for address: {blockchain_address}.')
-    person_metadata_client = PreferencesMetadata(identifier=identifier)
+    preferences_metadata_client = PreferencesMetadata(identifier=identifier)
-    return person_metadata_client.query()
+    response = preferences_metadata_client.query()
+    data = json.dumps(response.json())
+    preferences_metadata_client.cache_metadata(data)
+    return data

apps/cic-ussd/requirements.txt

@@ -4,10 +4,10 @@ billiard==3.6.4.0
 bcrypt==3.2.0
 celery==4.4.7
 cffi==1.14.6
-cic-eth[services]~=0.12.4a7
+cic-eth[services]~=0.12.4a11
 cic-notify~=0.4.0a10
-cic-types~=0.1.0a15
+cic-types~=0.2.0a3
-confini>=0.4.1a1,<0.5.0
+confini>=0.3.6rc4,<0.5.0
 phonenumbers==8.12.12
 psycopg2==2.8.6
 python-i18n[YAML]==0.3.9

apps/cic-ussd/tests/cic_ussd/metadata/test_base.py

@@ -5,24 +5,25 @@ import os
 # external imports
 import requests_mock
 from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
-from cic_ussd.metadata.base import MetadataRequestsHandler
+from cic_ussd.metadata.base import UssdMetadataHandler
 
 
 # external imports
 
 
-def test_metadata_requests_handler(activated_account,
+def test_ussd_metadata_handler(activated_account,
-                                   init_cache,
+                               init_cache,
-                                   load_config,
+                               load_config,
-                                   person_metadata,
+                               person_metadata,
-                                   setup_metadata_request_handler,
+                               setup_metadata_request_handler,
-                                   setup_metadata_signer):
+                               setup_metadata_signer):
     identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
-    cic_type = ':cic.person'
+    cic_type = MetadataPointer.PERSON
-    metadata_client = MetadataRequestsHandler(cic_type, identifier)
+    metadata_client = UssdMetadataHandler(cic_type, identifier)
     assert metadata_client.cic_type == cic_type
     assert metadata_client.engine == 'pgp'
     assert metadata_client.identifier == identifier
@@ -38,7 +39,5 @@ def test_metadata_requests_handler(activated_account,
         assert result.status_code == 200
         person_metadata.pop('digest')
         request_mocker.register_uri('GET', metadata_client.url, status_code=200, reason='OK', json=person_metadata)
-        result = metadata_client.query()
+        result = metadata_client.query().json()
         assert result == person_metadata
-        cached_metadata = metadata_client.get_cached_metadata()
-        assert json.loads(cached_metadata) == person_metadata

apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -11,8 +11,8 @@ from cic_ussd.metadata import CustomMetadata
 
 
 def test_custom_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.custom'
+    cic_type = MetadataPointer.CUSTOM
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     custom_metadata_client = CustomMetadata(identifier)
     assert custom_metadata_client.cic_type == cic_type
     assert custom_metadata_client.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_person.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -11,8 +11,8 @@ from cic_ussd.metadata import PersonMetadata
 
 
 def test_person_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.person'
+    cic_type = MetadataPointer.PERSON
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     person_metadata_client = PersonMetadata(identifier)
     assert person_metadata_client.cic_type == cic_type
     assert person_metadata_client.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -12,8 +12,8 @@ from cic_ussd.metadata import PhonePointerMetadata
 
 
 def test_phone_pointer_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.phone'
+    cic_type = MetadataPointer.PHONE
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     phone_pointer_metadata = PhonePointerMetadata(identifier)
     assert phone_pointer_metadata.cic_type == cic_type
     assert phone_pointer_metadata.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -11,8 +11,8 @@ from cic_ussd.metadata import PreferencesMetadata
 
 
 def test_preferences_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.preferences'
+    cic_type = MetadataPointer.PREFERENCES
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     preferences_metadata_client = PreferencesMetadata(identifier)
     assert preferences_metadata_client.cic_type == cic_type
     assert preferences_metadata_client.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py

@@ -1,17 +0,0 @@
-# standard imports
-import shutil
-
-# third-party imports
-
-# local imports
-from cic_ussd.metadata.signer import Signer
-
-
-def test_client(load_config, setup_metadata_signer, person_metadata):
-    signer = Signer()
-    gpg = signer.gpg
-    assert signer.key_data is not None
-    gpg.import_keys(key_data=signer.key_data)
-    gpg_keys = gpg.list_keys()
-    assert signer.get_operational_key() == gpg_keys[0]
-    shutil.rmtree(Signer.gpg_path)

apps/cic-ussd/tests/fixtures/metadata.py

@@ -6,33 +6,19 @@ import tempfile
 # external imports
 import pytest
 from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
-from cic_ussd.metadata import Metadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
+from cic_ussd.metadata import PersonMetadata, PhonePointerMetadata, PreferencesMetadata
-from cic_ussd.metadata.signer import Signer
 
 logg = logging.getLogger(__name__)
 
 
-@pytest.fixture(scope='function')
-def setup_metadata_signer(load_config):
-    temp_dir = tempfile.mkdtemp(dir='/tmp')
-    logg.debug(f'Created temp dir: {temp_dir}')
-    Signer.gpg_path = temp_dir
-    Signer.gpg_passphrase = load_config.get('PGP_PASSPHRASE')
-    Signer.key_file_path = os.path.join(load_config.get('PGP_KEYS_PATH'), load_config.get('PGP_PRIVATE_KEYS'))
-
-
-@pytest.fixture(scope='function')
-def setup_metadata_request_handler(load_config):
-    Metadata.base_url = load_config.get('CIC_META_URL')
-
-
 @pytest.fixture(scope='function')
 def account_phone_pointer(activated_account):
     identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
-    return generate_metadata_pointer(identifier, ':cic.phone')
+    return generate_metadata_pointer(identifier, MetadataPointer.PERSON)
 
 
 @pytest.fixture(scope='function')