kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'philip/ussd-cic-types-meta-tool' into 'master'

Browse Source 
Philip/ussd cic types meta tool

See merge request grassrootseconomics/cic-internal-integration!294
This commit is contained in:
Louis Holbrook 2021-10-15 11:21:42 +00:00
commit b6a4bab1c8
16 changed files with 77 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/cic-ussd/cic_ussd/metadata/__init__.py
View File

@ -3,7 +3,6 @@
# external imports
# local imports
from .base import Metadata
from .custom import CustomMetadata
from .person import PersonMetadata
from .phone import PhonePointerMetadata

101
apps/cic-ussd/cic_ussd/metadata/base.py
View File

@ -1,99 +1,30 @@
# standard imports
import json
import logging
import os
from typing import Dict, Union
# third-part imports
from cic_types.models.person import generate_metadata_pointer, Person
# external imports
from cic_types.condiments import MetadataPointer
from cic_types.ext.metadata import MetadataRequestsHandler
from cic_types.processor import generate_metadata_pointer
# local imports
from cic_ussd.cache import cache_data, get_cached_data
from cic_ussd.http.requests import error_handler, make_request
from cic_ussd.metadata.signer import Signer
logg = logging.getLogger(__file__)
class Metadata:
"""
:cvar base_url: The base url or the metadata server.
:type base_url: str
"""
class UssdMetadataHandler(MetadataRequestsHandler):
def __init__(self, cic_type: MetadataPointer, identifier: bytes):
super().__init__(cic_type, identifier)
base_url = None
class MetadataRequestsHandler(Metadata):
def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'):
""""""
self.cic_type = cic_type
self.engine = engine
self.headers = {
'X-CIC-AUTOMERGE': 'server',
'Content-Type': 'application/json'
}
self.identifier = identifier
self.metadata_pointer = generate_metadata_pointer(
identifier=self.identifier,
cic_type=self.cic_type
)
if self.base_url:
self.url = os.path.join(self.base_url, self.metadata_pointer)
def create(self, data: Union[Dict, str]):
""""""
data = json.dumps(data).encode('utf-8')
result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
error_handler(result=result)
metadata = result.json()
return self.edit(data=metadata)
def edit(self, data: Union[Dict, str]):
""""""
cic_meta_signer = Signer()
signature = cic_meta_signer.sign_digest(data=data)
algorithm = cic_meta_signer.get_operational_key().get('algo')
formatted_data = {
'm': json.dumps(data),
's': {
'engine': self.engine,
'algo': algorithm,
'data': signature,
'digest': data.get('digest'),
}
}
formatted_data = json.dumps(formatted_data)
result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
logg.info(f'signed metadata submission status: {result.status_code}.')
error_handler(result=result)
try:
decoded_identifier = self.identifier.decode("utf-8")
except UnicodeDecodeError:
decoded_identifier = self.identifier.hex()
logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.')
return result
def query(self):
""""""
result = make_request(method='GET', url=self.url)
error_handler(result=result)
result_data = result.json()
if not isinstance(result_data, dict):
raise ValueError(f'Invalid result data object: {result_data}.')
if result.status_code == 200:
if self.cic_type == ':cic.person':
person = Person()
person_data = person.deserialize(person_data=result_data)
serialized_person_data = person_data.serialize()
data = json.dumps(serialized_person_data)
else:
data = json.dumps(result_data)
cache_data(key=self.metadata_pointer, data=data)
logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
return result_data
def cache_metadata(self, data: str):
"""
:param data:
:type data:
:return:
:rtype:
"""
cache_data(self.metadata_pointer, data)
logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
def get_cached_metadata(self):
""""""

7
apps/cic-ussd/cic_ussd/metadata/custom.py
View File

@ -1,12 +1,13 @@
# standard imports
# external imports
from cic_types.condiments import MetadataPointer
# local imports
from .base import MetadataRequestsHandler
from .base import UssdMetadataHandler
class CustomMetadata(MetadataRequestsHandler):
class CustomMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.custom', identifier=identifier)
super().__init__(cic_type=MetadataPointer.CUSTOM, identifier=identifier)

7
apps/cic-ussd/cic_ussd/metadata/person.py
View File

@ -1,12 +1,13 @@
# standard imports
# external imports
from cic_types.condiments import MetadataPointer
# local imports
from .base import MetadataRequestsHandler
from .base import UssdMetadataHandler
class PersonMetadata(MetadataRequestsHandler):
class PersonMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.person', identifier=identifier)
super().__init__(cic_type=MetadataPointer.PERSON, identifier=identifier)

7
apps/cic-ussd/cic_ussd/metadata/phone.py
View File

@ -2,12 +2,13 @@
import logging
# external imports
from cic_types.condiments import MetadataPointer
# local imports
from .base import MetadataRequestsHandler
from .base import UssdMetadataHandler
class PhonePointerMetadata(MetadataRequestsHandler):
class PhonePointerMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.phone', identifier=identifier)
super().__init__(cic_type=MetadataPointer.PHONE, identifier=identifier)

8
apps/cic-ussd/cic_ussd/metadata/preferences.py
View File

@ -1,13 +1,13 @@
# standard imports
# external imports
import celery
from cic_types.condiments import MetadataPointer
# local imports
from .base import MetadataRequestsHandler
from .base import UssdMetadataHandler
class PreferencesMetadata(MetadataRequestsHandler):
class PreferencesMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.preferences', identifier=identifier)
super().__init__(cic_type=MetadataPointer.PREFERENCES, identifier=identifier)

60
apps/cic-ussd/cic_ussd/metadata/signer.py
View File

@ -1,60 +0,0 @@
# standard imports
import json
import logging
from typing import Optional
from urllib.request import Request, urlopen
# third-party imports
import gnupg
# local imports
logg = logging.getLogger()
class Signer:
"""
:cvar gpg_path:
:type gpg_path:
:cvar gpg_passphrase:
:type gpg_passphrase:
:cvar key_file_path:
:type key_file_path:
"""
gpg_path: str = None
gpg_passphrase: str = None
key_file_path: str = None
def __init__(self):
self.gpg = gnupg.GPG(gnupghome=self.gpg_path)
with open(self.key_file_path, 'r') as key_file:
self.key_data = key_file.read()
def get_operational_key(self):
"""
:return:
:rtype:
"""
# import key data into keyring
self.gpg.import_keys(key_data=self.key_data)
gpg_keys = self.gpg.list_keys()
key_algorithm = gpg_keys[0].get('algo')
key_id = gpg_keys[0].get("keyid")
logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}')
return gpg_keys[0]
def sign_digest(self, data: dict):
"""
:param data:
:type data:
:return:
:rtype:
"""
digest = data['digest']
key_id = self.get_operational_key().get('keyid')
signature = self.gpg.sign(digest, passphrase=self.gpg_passphrase, keyid=key_id)
return str(signature)

21
apps/cic-ussd/cic_ussd/tasks/metadata.py
View File

@ -1,15 +1,17 @@
# standard imports
import json
import logging
# third-party imports
import celery
from cic_types.models.person import Person
# local imports
from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
from cic_ussd.tasks.base import CriticalMetadataTask
celery_app = celery.current_app
logg = logging.getLogger().getChild(__name__)
logg = logging.getLogger(__file__)
@celery_app.task
@ -22,7 +24,13 @@ def query_person_metadata(blockchain_address: str):
"""
identifier = bytes.fromhex(blockchain_address)
person_metadata_client = PersonMetadata(identifier=identifier)
person_metadata_client.query()
response = person_metadata_client.query()
data = response.json()
person = Person()
person_data = person.deserialize(person_data=data)
serialized_person_data = person_data.serialize()
data = json.dumps(serialized_person_data)
person_metadata_client.cache_metadata(data=data)
@celery_app.task
@ -76,6 +84,9 @@ def query_preferences_metadata(blockchain_address: str):
:type blockchain_address: str | Ox-hex
"""
identifier = bytes.fromhex(blockchain_address)
logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
person_metadata_client = PreferencesMetadata(identifier=identifier)
return person_metadata_client.query()
logg.debug(f'retrieving preferences metadata for address: {blockchain_address}.')
preferences_metadata_client = PreferencesMetadata(identifier=identifier)
response = preferences_metadata_client.query()
data = json.dumps(response.json())
preferences_metadata_client.cache_metadata(data)
return data

6
apps/cic-ussd/requirements.txt
View File

@ -4,10 +4,10 @@ billiard==3.6.4.0
bcrypt==3.2.0
celery==4.4.7
cffi==1.14.6
cic-eth[services]~=0.12.4a7
cic-eth[services]~=0.12.4a11
cic-notify~=0.4.0a10
cic-types~=0.1.0a15
confini>=0.4.1a1,<0.5.0
cic-types~=0.2.0a3
confini>=0.3.6rc4,<0.5.0
phonenumbers==8.12.12
psycopg2==2.8.6
python-i18n[YAML]==0.3.9

23
apps/cic-ussd/tests/cic_ussd/metadata/test_base.py
View File

@ -5,24 +5,25 @@ import os
# external imports
import requests_mock
from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
# local imports
from cic_ussd.metadata.base import MetadataRequestsHandler
from cic_ussd.metadata.base import UssdMetadataHandler
# external imports
def test_metadata_requests_handler(activated_account,
init_cache,
load_config,
person_metadata,
setup_metadata_request_handler,
setup_metadata_signer):
def test_ussd_metadata_handler(activated_account,
init_cache,
load_config,
person_metadata,
setup_metadata_request_handler,
setup_metadata_signer):
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
cic_type = ':cic.person'
metadata_client = MetadataRequestsHandler(cic_type, identifier)
cic_type = MetadataPointer.PERSON
metadata_client = UssdMetadataHandler(cic_type, identifier)
assert metadata_client.cic_type == cic_type
assert metadata_client.engine == 'pgp'
assert metadata_client.identifier == identifier
@ -38,7 +39,5 @@ def test_metadata_requests_handler(activated_account,
assert result.status_code == 200
person_metadata.pop('digest')
request_mocker.register_uri('GET', metadata_client.url, status_code=200, reason='OK', json=person_metadata)
result = metadata_client.query()
result = metadata_client.query().json()
assert result == person_metadata
cached_metadata = metadata_client.get_cached_metadata()
assert json.loads(cached_metadata) == person_metadata

6
apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py
View File

@ -1,7 +1,7 @@
# standard imports
import os
# external imports
from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
# local imports
@ -11,8 +11,8 @@ from cic_ussd.metadata import CustomMetadata
def test_custom_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.custom'
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
cic_type = MetadataPointer.CUSTOM
identifier = bytes.fromhex(activated_account.blockchain_address)
custom_metadata_client = CustomMetadata(identifier)
assert custom_metadata_client.cic_type == cic_type
assert custom_metadata_client.engine == 'pgp'

6
apps/cic-ussd/tests/cic_ussd/metadata/test_person.py
View File

@ -1,7 +1,7 @@
# standard imports
import os
# external imports
from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
# local imports
@ -11,8 +11,8 @@ from cic_ussd.metadata import PersonMetadata
def test_person_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.person'
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
cic_type = MetadataPointer.PERSON
identifier = bytes.fromhex(activated_account.blockchain_address)
person_metadata_client = PersonMetadata(identifier)
assert person_metadata_client.cic_type == cic_type
assert person_metadata_client.engine == 'pgp'

6
apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py
View File

@ -1,7 +1,7 @@
# standard imports
import os
# external imports
from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
# local imports
@ -12,8 +12,8 @@ from cic_ussd.metadata import PhonePointerMetadata
def test_phone_pointer_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.phone'
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
cic_type = MetadataPointer.PHONE
identifier = bytes.fromhex(activated_account.blockchain_address)
phone_pointer_metadata = PhonePointerMetadata(identifier)
assert phone_pointer_metadata.cic_type == cic_type
assert phone_pointer_metadata.engine == 'pgp'

6
apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py
View File

@ -1,7 +1,7 @@
# standard imports
import os
# external imports
from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
# local imports
@ -11,8 +11,8 @@ from cic_ussd.metadata import PreferencesMetadata
def test_preferences_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.preferences'
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
cic_type = MetadataPointer.PREFERENCES
identifier = bytes.fromhex(activated_account.blockchain_address)
preferences_metadata_client = PreferencesMetadata(identifier)
assert preferences_metadata_client.cic_type == cic_type
assert preferences_metadata_client.engine == 'pgp'

17
apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py
View File

@ -1,17 +0,0 @@
# standard imports
import shutil
# third-party imports
# local imports
from cic_ussd.metadata.signer import Signer
def test_client(load_config, setup_metadata_signer, person_metadata):
signer = Signer()
gpg = signer.gpg
assert signer.key_data is not None
gpg.import_keys(key_data=signer.key_data)
gpg_keys = gpg.list_keys()
assert signer.get_operational_key() == gpg_keys[0]
shutil.rmtree(Signer.gpg_path)

20
apps/cic-ussd/tests/fixtures/metadata.py vendored
View File

@ -6,33 +6,19 @@ import tempfile
# external imports
import pytest
from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
# local imports
from cic_ussd.metadata import Metadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
from cic_ussd.metadata.signer import Signer
from cic_ussd.metadata import PersonMetadata, PhonePointerMetadata, PreferencesMetadata
logg = logging.getLogger(__name__)
@pytest.fixture(scope='function')
def setup_metadata_signer(load_config):
temp_dir = tempfile.mkdtemp(dir='/tmp')
logg.debug(f'Created temp dir: {temp_dir}')
Signer.gpg_path = temp_dir
Signer.gpg_passphrase = load_config.get('PGP_PASSPHRASE')
Signer.key_file_path = os.path.join(load_config.get('PGP_KEYS_PATH'), load_config.get('PGP_PRIVATE_KEYS'))
@pytest.fixture(scope='function')
def setup_metadata_request_handler(load_config):
Metadata.base_url = load_config.get('CIC_META_URL')
@pytest.fixture(scope='function')
def account_phone_pointer(activated_account):
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
return generate_metadata_pointer(identifier, ':cic.phone')
return generate_metadata_pointer(identifier, MetadataPointer.PERSON)
@pytest.fixture(scope='function')