Merge remote-tracking branch 'origin/master' into lash/split-migration

This commit is contained in:
nolash 2021-10-15 22:19:01 +02:00
commit 4634ac41df
34 changed files with 614 additions and 263 deletions

View File

@ -1,4 +1,5 @@
celery==4.4.7 celery==4.4.7
erc20-demurrage-token~=0.0.3a1 erc20-demurrage-token~=0.0.5a3
cic-eth-registry>=0.6.1a2,<0.7.0 cic-eth-registry~=0.6.1a5
cic-eth[services]~=0.12.4a8 chainlib~=0.0.9rc3
cic_eth~=0.12.4a9

View File

@ -1,21 +1,2 @@
# standard imports
import logging
# external imports
import celery
# local imports # local imports
from cic_eth.task import BaseTask from cic_eth.eth.erc20 import default_token
celery_app = celery.current_app
logg = logging.getLogger()
@celery_app.task(bind=True, base=BaseTask)
def default_token(self):
return {
'symbol': self.default_token_symbol,
'address': self.default_token_address,
'name': self.default_token_name,
'decimals': self.default_token_decimals,
}

View File

@ -17,15 +17,50 @@ from cic_eth.enum import LockEnum
app = celery.current_app app = celery.current_app
logg = logging.getLogger(__name__) #logg = logging.getLogger(__name__)
logg = logging.getLogger()
class Api(ApiBase): class Api(ApiBase):
@staticmethod
def to_v_list(v, n):
"""Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
:param v: Arguments
:type v: str or list
:param n: Number of elements to generate arguments for
:type n: int
:rtype: list
:returns: list of assembled arguments
"""
if isinstance(v, str):
vv = v
v = []
for i in range(n):
v.append([vv])
elif not isinstance(v, list):
raise ValueError('argument must be single string, or list or strings or lists')
else:
if len(v) != n:
raise ValueError('v argument count must match integer n')
for i in range(n):
if isinstance(v[i], str):
v[i] = [v[i]]
elif not isinstance(v, list):
raise ValueError('proof argument must be single string, or list or strings or lists')
return v
def default_token(self): def default_token(self):
"""Retrieves the default fallback token of the custodial network.
:returns: uuid of root task
:rtype: celery.Task
"""
s_token = celery.signature( s_token = celery.signature(
'cic_eth.admin.token.default_token', 'cic_eth.eth.erc20.default_token',
[], [],
queue=self.queue, queue=self.queue,
) )
@ -35,6 +70,97 @@ class Api(ApiBase):
return s_token.apply_async() return s_token.apply_async()
def token(self, token_symbol, proof=None):
"""Single-token alias for tokens method.
See tokens method for details.
:param token_symbol: Token symbol to look up
:type token_symbol: str
:param proof: Proofs to add to signature verification for the token
:type proof: str or list
:returns: uuid of root task
:rtype: celery.Task
"""
if not isinstance(token_symbol, str):
raise ValueError('token symbol must be string')
return self.tokens([token_symbol], proof=proof)
def tokens(self, token_symbols, proof=None):
"""Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
Token symbols are always strings, and should be specified using uppercase letters.
If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
The proof argument may be specified in a number of ways:
- as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
- as a single string, where the same proof is used for each token lookup
- as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
- as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
This method is not intended to be used synchronously. Do so at your peril.
:param token_symbols: Token symbol strings to look up
:type token_symbol: list
:param proof: Proof(s) to verify tokens against
:type proof: None, str or list
:returns: uuid of root task
:rtype: celery.Task
"""
if not isinstance(token_symbols, list):
raise ValueError('token symbols argument must be list')
if proof == None:
logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
proof = ''
logg.debug('proof is {}'.format(proof))
l = len(token_symbols)
if len(proof) == 0:
l = 0
proof = Api.to_v_list(proof, l)
chain_spec_dict = self.chain_spec.asdict()
s_token_resolve = celery.signature(
'cic_eth.eth.erc20.resolve_tokens_by_symbol',
[
token_symbols,
chain_spec_dict,
],
queue=self.queue,
)
s_token_info = celery.signature(
'cic_eth.eth.erc20.token_info',
[
chain_spec_dict,
proof,
],
queue=self.queue,
)
s_token_verify = celery.signature(
'cic_eth.eth.erc20.verify_token_info',
[
chain_spec_dict,
self.callback_success,
self.callback_error,
],
queue=self.queue,
)
s_token_info.link(s_token_verify)
s_token_resolve.link(s_token_info)
return s_token_resolve.apply_async()
# def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol): # def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
# """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed. # """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
# #

View File

@ -1,7 +1,10 @@
import logging
import celery import celery
celery_app = celery.current_app celery_app = celery.current_app
logg = celery_app.log.get_default_logger() #logg = celery_app.log.get_default_logger()
logg = logging.getLogger()
@celery_app.task(bind=True) @celery_app.task(bind=True)

View File

@ -48,8 +48,6 @@ class RoleMissingError(Exception):
pass pass
class IntegrityError(Exception): class IntegrityError(Exception):
"""Exception raised to signal irregularities with deduplication and ordering of tasks """Exception raised to signal irregularities with deduplication and ordering of tasks
@ -85,3 +83,8 @@ class RoleAgencyError(SeppukuError):
class YouAreBrokeError(Exception): class YouAreBrokeError(Exception):
"""Exception raised when a value transfer is attempted without access to sufficient funds """Exception raised when a value transfer is attempted without access to sufficient funds
""" """
class TrustError(Exception):
"""Exception raised when required trust proofs are missing for a request
"""

View File

@ -19,6 +19,7 @@ from hexathon import (
from chainqueue.error import NotLocalTxError from chainqueue.error import NotLocalTxError
from eth_erc20 import ERC20 from eth_erc20 import ERC20
from chainqueue.sql.tx import cache_tx_dict from chainqueue.sql.tx import cache_tx_dict
from okota.token_index import to_identifier
# local imports # local imports
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
@ -39,9 +40,11 @@ from cic_eth.task import (
CriticalSQLAlchemyTask, CriticalSQLAlchemyTask,
CriticalWeb3Task, CriticalWeb3Task,
CriticalSQLAlchemyAndSignerTask, CriticalSQLAlchemyAndSignerTask,
BaseTask,
) )
from cic_eth.eth.nonce import CustodialTaskNonceOracle from cic_eth.eth.nonce import CustodialTaskNonceOracle
from cic_eth.encode import tx_normalize from cic_eth.encode import tx_normalize
from cic_eth.eth.trust import verify_proofs
celery_app = celery.current_app celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()
@ -473,3 +476,69 @@ def cache_approve_data(
session.close() session.close()
return (tx_hash_hex, cache_id) return (tx_hash_hex, cache_id)
@celery_app.task(bind=True, base=BaseTask)
def token_info(self, tokens, chain_spec_dict, proofs=[]):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
rpc = RPCConnection.connect(chain_spec, 'default')
i = 0
for token in tokens:
result_data = []
token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
token_chain_object.load(rpc)
token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
token_proofs = [token_symbol_proof_hex]
if len(proofs) > 0:
token_proofs += proofs[i]
tokens[i] = {
'decimals': token_chain_object.decimals,
'name': token_chain_object.name,
'symbol': token_chain_object.symbol,
'address': tx_normalize.executable_address(token_chain_object.address),
'proofs': token_proofs,
'converters': tokens[i]['converters'],
}
i += 1
return tokens
@celery_app.task(bind=True, base=BaseTask)
def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
queue = self.request.delivery_info.get('routing_key')
for token in tokens:
s = celery.signature(
'cic_eth.eth.trust.verify_proofs',
[
token,
token['address'],
token['proofs'],
chain_spec_dict,
success_callback,
error_callback,
],
queue=queue,
)
if success_callback != None:
s.link(success_callback)
if error_callback != None:
s.on_error(error_callback)
s.apply_async()
return tokens
@celery_app.task(bind=True, base=BaseTask)
def default_token(self):
return {
'symbol': self.default_token_symbol,
'address': self.default_token_address,
'name': self.default_token_name,
'decimals': self.default_token_decimals,
}

View File

@ -0,0 +1,77 @@
# standard imports
import logging
# external imports
import celery
from eth_address_declarator import Declarator
from chainlib.connection import RPCConnection
from chainlib.chain import ChainSpec
from cic_eth.db.models.role import AccountRole
from cic_eth_registry import CICRegistry
from hexathon import strip_0x
# local imports
from cic_eth.task import BaseTask
from cic_eth.error import TrustError
celery_app = celery.current_app
logg = logging.getLogger()
@celery_app.task(bind=True, base=BaseTask)
def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
proof = strip_0x(proof)
proofs = []
logg.debug('proof count {}'.format(len(proofs)))
if len(proofs) == 0:
logg.debug('error {}'.format(len(proofs)))
raise TrustError('foo')
return (chained_input, (proof, proofs))
@celery_app.task(bind=True, base=BaseTask)
def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
queue = self.request.delivery_info.get('routing_key')
chain_spec = ChainSpec.from_dict(chain_spec_dict)
rpc = RPCConnection.connect(chain_spec, 'default')
session = self.create_session()
sender_address = AccountRole.get_address('DEFAULT', session)
registry = CICRegistry(chain_spec, rpc)
declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
declarator = Declarator(chain_spec)
have_proofs = {}
for proof in proofs:
proof = strip_0x(proof)
have_proofs[proof] = []
for trusted_address in self.trusted_addresses:
o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
r = rpc.do(o)
declarations = declarator.parse_declaration(r)
logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
for declaration in declarations:
declaration = strip_0x(declaration)
if declaration == proof:
logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
have_proofs[proof].append(trusted_address)
out_proofs = {}
for proof in have_proofs.keys():
if len(have_proofs[proof]) == 0:
logg.error('missing signer for proof {} subject {}'.format(proof, subject))
raise TrustError((subject, proof,))
out_proofs[proof] = have_proofs[proof]
return (chained_input, out_proofs)

View File

@ -4,18 +4,21 @@ import tempfile
import logging import logging
import shutil import shutil
# local impors # local imports
from cic_eth.task import BaseTask from cic_eth.task import BaseTask
#logg = logging.getLogger(__name__) #logg = logging.getLogger(__name__)
logg = logging.getLogger() logg = logging.getLogger()
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
def init_celery_tasks( def init_celery_tasks(
contract_roles, contract_roles,
): ):
BaseTask.call_address = contract_roles['DEFAULT'] BaseTask.call_address = contract_roles['DEFAULT']
BaseTask.trusted_addresses = [
contract_roles['TRUSTED_DECLARATOR'],
contract_roles['CONTRACT_DEPLOYER'],
]
# celery fixtures # celery fixtures
@ -38,6 +41,7 @@ def celery_includes():
'cic_eth.callbacks.noop', 'cic_eth.callbacks.noop',
'cic_eth.callbacks.http', 'cic_eth.callbacks.http',
'cic_eth.pytest.mock.filter', 'cic_eth.pytest.mock.filter',
'cic_eth.pytest.mock.callback',
] ]

View File

@ -1 +1,2 @@
from .filter import * from .filter import *
from .callback import *

View File

@ -0,0 +1,38 @@
# standard imports
import os
import logging
import mmap
# standard imports
import tempfile
# external imports
import celery
#logg = logging.getLogger(__name__)
logg = logging.getLogger()
celery_app = celery.current_app
class CallbackTask(celery.Task):
mmap_path = tempfile.mkdtemp()
@celery_app.task(bind=True, base=CallbackTask)
def test_callback(self, a, b, c):
s = 'ok'
if c > 0:
s = 'err'
fp = os.path.join(self.mmap_path, b)
f = open(fp, 'wb+')
f.write(b'\x00')
f.seek(0)
m = mmap.mmap(f.fileno(), length=1)
m.write(c.to_bytes(1, 'big'))
m.close()
f.close()
logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))

View File

@ -214,6 +214,7 @@ def main():
default_token.load(conn) default_token.load(conn)
BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_decimals = default_token.decimals
BaseTask.default_token_name = default_token.name BaseTask.default_token_name = default_token.name
BaseTask.trusted_addresses = trusted_addresses
BaseTask.run_dir = config.get('CIC_RUN_DIR') BaseTask.run_dir = config.get('CIC_RUN_DIR')
logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

View File

@ -28,6 +28,7 @@ class BaseTask(celery.Task):
session_func = SessionBase.create_session session_func = SessionBase.create_session
call_address = ZERO_ADDRESS call_address = ZERO_ADDRESS
trusted_addresses = []
create_nonce_oracle = RPCNonceOracle create_nonce_oracle = RPCNonceOracle
create_gas_oracle = RPCGasOracle create_gas_oracle = RPCGasOracle
default_token_address = None default_token_address = None

View File

@ -1,4 +1,4 @@
celery==4.4.7 celery==4.4.7
chainlib-eth>=0.0.9rc2,<0.1.0 chainlib-eth>=0.0.9rc4,<0.1.0
semver==2.13.0 semver==2.13.0
crypto-dev-signer>=0.4.15rc2,<0.5.0 crypto-dev-signer>=0.4.15rc2,<0.5.0

View File

@ -6,10 +6,11 @@ redis==3.5.3
hexathon~=0.0.1a8 hexathon~=0.0.1a8
pycryptodome==3.10.1 pycryptodome==3.10.1
liveness~=0.0.1a7 liveness~=0.0.1a7
eth-address-index>=0.2.3a4,<0.3.0 eth-address-index>=0.2.4a1,<0.3.0
eth-accounts-index>=0.1.2a3,<0.2.0 eth-accounts-index>=0.1.2a3,<0.2.0
cic-eth-registry>=0.6.1a6,<0.7.0 cic-eth-registry>=0.6.1a6,<0.7.0
erc20-faucet>=0.3.2a2,<0.4.0 erc20-faucet>=0.3.2a2,<0.4.0
erc20-transfer-authorization>=0.3.5a2,<0.4.0 erc20-transfer-authorization>=0.3.5a2,<0.4.0
sarafu-faucet>=0.0.7a2,<0.1.0 sarafu-faucet>=0.0.7a2,<0.1.0
moolb~=0.1.1b2 moolb~=0.1.1b2
okota>=0.2.4a6,<0.3.0

View File

@ -1,6 +1,27 @@
# standard imports
import logging
import os
import uuid
import time
import mmap
# external imports
import celery
import pytest
from hexathon import (
strip_0x,
uniform as hex_uniform,
)
# local imports # local imports
from cic_eth.api.api_task import Api from cic_eth.api.api_task import Api
from cic_eth.task import BaseTask from cic_eth.task import BaseTask
from cic_eth.error import TrustError
from cic_eth.encode import tx_normalize
from cic_eth.pytest.mock.callback import CallbackTask
logg = logging.getLogger()
def test_default_token( def test_default_token(
default_chain_spec, default_chain_spec,
@ -17,3 +38,175 @@ def test_default_token(
t = api.default_token() t = api.default_token()
r = t.get_leaf() r = t.get_leaf()
assert r['address'] == foo_token assert r['address'] == foo_token
def test_to_v_list():
assert Api.to_v_list('', 0) == []
assert Api.to_v_list([], 0) == []
assert Api.to_v_list('foo', 1) == [['foo']]
assert Api.to_v_list(['foo'], 1) == [['foo']]
assert Api.to_v_list(['foo', 'bar'], 2) == [['foo'], ['bar']]
assert Api.to_v_list('foo', 3) == [['foo'], ['foo'], ['foo']]
assert Api.to_v_list([['foo'], ['bar']], 2) == [['foo'], ['bar']]
with pytest.raises(ValueError):
Api.to_v_list([['foo'], ['bar']], 3)
with pytest.raises(ValueError):
Api.to_v_list(['foo', 'bar'], 3)
with pytest.raises(ValueError):
Api.to_v_list([['foo'], ['bar'], ['baz']], 2)
assert Api.to_v_list([
['foo'],
'bar',
['inky', 'pinky', 'blinky', 'clyde'],
], 3) == [
['foo'],
['bar'],
['inky', 'pinky', 'blinky', 'clyde'],
]
def test_token_single(
default_chain_spec,
foo_token,
bar_token,
token_registry,
register_tokens,
register_lookups,
cic_registry,
init_database,
init_celery_tasks,
custodial_roles,
foo_token_declaration,
bar_token_declaration,
celery_session_worker,
):
api = Api(str(default_chain_spec), queue=None, callback_param='foo')
t = api.token('FOO', proof=None)
r = t.get()
logg.debug('rr {}'.format(r))
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
t = api.token('FOO', proof=foo_token_declaration)
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
def test_tokens_noproof(
default_chain_spec,
foo_token,
bar_token,
token_registry,
register_tokens,
register_lookups,
cic_registry,
init_database,
init_celery_tasks,
custodial_roles,
foo_token_declaration,
bar_token_declaration,
celery_worker,
):
api = Api(str(default_chain_spec), queue=None, callback_param='foo')
t = api.tokens(['FOO'], proof=[])
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
t = api.tokens(['BAR'], proof='')
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(bar_token)
t = api.tokens(['FOO'], proof=None)
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
def test_tokens(
default_chain_spec,
foo_token,
bar_token,
token_registry,
register_tokens,
register_lookups,
cic_registry,
init_database,
init_celery_tasks,
custodial_roles,
foo_token_declaration,
bar_token_declaration,
celery_session_worker,
):
api = Api(str(default_chain_spec), queue=None, callback_param='foo')
t = api.tokens(['FOO'], proof=[[foo_token_declaration]])
r = t.get()
logg.debug('rr {}'.format(r))
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
t = api.tokens(['BAR', 'FOO'], proof=[[bar_token_declaration], [foo_token_declaration]])
r = t.get()
logg.debug('results {}'.format(r))
assert len(r) == 2
assert r[1]['address'] == strip_0x(foo_token)
assert r[0]['address'] == strip_0x(bar_token)
celery_app = celery.current_app
results = []
targets = []
api_param = str(uuid.uuid4())
api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
bogus_proof = os.urandom(32).hex()
t = api.tokens(['FOO'], proof=[[bogus_proof]])
r = t.get()
logg.debug('r {}'.format(r))
while True:
fp = os.path.join(CallbackTask.mmap_path, api_param)
try:
f = open(fp, 'rb')
except FileNotFoundError:
time.sleep(0.1)
logg.debug('look for {}'.format(fp))
continue
f = open(fp, 'rb')
m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
v = m.read(1)
m.close()
f.close()
assert v == b'\x01'
break
api_param = str(uuid.uuid4())
api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
t = api.tokens(['BAR'], proof=[[bar_token_declaration]])
r = t.get()
logg.debug('rr {} {}'.format(r, t.children))
while True:
fp = os.path.join(CallbackTask.mmap_path, api_param)
try:
f = open(fp, 'rb')
except FileNotFoundError:
time.sleep(0.1)
continue
m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
v = m.read(1)
m.close()
f.close()
assert v == b'\x00'
break

View File

@ -10,7 +10,7 @@ def test_default_token(
): ):
s = celery.signature( s = celery.signature(
'cic_eth.admin.token.default_token', 'cic_eth.eth.erc20.default_token',
[], [],
queue=None, queue=None,
) )

View File

@ -1,4 +1,4 @@
crypto-dev-signer>=0.4.15a7,<=0.4.15 crypto-dev-signer>=0.4.15rc2,<=0.4.15
chainqueue>=0.0.5a3,<0.1.0 chainqueue>=0.0.5a3,<0.1.0
cic-eth-registry>=0.6.1a6,<0.7.0 cic-eth-registry>=0.6.1a6,<0.7.0
redis==3.5.3 redis==3.5.3

View File

@ -3,7 +3,6 @@
# external imports # external imports
# local imports # local imports
from .base import Metadata
from .custom import CustomMetadata from .custom import CustomMetadata
from .person import PersonMetadata from .person import PersonMetadata
from .phone import PhonePointerMetadata from .phone import PhonePointerMetadata

View File

@ -1,99 +1,30 @@
# standard imports # standard imports
import json
import logging import logging
import os
from typing import Dict, Union
# third-part imports # external imports
from cic_types.models.person import generate_metadata_pointer, Person from cic_types.condiments import MetadataPointer
from cic_types.ext.metadata import MetadataRequestsHandler
from cic_types.processor import generate_metadata_pointer
# local imports # local imports
from cic_ussd.cache import cache_data, get_cached_data from cic_ussd.cache import cache_data, get_cached_data
from cic_ussd.http.requests import error_handler, make_request
from cic_ussd.metadata.signer import Signer
logg = logging.getLogger(__file__) logg = logging.getLogger(__file__)
class Metadata: class UssdMetadataHandler(MetadataRequestsHandler):
def __init__(self, cic_type: MetadataPointer, identifier: bytes):
super().__init__(cic_type, identifier)
def cache_metadata(self, data: str):
""" """
:cvar base_url: The base url or the metadata server. :param data:
:type base_url: str :type data:
:return:
:rtype:
""" """
cache_data(self.metadata_pointer, data)
base_url = None
class MetadataRequestsHandler(Metadata):
def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'):
""""""
self.cic_type = cic_type
self.engine = engine
self.headers = {
'X-CIC-AUTOMERGE': 'server',
'Content-Type': 'application/json'
}
self.identifier = identifier
self.metadata_pointer = generate_metadata_pointer(
identifier=self.identifier,
cic_type=self.cic_type
)
if self.base_url:
self.url = os.path.join(self.base_url, self.metadata_pointer)
def create(self, data: Union[Dict, str]):
""""""
data = json.dumps(data).encode('utf-8')
result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
error_handler(result=result)
metadata = result.json()
return self.edit(data=metadata)
def edit(self, data: Union[Dict, str]):
""""""
cic_meta_signer = Signer()
signature = cic_meta_signer.sign_digest(data=data)
algorithm = cic_meta_signer.get_operational_key().get('algo')
formatted_data = {
'm': json.dumps(data),
's': {
'engine': self.engine,
'algo': algorithm,
'data': signature,
'digest': data.get('digest'),
}
}
formatted_data = json.dumps(formatted_data)
result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
logg.info(f'signed metadata submission status: {result.status_code}.')
error_handler(result=result)
try:
decoded_identifier = self.identifier.decode("utf-8")
except UnicodeDecodeError:
decoded_identifier = self.identifier.hex()
logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.')
return result
def query(self):
""""""
result = make_request(method='GET', url=self.url)
error_handler(result=result)
result_data = result.json()
if not isinstance(result_data, dict):
raise ValueError(f'Invalid result data object: {result_data}.')
if result.status_code == 200:
if self.cic_type == ':cic.person':
person = Person()
person_data = person.deserialize(person_data=result_data)
serialized_person_data = person_data.serialize()
data = json.dumps(serialized_person_data)
else:
data = json.dumps(result_data)
cache_data(key=self.metadata_pointer, data=data)
logg.debug(f'caching: {data} with key: {self.metadata_pointer}') logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
return result_data
def get_cached_metadata(self): def get_cached_metadata(self):
"""""" """"""

View File

@ -1,12 +1,13 @@
# standard imports # standard imports
# external imports # external imports
from cic_types.condiments import MetadataPointer
# local imports # local imports
from .base import MetadataRequestsHandler from .base import UssdMetadataHandler
class CustomMetadata(MetadataRequestsHandler): class CustomMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes): def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.custom', identifier=identifier) super().__init__(cic_type=MetadataPointer.CUSTOM, identifier=identifier)

View File

@ -1,12 +1,13 @@
# standard imports # standard imports
# external imports # external imports
from cic_types.condiments import MetadataPointer
# local imports # local imports
from .base import MetadataRequestsHandler from .base import UssdMetadataHandler
class PersonMetadata(MetadataRequestsHandler): class PersonMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes): def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.person', identifier=identifier) super().__init__(cic_type=MetadataPointer.PERSON, identifier=identifier)

View File

@ -2,12 +2,13 @@
import logging import logging
# external imports # external imports
from cic_types.condiments import MetadataPointer
# local imports # local imports
from .base import MetadataRequestsHandler from .base import UssdMetadataHandler
class PhonePointerMetadata(MetadataRequestsHandler): class PhonePointerMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes): def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.phone', identifier=identifier) super().__init__(cic_type=MetadataPointer.PHONE, identifier=identifier)

View File

@ -1,13 +1,13 @@
# standard imports # standard imports
# external imports # external imports
import celery from cic_types.condiments import MetadataPointer
# local imports # local imports
from .base import MetadataRequestsHandler from .base import UssdMetadataHandler
class PreferencesMetadata(MetadataRequestsHandler): class PreferencesMetadata(UssdMetadataHandler):
def __init__(self, identifier: bytes): def __init__(self, identifier: bytes):
super().__init__(cic_type=':cic.preferences', identifier=identifier) super().__init__(cic_type=MetadataPointer.PREFERENCES, identifier=identifier)

View File

@ -1,60 +0,0 @@
# standard imports
import json
import logging
from typing import Optional
from urllib.request import Request, urlopen
# third-party imports
import gnupg
# local imports
logg = logging.getLogger()
class Signer:
"""
:cvar gpg_path:
:type gpg_path:
:cvar gpg_passphrase:
:type gpg_passphrase:
:cvar key_file_path:
:type key_file_path:
"""
gpg_path: str = None
gpg_passphrase: str = None
key_file_path: str = None
def __init__(self):
self.gpg = gnupg.GPG(gnupghome=self.gpg_path)
with open(self.key_file_path, 'r') as key_file:
self.key_data = key_file.read()
def get_operational_key(self):
"""
:return:
:rtype:
"""
# import key data into keyring
self.gpg.import_keys(key_data=self.key_data)
gpg_keys = self.gpg.list_keys()
key_algorithm = gpg_keys[0].get('algo')
key_id = gpg_keys[0].get("keyid")
logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}')
return gpg_keys[0]
def sign_digest(self, data: dict):
"""
:param data:
:type data:
:return:
:rtype:
"""
digest = data['digest']
key_id = self.get_operational_key().get('keyid')
signature = self.gpg.sign(digest, passphrase=self.gpg_passphrase, keyid=key_id)
return str(signature)

View File

@ -1,15 +1,17 @@
# standard imports # standard imports
import json
import logging import logging
# third-party imports # third-party imports
import celery import celery
from cic_types.models.person import Person
# local imports # local imports
from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
from cic_ussd.tasks.base import CriticalMetadataTask from cic_ussd.tasks.base import CriticalMetadataTask
celery_app = celery.current_app celery_app = celery.current_app
logg = logging.getLogger().getChild(__name__) logg = logging.getLogger(__file__)
@celery_app.task @celery_app.task
@ -22,7 +24,13 @@ def query_person_metadata(blockchain_address: str):
""" """
identifier = bytes.fromhex(blockchain_address) identifier = bytes.fromhex(blockchain_address)
person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client = PersonMetadata(identifier=identifier)
person_metadata_client.query() response = person_metadata_client.query()
data = response.json()
person = Person()
person_data = person.deserialize(person_data=data)
serialized_person_data = person_data.serialize()
data = json.dumps(serialized_person_data)
person_metadata_client.cache_metadata(data=data)
@celery_app.task @celery_app.task
@ -76,6 +84,9 @@ def query_preferences_metadata(blockchain_address: str):
:type blockchain_address: str | Ox-hex :type blockchain_address: str | Ox-hex
""" """
identifier = bytes.fromhex(blockchain_address) identifier = bytes.fromhex(blockchain_address)
logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.') logg.debug(f'retrieving preferences metadata for address: {blockchain_address}.')
person_metadata_client = PreferencesMetadata(identifier=identifier) preferences_metadata_client = PreferencesMetadata(identifier=identifier)
return person_metadata_client.query() response = preferences_metadata_client.query()
data = json.dumps(response.json())
preferences_metadata_client.cache_metadata(data)
return data

View File

@ -4,10 +4,10 @@ billiard==3.6.4.0
bcrypt==3.2.0 bcrypt==3.2.0
celery==4.4.7 celery==4.4.7
cffi==1.14.6 cffi==1.14.6
cic-eth[services]~=0.12.4a7 cic-eth[services]~=0.12.4a11
cic-notify~=0.4.0a10 cic-notify~=0.4.0a10
cic-types~=0.1.0a15 cic-types~=0.2.0a3
confini>=0.4.1a1,<0.5.0 confini>=0.3.6rc4,<0.5.0
phonenumbers==8.12.12 phonenumbers==8.12.12
psycopg2==2.8.6 psycopg2==2.8.6
python-i18n[YAML]==0.3.9 python-i18n[YAML]==0.3.9

View File

@ -5,24 +5,25 @@ import os
# external imports # external imports
import requests_mock import requests_mock
from chainlib.hash import strip_0x from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
# local imports # local imports
from cic_ussd.metadata.base import MetadataRequestsHandler from cic_ussd.metadata.base import UssdMetadataHandler
# external imports # external imports
def test_metadata_requests_handler(activated_account, def test_ussd_metadata_handler(activated_account,
init_cache, init_cache,
load_config, load_config,
person_metadata, person_metadata,
setup_metadata_request_handler, setup_metadata_request_handler,
setup_metadata_signer): setup_metadata_signer):
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
cic_type = ':cic.person' cic_type = MetadataPointer.PERSON
metadata_client = MetadataRequestsHandler(cic_type, identifier) metadata_client = UssdMetadataHandler(cic_type, identifier)
assert metadata_client.cic_type == cic_type assert metadata_client.cic_type == cic_type
assert metadata_client.engine == 'pgp' assert metadata_client.engine == 'pgp'
assert metadata_client.identifier == identifier assert metadata_client.identifier == identifier
@ -38,7 +39,5 @@ def test_metadata_requests_handler(activated_account,
assert result.status_code == 200 assert result.status_code == 200
person_metadata.pop('digest') person_metadata.pop('digest')
request_mocker.register_uri('GET', metadata_client.url, status_code=200, reason='OK', json=person_metadata) request_mocker.register_uri('GET', metadata_client.url, status_code=200, reason='OK', json=person_metadata)
result = metadata_client.query() result = metadata_client.query().json()
assert result == person_metadata assert result == person_metadata
cached_metadata = metadata_client.get_cached_metadata()
assert json.loads(cached_metadata) == person_metadata

View File

@ -1,7 +1,7 @@
# standard imports # standard imports
import os import os
# external imports # external imports
from chainlib.hash import strip_0x from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
# local imports # local imports
@ -11,8 +11,8 @@ from cic_ussd.metadata import CustomMetadata
def test_custom_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): def test_custom_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.custom' cic_type = MetadataPointer.CUSTOM
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) identifier = bytes.fromhex(activated_account.blockchain_address)
custom_metadata_client = CustomMetadata(identifier) custom_metadata_client = CustomMetadata(identifier)
assert custom_metadata_client.cic_type == cic_type assert custom_metadata_client.cic_type == cic_type
assert custom_metadata_client.engine == 'pgp' assert custom_metadata_client.engine == 'pgp'

View File

@ -1,7 +1,7 @@
# standard imports # standard imports
import os import os
# external imports # external imports
from chainlib.hash import strip_0x from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
# local imports # local imports
@ -11,8 +11,8 @@ from cic_ussd.metadata import PersonMetadata
def test_person_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): def test_person_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.person' cic_type = MetadataPointer.PERSON
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) identifier = bytes.fromhex(activated_account.blockchain_address)
person_metadata_client = PersonMetadata(identifier) person_metadata_client = PersonMetadata(identifier)
assert person_metadata_client.cic_type == cic_type assert person_metadata_client.cic_type == cic_type
assert person_metadata_client.engine == 'pgp' assert person_metadata_client.engine == 'pgp'

View File

@ -1,7 +1,7 @@
# standard imports # standard imports
import os import os
# external imports # external imports
from chainlib.hash import strip_0x from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
# local imports # local imports
@ -12,8 +12,8 @@ from cic_ussd.metadata import PhonePointerMetadata
def test_phone_pointer_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): def test_phone_pointer_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.phone' cic_type = MetadataPointer.PHONE
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) identifier = bytes.fromhex(activated_account.blockchain_address)
phone_pointer_metadata = PhonePointerMetadata(identifier) phone_pointer_metadata = PhonePointerMetadata(identifier)
assert phone_pointer_metadata.cic_type == cic_type assert phone_pointer_metadata.cic_type == cic_type
assert phone_pointer_metadata.engine == 'pgp' assert phone_pointer_metadata.engine == 'pgp'

View File

@ -1,7 +1,7 @@
# standard imports # standard imports
import os import os
# external imports # external imports
from chainlib.hash import strip_0x from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
# local imports # local imports
@ -11,8 +11,8 @@ from cic_ussd.metadata import PreferencesMetadata
def test_preferences_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): def test_preferences_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
cic_type = ':cic.preferences' cic_type = MetadataPointer.PREFERENCES
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) identifier = bytes.fromhex(activated_account.blockchain_address)
preferences_metadata_client = PreferencesMetadata(identifier) preferences_metadata_client = PreferencesMetadata(identifier)
assert preferences_metadata_client.cic_type == cic_type assert preferences_metadata_client.cic_type == cic_type
assert preferences_metadata_client.engine == 'pgp' assert preferences_metadata_client.engine == 'pgp'

View File

@ -1,17 +0,0 @@
# standard imports
import shutil
# third-party imports
# local imports
from cic_ussd.metadata.signer import Signer
def test_client(load_config, setup_metadata_signer, person_metadata):
signer = Signer()
gpg = signer.gpg
assert signer.key_data is not None
gpg.import_keys(key_data=signer.key_data)
gpg_keys = gpg.list_keys()
assert signer.get_operational_key() == gpg_keys[0]
shutil.rmtree(Signer.gpg_path)

View File

@ -6,33 +6,19 @@ import tempfile
# external imports # external imports
import pytest import pytest
from chainlib.hash import strip_0x from chainlib.hash import strip_0x
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
# local imports # local imports
from cic_ussd.metadata import Metadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata from cic_ussd.metadata import PersonMetadata, PhonePointerMetadata, PreferencesMetadata
from cic_ussd.metadata.signer import Signer
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)
@pytest.fixture(scope='function')
def setup_metadata_signer(load_config):
temp_dir = tempfile.mkdtemp(dir='/tmp')
logg.debug(f'Created temp dir: {temp_dir}')
Signer.gpg_path = temp_dir
Signer.gpg_passphrase = load_config.get('PGP_PASSPHRASE')
Signer.key_file_path = os.path.join(load_config.get('PGP_KEYS_PATH'), load_config.get('PGP_PRIVATE_KEYS'))
@pytest.fixture(scope='function')
def setup_metadata_request_handler(load_config):
Metadata.base_url = load_config.get('CIC_META_URL')
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
def account_phone_pointer(activated_account): def account_phone_pointer(activated_account):
identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
return generate_metadata_pointer(identifier, ':cic.phone') return generate_metadata_pointer(identifier, MetadataPointer.PERSON)
@pytest.fixture(scope='function') @pytest.fixture(scope='function')

View File

@ -1,6 +1,6 @@
cic-eth[tools]==0.12.4a8 cic-eth[tools]==0.12.4a11
chainlib-eth>=0.0.9rc4,<0.0.10 chainlib-eth>=0.0.9rc4,<0.1.0
chainlib==0.0.9rc1,<0.0.10 chainlib==0.0.9rc1,<0.1.0
eth-erc20>=0.1.2a3,<0.2.0 eth-erc20>=0.1.2a3,<0.2.0
erc20-demurrage-token>=0.0.5a2,<0.1.0 erc20-demurrage-token>=0.0.5a2,<0.1.0
eth-address-index>=0.2.4a1,<0.3.0 eth-address-index>=0.2.4a1,<0.3.0