diff --git a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt index 04e3428b..5a01755e 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt @@ -1,4 +1,5 @@ celery==4.4.7 -erc20-demurrage-token~=0.0.3a1 -cic-eth-registry>=0.6.1a2,<0.7.0 -cic-eth[services]~=0.12.4a8 +erc20-demurrage-token~=0.0.5a3 +cic-eth-registry~=0.6.1a5 +chainlib~=0.0.9rc3 +cic_eth~=0.12.4a9 diff --git a/apps/cic-eth/cic_eth/admin/token.py b/apps/cic-eth/cic_eth/admin/token.py index 985a5f2c..6b718cbc 100644 --- a/apps/cic-eth/cic_eth/admin/token.py +++ b/apps/cic-eth/cic_eth/admin/token.py @@ -1,21 +1,2 @@ -# standard imports -import logging - -# external imports -import celery - # local imports -from cic_eth.task import BaseTask - -celery_app = celery.current_app -logg = logging.getLogger() - - -@celery_app.task(bind=True, base=BaseTask) -def default_token(self): - return { - 'symbol': self.default_token_symbol, - 'address': self.default_token_address, - 'name': self.default_token_name, - 'decimals': self.default_token_decimals, - } +from cic_eth.eth.erc20 import default_token diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index 1ae14123..b0ecf047 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -17,15 +17,50 @@ from cic_eth.enum import LockEnum app = celery.current_app -logg = logging.getLogger(__name__) +#logg = logging.getLogger(__name__) +logg = logging.getLogger() class Api(ApiBase): - + + @staticmethod + def to_v_list(v, n): + """Translate an arbitrary number of string and/or list arguments to a list of list of string arguments + + :param v: Arguments + :type v: str or list + :param n: Number of elements to generate arguments for + :type n: int + :rtype: list + :returns: list of assembled arguments + """ + if isinstance(v, str): + vv = v + v = [] + for i in range(n): + v.append([vv]) + elif not isinstance(v, list): + raise ValueError('argument must be single string, or list or strings or lists') + else: + if len(v) != n: + raise ValueError('v argument count must match integer n') + for i in range(n): + if isinstance(v[i], str): + v[i] = [v[i]] + elif not isinstance(v, list): + raise ValueError('proof argument must be single string, or list or strings or lists') + + return v + def default_token(self): + """Retrieves the default fallback token of the custodial network. + + :returns: uuid of root task + :rtype: celery.Task + """ s_token = celery.signature( - 'cic_eth.admin.token.default_token', + 'cic_eth.eth.erc20.default_token', [], queue=self.queue, ) @@ -35,6 +70,97 @@ class Api(ApiBase): return s_token.apply_async() + def token(self, token_symbol, proof=None): + """Single-token alias for tokens method. + + See tokens method for details. + + :param token_symbol: Token symbol to look up + :type token_symbol: str + :param proof: Proofs to add to signature verification for the token + :type proof: str or list + :returns: uuid of root task + :rtype: celery.Task + """ + if not isinstance(token_symbol, str): + raise ValueError('token symbol must be string') + + return self.tokens([token_symbol], proof=proof) + + + def tokens(self, token_symbols, proof=None): + """Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address. + + Token symbols are always strings, and should be specified using uppercase letters. + + If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup. + + The proof argument may be specified in a number of ways: + + - as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed) + - as a single string, where the same proof is used for each token lookup + - as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens. + - as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens. + + The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback. + + This method is not intended to be used synchronously. Do so at your peril. + + :param token_symbols: Token symbol strings to look up + :type token_symbol: list + :param proof: Proof(s) to verify tokens against + :type proof: None, str or list + :returns: uuid of root task + :rtype: celery.Task + """ + if not isinstance(token_symbols, list): + raise ValueError('token symbols argument must be list') + + if proof == None: + logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols))) + proof = '' + + logg.debug('proof is {}'.format(proof)) + l = len(token_symbols) + if len(proof) == 0: + l = 0 + proof = Api.to_v_list(proof, l) + + chain_spec_dict = self.chain_spec.asdict() + + s_token_resolve = celery.signature( + 'cic_eth.eth.erc20.resolve_tokens_by_symbol', + [ + token_symbols, + chain_spec_dict, + ], + queue=self.queue, + ) + + s_token_info = celery.signature( + 'cic_eth.eth.erc20.token_info', + [ + chain_spec_dict, + proof, + ], + queue=self.queue, + ) + + s_token_verify = celery.signature( + 'cic_eth.eth.erc20.verify_token_info', + [ + chain_spec_dict, + self.callback_success, + self.callback_error, + ], + queue=self.queue, + ) + + s_token_info.link(s_token_verify) + s_token_resolve.link(s_token_info) + return s_token_resolve.apply_async() + + # def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol): # """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed. # diff --git a/apps/cic-eth/cic_eth/callbacks/noop.py b/apps/cic-eth/cic_eth/callbacks/noop.py index 2fd579e3..2308734c 100644 --- a/apps/cic-eth/cic_eth/callbacks/noop.py +++ b/apps/cic-eth/cic_eth/callbacks/noop.py @@ -1,7 +1,10 @@ +import logging + import celery celery_app = celery.current_app -logg = celery_app.log.get_default_logger() +#logg = celery_app.log.get_default_logger() +logg = logging.getLogger() @celery_app.task(bind=True) diff --git a/apps/cic-eth/cic_eth/error.py b/apps/cic-eth/cic_eth/error.py index 3e72a439..7e8dbfa6 100644 --- a/apps/cic-eth/cic_eth/error.py +++ b/apps/cic-eth/cic_eth/error.py @@ -48,8 +48,6 @@ class RoleMissingError(Exception): pass - - class IntegrityError(Exception): """Exception raised to signal irregularities with deduplication and ordering of tasks @@ -85,3 +83,8 @@ class RoleAgencyError(SeppukuError): class YouAreBrokeError(Exception): """Exception raised when a value transfer is attempted without access to sufficient funds """ + + +class TrustError(Exception): + """Exception raised when required trust proofs are missing for a request + """ diff --git a/apps/cic-eth/cic_eth/eth/erc20.py b/apps/cic-eth/cic_eth/eth/erc20.py index cf433c9e..4fbe2e57 100644 --- a/apps/cic-eth/cic_eth/eth/erc20.py +++ b/apps/cic-eth/cic_eth/eth/erc20.py @@ -19,6 +19,7 @@ from hexathon import ( from chainqueue.error import NotLocalTxError from eth_erc20 import ERC20 from chainqueue.sql.tx import cache_tx_dict +from okota.token_index import to_identifier # local imports from cic_eth.db.models.base import SessionBase @@ -39,9 +40,11 @@ from cic_eth.task import ( CriticalSQLAlchemyTask, CriticalWeb3Task, CriticalSQLAlchemyAndSignerTask, + BaseTask, ) from cic_eth.eth.nonce import CustodialTaskNonceOracle from cic_eth.encode import tx_normalize +from cic_eth.eth.trust import verify_proofs celery_app = celery.current_app logg = logging.getLogger() @@ -473,3 +476,69 @@ def cache_approve_data( session.close() return (tx_hash_hex, cache_id) + +@celery_app.task(bind=True, base=BaseTask) +def token_info(self, tokens, chain_spec_dict, proofs=[]): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + rpc = RPCConnection.connect(chain_spec, 'default') + + i = 0 + + for token in tokens: + result_data = [] + token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address'])) + token_chain_object.load(rpc) + + token_symbol_proof_hex = to_identifier(token_chain_object.symbol) + token_proofs = [token_symbol_proof_hex] + if len(proofs) > 0: + token_proofs += proofs[i] + + tokens[i] = { + 'decimals': token_chain_object.decimals, + 'name': token_chain_object.name, + 'symbol': token_chain_object.symbol, + 'address': tx_normalize.executable_address(token_chain_object.address), + 'proofs': token_proofs, + 'converters': tokens[i]['converters'], + } + i += 1 + + return tokens + + +@celery_app.task(bind=True, base=BaseTask) +def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback): + queue = self.request.delivery_info.get('routing_key') + + for token in tokens: + s = celery.signature( + 'cic_eth.eth.trust.verify_proofs', + [ + token, + token['address'], + token['proofs'], + chain_spec_dict, + success_callback, + error_callback, + ], + queue=queue, + ) + + if success_callback != None: + s.link(success_callback) + if error_callback != None: + s.on_error(error_callback) + s.apply_async() + + return tokens + + +@celery_app.task(bind=True, base=BaseTask) +def default_token(self): + return { + 'symbol': self.default_token_symbol, + 'address': self.default_token_address, + 'name': self.default_token_name, + 'decimals': self.default_token_decimals, + } diff --git a/apps/cic-eth/cic_eth/eth/trust.py b/apps/cic-eth/cic_eth/eth/trust.py new file mode 100644 index 00000000..6298b8e6 --- /dev/null +++ b/apps/cic-eth/cic_eth/eth/trust.py @@ -0,0 +1,77 @@ +# standard imports +import logging + +# external imports +import celery +from eth_address_declarator import Declarator +from chainlib.connection import RPCConnection +from chainlib.chain import ChainSpec +from cic_eth.db.models.role import AccountRole +from cic_eth_registry import CICRegistry +from hexathon import strip_0x + +# local imports +from cic_eth.task import BaseTask +from cic_eth.error import TrustError + +celery_app = celery.current_app +logg = logging.getLogger() + + +@celery_app.task(bind=True, base=BaseTask) +def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback): + proof = strip_0x(proof) + + proofs = [] + + logg.debug('proof count {}'.format(len(proofs))) + if len(proofs) == 0: + logg.debug('error {}'.format(len(proofs))) + raise TrustError('foo') + + return (chained_input, (proof, proofs)) + + +@celery_app.task(bind=True, base=BaseTask) +def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback): + queue = self.request.delivery_info.get('routing_key') + + chain_spec = ChainSpec.from_dict(chain_spec_dict) + rpc = RPCConnection.connect(chain_spec, 'default') + + session = self.create_session() + sender_address = AccountRole.get_address('DEFAULT', session) + + registry = CICRegistry(chain_spec, rpc) + declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address) + + declarator = Declarator(chain_spec) + + have_proofs = {} + + for proof in proofs: + + proof = strip_0x(proof) + + have_proofs[proof] = [] + + for trusted_address in self.trusted_addresses: + o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address) + r = rpc.do(o) + declarations = declarator.parse_declaration(r) + logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations)) + + for declaration in declarations: + declaration = strip_0x(declaration) + if declaration == proof: + logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address)) + have_proofs[proof].append(trusted_address) + + out_proofs = {} + for proof in have_proofs.keys(): + if len(have_proofs[proof]) == 0: + logg.error('missing signer for proof {} subject {}'.format(proof, subject)) + raise TrustError((subject, proof,)) + out_proofs[proof] = have_proofs[proof] + + return (chained_input, out_proofs) diff --git a/apps/cic-eth/cic_eth/pytest/fixtures_celery.py b/apps/cic-eth/cic_eth/pytest/fixtures_celery.py index e0f16406..f2287dab 100644 --- a/apps/cic-eth/cic_eth/pytest/fixtures_celery.py +++ b/apps/cic-eth/cic_eth/pytest/fixtures_celery.py @@ -4,18 +4,21 @@ import tempfile import logging import shutil -# local impors +# local imports from cic_eth.task import BaseTask #logg = logging.getLogger(__name__) logg = logging.getLogger() - @pytest.fixture(scope='function') def init_celery_tasks( contract_roles, ): BaseTask.call_address = contract_roles['DEFAULT'] + BaseTask.trusted_addresses = [ + contract_roles['TRUSTED_DECLARATOR'], + contract_roles['CONTRACT_DEPLOYER'], + ] # celery fixtures @@ -38,6 +41,7 @@ def celery_includes(): 'cic_eth.callbacks.noop', 'cic_eth.callbacks.http', 'cic_eth.pytest.mock.filter', + 'cic_eth.pytest.mock.callback', ] diff --git a/apps/cic-eth/cic_eth/pytest/mock/__init__.py b/apps/cic-eth/cic_eth/pytest/mock/__init__.py index 1e78c9be..71866a19 100644 --- a/apps/cic-eth/cic_eth/pytest/mock/__init__.py +++ b/apps/cic-eth/cic_eth/pytest/mock/__init__.py @@ -1 +1,2 @@ from .filter import * +from .callback import * diff --git a/apps/cic-eth/cic_eth/pytest/mock/callback.py b/apps/cic-eth/cic_eth/pytest/mock/callback.py new file mode 100644 index 00000000..6825b208 --- /dev/null +++ b/apps/cic-eth/cic_eth/pytest/mock/callback.py @@ -0,0 +1,38 @@ +# standard imports +import os +import logging +import mmap + +# standard imports +import tempfile + +# external imports +import celery + +#logg = logging.getLogger(__name__) +logg = logging.getLogger() + +celery_app = celery.current_app + + +class CallbackTask(celery.Task): + + mmap_path = tempfile.mkdtemp() + + +@celery_app.task(bind=True, base=CallbackTask) +def test_callback(self, a, b, c): + s = 'ok' + if c > 0: + s = 'err' + + fp = os.path.join(self.mmap_path, b) + f = open(fp, 'wb+') + f.write(b'\x00') + f.seek(0) + m = mmap.mmap(f.fileno(), length=1) + m.write(c.to_bytes(1, 'big')) + m.close() + f.close() + + logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c)) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 1e232825..3ccf2c03 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -214,6 +214,7 @@ def main(): default_token.load(conn) BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_name = default_token.name + BaseTask.trusted_addresses = trusted_addresses BaseTask.run_dir = config.get('CIC_RUN_DIR') logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index 014565dd..149e1800 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -28,6 +28,7 @@ class BaseTask(celery.Task): session_func = SessionBase.create_session call_address = ZERO_ADDRESS + trusted_addresses = [] create_nonce_oracle = RPCNonceOracle create_gas_oracle = RPCGasOracle default_token_address = None diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 542580c9..9eba70d4 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ celery==4.4.7 -chainlib-eth>=0.0.9rc2,<0.1.0 +chainlib-eth>=0.0.9rc4,<0.1.0 semver==2.13.0 crypto-dev-signer>=0.4.15rc2,<0.5.0 diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index bd44e17f..3cce090c 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -6,10 +6,11 @@ redis==3.5.3 hexathon~=0.0.1a8 pycryptodome==3.10.1 liveness~=0.0.1a7 -eth-address-index>=0.2.3a4,<0.3.0 +eth-address-index>=0.2.4a1,<0.3.0 eth-accounts-index>=0.1.2a3,<0.2.0 cic-eth-registry>=0.6.1a6,<0.7.0 erc20-faucet>=0.3.2a2,<0.4.0 erc20-transfer-authorization>=0.3.5a2,<0.4.0 sarafu-faucet>=0.0.7a2,<0.1.0 moolb~=0.1.1b2 +okota>=0.2.4a6,<0.3.0 diff --git a/apps/cic-eth/tests/task/api/test_app_noncritical.py b/apps/cic-eth/tests/task/api/test_app_noncritical.py index 7ebb8700..ba8acb3b 100644 --- a/apps/cic-eth/tests/task/api/test_app_noncritical.py +++ b/apps/cic-eth/tests/task/api/test_app_noncritical.py @@ -1,6 +1,27 @@ +# standard imports +import logging +import os +import uuid +import time +import mmap + +# external imports +import celery +import pytest +from hexathon import ( + strip_0x, + uniform as hex_uniform, + ) + # local imports from cic_eth.api.api_task import Api from cic_eth.task import BaseTask +from cic_eth.error import TrustError +from cic_eth.encode import tx_normalize +from cic_eth.pytest.mock.callback import CallbackTask + +logg = logging.getLogger() + def test_default_token( default_chain_spec, @@ -17,3 +38,175 @@ def test_default_token( t = api.default_token() r = t.get_leaf() assert r['address'] == foo_token + + +def test_to_v_list(): + assert Api.to_v_list('', 0) == [] + assert Api.to_v_list([], 0) == [] + assert Api.to_v_list('foo', 1) == [['foo']] + assert Api.to_v_list(['foo'], 1) == [['foo']] + assert Api.to_v_list(['foo', 'bar'], 2) == [['foo'], ['bar']] + assert Api.to_v_list('foo', 3) == [['foo'], ['foo'], ['foo']] + assert Api.to_v_list([['foo'], ['bar']], 2) == [['foo'], ['bar']] + with pytest.raises(ValueError): + Api.to_v_list([['foo'], ['bar']], 3) + with pytest.raises(ValueError): + Api.to_v_list(['foo', 'bar'], 3) + with pytest.raises(ValueError): + Api.to_v_list([['foo'], ['bar'], ['baz']], 2) + + assert Api.to_v_list([ + ['foo'], + 'bar', + ['inky', 'pinky', 'blinky', 'clyde'], + ], 3) == [ + ['foo'], + ['bar'], + ['inky', 'pinky', 'blinky', 'clyde'], + ] + + +def test_token_single( + default_chain_spec, + foo_token, + bar_token, + token_registry, + register_tokens, + register_lookups, + cic_registry, + init_database, + init_celery_tasks, + custodial_roles, + foo_token_declaration, + bar_token_declaration, + celery_session_worker, + ): + + api = Api(str(default_chain_spec), queue=None, callback_param='foo') + + t = api.token('FOO', proof=None) + r = t.get() + logg.debug('rr {}'.format(r)) + assert len(r) == 1 + assert r[0]['address'] == strip_0x(foo_token) + + + t = api.token('FOO', proof=foo_token_declaration) + r = t.get() + assert len(r) == 1 + assert r[0]['address'] == strip_0x(foo_token) + + +def test_tokens_noproof( + default_chain_spec, + foo_token, + bar_token, + token_registry, + register_tokens, + register_lookups, + cic_registry, + init_database, + init_celery_tasks, + custodial_roles, + foo_token_declaration, + bar_token_declaration, + celery_worker, + ): + + api = Api(str(default_chain_spec), queue=None, callback_param='foo') + + t = api.tokens(['FOO'], proof=[]) + r = t.get() + assert len(r) == 1 + assert r[0]['address'] == strip_0x(foo_token) + + t = api.tokens(['BAR'], proof='') + r = t.get() + assert len(r) == 1 + assert r[0]['address'] == strip_0x(bar_token) + + t = api.tokens(['FOO'], proof=None) + r = t.get() + assert len(r) == 1 + assert r[0]['address'] == strip_0x(foo_token) + + +def test_tokens( + default_chain_spec, + foo_token, + bar_token, + token_registry, + register_tokens, + register_lookups, + cic_registry, + init_database, + init_celery_tasks, + custodial_roles, + foo_token_declaration, + bar_token_declaration, + celery_session_worker, + ): + + api = Api(str(default_chain_spec), queue=None, callback_param='foo') + + t = api.tokens(['FOO'], proof=[[foo_token_declaration]]) + r = t.get() + logg.debug('rr {}'.format(r)) + assert len(r) == 1 + assert r[0]['address'] == strip_0x(foo_token) + + t = api.tokens(['BAR', 'FOO'], proof=[[bar_token_declaration], [foo_token_declaration]]) + r = t.get() + logg.debug('results {}'.format(r)) + assert len(r) == 2 + assert r[1]['address'] == strip_0x(foo_token) + assert r[0]['address'] == strip_0x(bar_token) + + celery_app = celery.current_app + + results = [] + targets = [] + + api_param = str(uuid.uuid4()) + api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback') + bogus_proof = os.urandom(32).hex() + t = api.tokens(['FOO'], proof=[[bogus_proof]]) + r = t.get() + logg.debug('r {}'.format(r)) + + while True: + fp = os.path.join(CallbackTask.mmap_path, api_param) + try: + f = open(fp, 'rb') + except FileNotFoundError: + time.sleep(0.1) + logg.debug('look for {}'.format(fp)) + continue + f = open(fp, 'rb') + m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1) + v = m.read(1) + m.close() + f.close() + assert v == b'\x01' + break + + api_param = str(uuid.uuid4()) + api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback') + t = api.tokens(['BAR'], proof=[[bar_token_declaration]]) + r = t.get() + logg.debug('rr {} {}'.format(r, t.children)) + + while True: + fp = os.path.join(CallbackTask.mmap_path, api_param) + try: + f = open(fp, 'rb') + except FileNotFoundError: + time.sleep(0.1) + continue + m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1) + v = m.read(1) + m.close() + f.close() + assert v == b'\x00' + break + diff --git a/apps/cic-eth/tests/unit/admin/test_default_token.py b/apps/cic-eth/tests/unit/admin/test_default_token.py index d49ed3fe..da1d3c90 100644 --- a/apps/cic-eth/tests/unit/admin/test_default_token.py +++ b/apps/cic-eth/tests/unit/admin/test_default_token.py @@ -10,7 +10,7 @@ def test_default_token( ): s = celery.signature( - 'cic_eth.admin.token.default_token', + 'cic_eth.eth.erc20.default_token', [], queue=None, ) diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index c4a31eb0..77eece3f 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,4 +1,4 @@ -crypto-dev-signer>=0.4.15a7,<=0.4.15 +crypto-dev-signer>=0.4.15rc2,<=0.4.15 chainqueue>=0.0.5a3,<0.1.0 cic-eth-registry>=0.6.1a6,<0.7.0 redis==3.5.3 diff --git a/apps/cic-ussd/cic_ussd/metadata/__init__.py b/apps/cic-ussd/cic_ussd/metadata/__init__.py index 33989352..a5deea10 100644 --- a/apps/cic-ussd/cic_ussd/metadata/__init__.py +++ b/apps/cic-ussd/cic_ussd/metadata/__init__.py @@ -3,7 +3,6 @@ # external imports # local imports -from .base import Metadata from .custom import CustomMetadata from .person import PersonMetadata from .phone import PhonePointerMetadata diff --git a/apps/cic-ussd/cic_ussd/metadata/base.py b/apps/cic-ussd/cic_ussd/metadata/base.py index c422ee7a..acba2655 100644 --- a/apps/cic-ussd/cic_ussd/metadata/base.py +++ b/apps/cic-ussd/cic_ussd/metadata/base.py @@ -1,99 +1,30 @@ # standard imports -import json import logging -import os -from typing import Dict, Union -# third-part imports -from cic_types.models.person import generate_metadata_pointer, Person +# external imports +from cic_types.condiments import MetadataPointer +from cic_types.ext.metadata import MetadataRequestsHandler +from cic_types.processor import generate_metadata_pointer # local imports from cic_ussd.cache import cache_data, get_cached_data -from cic_ussd.http.requests import error_handler, make_request -from cic_ussd.metadata.signer import Signer logg = logging.getLogger(__file__) -class Metadata: - """ - :cvar base_url: The base url or the metadata server. - :type base_url: str - """ +class UssdMetadataHandler(MetadataRequestsHandler): + def __init__(self, cic_type: MetadataPointer, identifier: bytes): + super().__init__(cic_type, identifier) - base_url = None - - -class MetadataRequestsHandler(Metadata): - - def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'): - """""" - self.cic_type = cic_type - self.engine = engine - self.headers = { - 'X-CIC-AUTOMERGE': 'server', - 'Content-Type': 'application/json' - } - self.identifier = identifier - self.metadata_pointer = generate_metadata_pointer( - identifier=self.identifier, - cic_type=self.cic_type - ) - if self.base_url: - self.url = os.path.join(self.base_url, self.metadata_pointer) - - def create(self, data: Union[Dict, str]): - """""" - data = json.dumps(data).encode('utf-8') - result = make_request(method='POST', url=self.url, data=data, headers=self.headers) - - error_handler(result=result) - metadata = result.json() - return self.edit(data=metadata) - - def edit(self, data: Union[Dict, str]): - """""" - cic_meta_signer = Signer() - signature = cic_meta_signer.sign_digest(data=data) - algorithm = cic_meta_signer.get_operational_key().get('algo') - formatted_data = { - 'm': json.dumps(data), - 's': { - 'engine': self.engine, - 'algo': algorithm, - 'data': signature, - 'digest': data.get('digest'), - } - } - formatted_data = json.dumps(formatted_data) - result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers) - logg.info(f'signed metadata submission status: {result.status_code}.') - error_handler(result=result) - try: - decoded_identifier = self.identifier.decode("utf-8") - except UnicodeDecodeError: - decoded_identifier = self.identifier.hex() - logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.') - return result - - def query(self): - """""" - result = make_request(method='GET', url=self.url) - error_handler(result=result) - result_data = result.json() - if not isinstance(result_data, dict): - raise ValueError(f'Invalid result data object: {result_data}.') - if result.status_code == 200: - if self.cic_type == ':cic.person': - person = Person() - person_data = person.deserialize(person_data=result_data) - serialized_person_data = person_data.serialize() - data = json.dumps(serialized_person_data) - else: - data = json.dumps(result_data) - cache_data(key=self.metadata_pointer, data=data) - logg.debug(f'caching: {data} with key: {self.metadata_pointer}') - return result_data + def cache_metadata(self, data: str): + """ + :param data: + :type data: + :return: + :rtype: + """ + cache_data(self.metadata_pointer, data) + logg.debug(f'caching: {data} with key: {self.metadata_pointer}') def get_cached_metadata(self): """""" diff --git a/apps/cic-ussd/cic_ussd/metadata/custom.py b/apps/cic-ussd/cic_ussd/metadata/custom.py index 6ab13d00..e653588b 100644 --- a/apps/cic-ussd/cic_ussd/metadata/custom.py +++ b/apps/cic-ussd/cic_ussd/metadata/custom.py @@ -1,12 +1,13 @@ # standard imports # external imports +from cic_types.condiments import MetadataPointer # local imports -from .base import MetadataRequestsHandler +from .base import UssdMetadataHandler -class CustomMetadata(MetadataRequestsHandler): +class CustomMetadata(UssdMetadataHandler): def __init__(self, identifier: bytes): - super().__init__(cic_type=':cic.custom', identifier=identifier) + super().__init__(cic_type=MetadataPointer.CUSTOM, identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/metadata/person.py b/apps/cic-ussd/cic_ussd/metadata/person.py index dff4fddc..f7762a69 100644 --- a/apps/cic-ussd/cic_ussd/metadata/person.py +++ b/apps/cic-ussd/cic_ussd/metadata/person.py @@ -1,12 +1,13 @@ # standard imports # external imports +from cic_types.condiments import MetadataPointer # local imports -from .base import MetadataRequestsHandler +from .base import UssdMetadataHandler -class PersonMetadata(MetadataRequestsHandler): +class PersonMetadata(UssdMetadataHandler): def __init__(self, identifier: bytes): - super().__init__(cic_type=':cic.person', identifier=identifier) + super().__init__(cic_type=MetadataPointer.PERSON, identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/metadata/phone.py b/apps/cic-ussd/cic_ussd/metadata/phone.py index d1de6c52..1d4cb2a5 100644 --- a/apps/cic-ussd/cic_ussd/metadata/phone.py +++ b/apps/cic-ussd/cic_ussd/metadata/phone.py @@ -2,12 +2,13 @@ import logging # external imports +from cic_types.condiments import MetadataPointer # local imports -from .base import MetadataRequestsHandler +from .base import UssdMetadataHandler -class PhonePointerMetadata(MetadataRequestsHandler): +class PhonePointerMetadata(UssdMetadataHandler): def __init__(self, identifier: bytes): - super().__init__(cic_type=':cic.phone', identifier=identifier) + super().__init__(cic_type=MetadataPointer.PHONE, identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/metadata/preferences.py b/apps/cic-ussd/cic_ussd/metadata/preferences.py index 300d2edf..21a086de 100644 --- a/apps/cic-ussd/cic_ussd/metadata/preferences.py +++ b/apps/cic-ussd/cic_ussd/metadata/preferences.py @@ -1,13 +1,13 @@ # standard imports # external imports -import celery +from cic_types.condiments import MetadataPointer # local imports -from .base import MetadataRequestsHandler +from .base import UssdMetadataHandler -class PreferencesMetadata(MetadataRequestsHandler): +class PreferencesMetadata(UssdMetadataHandler): def __init__(self, identifier: bytes): - super().__init__(cic_type=':cic.preferences', identifier=identifier) + super().__init__(cic_type=MetadataPointer.PREFERENCES, identifier=identifier) diff --git a/apps/cic-ussd/cic_ussd/metadata/signer.py b/apps/cic-ussd/cic_ussd/metadata/signer.py deleted file mode 100644 index 7b55d8bb..00000000 --- a/apps/cic-ussd/cic_ussd/metadata/signer.py +++ /dev/null @@ -1,60 +0,0 @@ -# standard imports -import json -import logging -from typing import Optional -from urllib.request import Request, urlopen - -# third-party imports -import gnupg - -# local imports - -logg = logging.getLogger() - - -class Signer: - """ - :cvar gpg_path: - :type gpg_path: - :cvar gpg_passphrase: - :type gpg_passphrase: - :cvar key_file_path: - :type key_file_path: - - """ - gpg_path: str = None - gpg_passphrase: str = None - key_file_path: str = None - - def __init__(self): - self.gpg = gnupg.GPG(gnupghome=self.gpg_path) - - with open(self.key_file_path, 'r') as key_file: - self.key_data = key_file.read() - - def get_operational_key(self): - """ - :return: - :rtype: - """ - # import key data into keyring - self.gpg.import_keys(key_data=self.key_data) - gpg_keys = self.gpg.list_keys() - key_algorithm = gpg_keys[0].get('algo') - key_id = gpg_keys[0].get("keyid") - logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}') - return gpg_keys[0] - - def sign_digest(self, data: dict): - """ - :param data: - :type data: - :return: - :rtype: - """ - digest = data['digest'] - key_id = self.get_operational_key().get('keyid') - signature = self.gpg.sign(digest, passphrase=self.gpg_passphrase, keyid=key_id) - return str(signature) - - diff --git a/apps/cic-ussd/cic_ussd/tasks/metadata.py b/apps/cic-ussd/cic_ussd/tasks/metadata.py index bd3b4d1b..77bdb762 100644 --- a/apps/cic-ussd/cic_ussd/tasks/metadata.py +++ b/apps/cic-ussd/cic_ussd/tasks/metadata.py @@ -1,15 +1,17 @@ # standard imports +import json import logging # third-party imports import celery +from cic_types.models.person import Person # local imports from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata from cic_ussd.tasks.base import CriticalMetadataTask celery_app = celery.current_app -logg = logging.getLogger().getChild(__name__) +logg = logging.getLogger(__file__) @celery_app.task @@ -22,7 +24,13 @@ def query_person_metadata(blockchain_address: str): """ identifier = bytes.fromhex(blockchain_address) person_metadata_client = PersonMetadata(identifier=identifier) - person_metadata_client.query() + response = person_metadata_client.query() + data = response.json() + person = Person() + person_data = person.deserialize(person_data=data) + serialized_person_data = person_data.serialize() + data = json.dumps(serialized_person_data) + person_metadata_client.cache_metadata(data=data) @celery_app.task @@ -76,6 +84,9 @@ def query_preferences_metadata(blockchain_address: str): :type blockchain_address: str | Ox-hex """ identifier = bytes.fromhex(blockchain_address) - logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.') - person_metadata_client = PreferencesMetadata(identifier=identifier) - return person_metadata_client.query() + logg.debug(f'retrieving preferences metadata for address: {blockchain_address}.') + preferences_metadata_client = PreferencesMetadata(identifier=identifier) + response = preferences_metadata_client.query() + data = json.dumps(response.json()) + preferences_metadata_client.cache_metadata(data) + return data diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 66ac133a..512ec24d 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -4,10 +4,10 @@ billiard==3.6.4.0 bcrypt==3.2.0 celery==4.4.7 cffi==1.14.6 -cic-eth[services]~=0.12.4a7 +cic-eth[services]~=0.12.4a11 cic-notify~=0.4.0a10 -cic-types~=0.1.0a15 -confini>=0.4.1a1,<0.5.0 +cic-types~=0.2.0a3 +confini>=0.3.6rc4,<0.5.0 phonenumbers==8.12.12 psycopg2==2.8.6 python-i18n[YAML]==0.3.9 diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_base.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_base.py index f7633e04..f3ad8e62 100644 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_base.py +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_base.py @@ -5,24 +5,25 @@ import os # external imports import requests_mock from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer from cic_types.processor import generate_metadata_pointer # local imports -from cic_ussd.metadata.base import MetadataRequestsHandler +from cic_ussd.metadata.base import UssdMetadataHandler # external imports -def test_metadata_requests_handler(activated_account, - init_cache, - load_config, - person_metadata, - setup_metadata_request_handler, - setup_metadata_signer): +def test_ussd_metadata_handler(activated_account, + init_cache, + load_config, + person_metadata, + setup_metadata_request_handler, + setup_metadata_signer): identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) - cic_type = ':cic.person' - metadata_client = MetadataRequestsHandler(cic_type, identifier) + cic_type = MetadataPointer.PERSON + metadata_client = UssdMetadataHandler(cic_type, identifier) assert metadata_client.cic_type == cic_type assert metadata_client.engine == 'pgp' assert metadata_client.identifier == identifier @@ -38,7 +39,5 @@ def test_metadata_requests_handler(activated_account, assert result.status_code == 200 person_metadata.pop('digest') request_mocker.register_uri('GET', metadata_client.url, status_code=200, reason='OK', json=person_metadata) - result = metadata_client.query() + result = metadata_client.query().json() assert result == person_metadata - cached_metadata = metadata_client.get_cached_metadata() - assert json.loads(cached_metadata) == person_metadata diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py index 846fbca7..0d28e8ad 100644 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py @@ -1,7 +1,7 @@ # standard imports import os # external imports -from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer from cic_types.processor import generate_metadata_pointer # local imports @@ -11,8 +11,8 @@ from cic_ussd.metadata import CustomMetadata def test_custom_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): - cic_type = ':cic.custom' - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + cic_type = MetadataPointer.CUSTOM + identifier = bytes.fromhex(activated_account.blockchain_address) custom_metadata_client = CustomMetadata(identifier) assert custom_metadata_client.cic_type == cic_type assert custom_metadata_client.engine == 'pgp' diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_person.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_person.py index 6a21810e..01e0ebd8 100644 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_person.py +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_person.py @@ -1,7 +1,7 @@ # standard imports import os # external imports -from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer from cic_types.processor import generate_metadata_pointer # local imports @@ -11,8 +11,8 @@ from cic_ussd.metadata import PersonMetadata def test_person_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): - cic_type = ':cic.person' - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + cic_type = MetadataPointer.PERSON + identifier = bytes.fromhex(activated_account.blockchain_address) person_metadata_client = PersonMetadata(identifier) assert person_metadata_client.cic_type == cic_type assert person_metadata_client.engine == 'pgp' diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py index 3bee949a..5ff0a721 100644 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py @@ -1,7 +1,7 @@ # standard imports import os # external imports -from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer from cic_types.processor import generate_metadata_pointer # local imports @@ -12,8 +12,8 @@ from cic_ussd.metadata import PhonePointerMetadata def test_phone_pointer_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): - cic_type = ':cic.phone' - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + cic_type = MetadataPointer.PHONE + identifier = bytes.fromhex(activated_account.blockchain_address) phone_pointer_metadata = PhonePointerMetadata(identifier) assert phone_pointer_metadata.cic_type == cic_type assert phone_pointer_metadata.engine == 'pgp' diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py index 2094f035..bc8d3959 100644 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py @@ -1,7 +1,7 @@ # standard imports import os # external imports -from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer from cic_types.processor import generate_metadata_pointer # local imports @@ -11,8 +11,8 @@ from cic_ussd.metadata import PreferencesMetadata def test_preferences_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): - cic_type = ':cic.preferences' - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + cic_type = MetadataPointer.PREFERENCES + identifier = bytes.fromhex(activated_account.blockchain_address) preferences_metadata_client = PreferencesMetadata(identifier) assert preferences_metadata_client.cic_type == cic_type assert preferences_metadata_client.engine == 'pgp' diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py deleted file mode 100644 index 88a5c9a9..00000000 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py +++ /dev/null @@ -1,17 +0,0 @@ -# standard imports -import shutil - -# third-party imports - -# local imports -from cic_ussd.metadata.signer import Signer - - -def test_client(load_config, setup_metadata_signer, person_metadata): - signer = Signer() - gpg = signer.gpg - assert signer.key_data is not None - gpg.import_keys(key_data=signer.key_data) - gpg_keys = gpg.list_keys() - assert signer.get_operational_key() == gpg_keys[0] - shutil.rmtree(Signer.gpg_path) diff --git a/apps/cic-ussd/tests/fixtures/metadata.py b/apps/cic-ussd/tests/fixtures/metadata.py index d7925029..e4bf7c47 100644 --- a/apps/cic-ussd/tests/fixtures/metadata.py +++ b/apps/cic-ussd/tests/fixtures/metadata.py @@ -6,33 +6,19 @@ import tempfile # external imports import pytest from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer from cic_types.processor import generate_metadata_pointer # local imports -from cic_ussd.metadata import Metadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata -from cic_ussd.metadata.signer import Signer +from cic_ussd.metadata import PersonMetadata, PhonePointerMetadata, PreferencesMetadata logg = logging.getLogger(__name__) -@pytest.fixture(scope='function') -def setup_metadata_signer(load_config): - temp_dir = tempfile.mkdtemp(dir='/tmp') - logg.debug(f'Created temp dir: {temp_dir}') - Signer.gpg_path = temp_dir - Signer.gpg_passphrase = load_config.get('PGP_PASSPHRASE') - Signer.key_file_path = os.path.join(load_config.get('PGP_KEYS_PATH'), load_config.get('PGP_PRIVATE_KEYS')) - - -@pytest.fixture(scope='function') -def setup_metadata_request_handler(load_config): - Metadata.base_url = load_config.get('CIC_META_URL') - - @pytest.fixture(scope='function') def account_phone_pointer(activated_account): identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) - return generate_metadata_pointer(identifier, ':cic.phone') + return generate_metadata_pointer(identifier, MetadataPointer.PERSON) @pytest.fixture(scope='function') diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 56f7da67..3d9ab53f 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,6 +1,6 @@ -cic-eth[tools]==0.12.4a8 -chainlib-eth>=0.0.9rc4,<0.0.10 -chainlib==0.0.9rc1,<0.0.10 +cic-eth[tools]==0.12.4a11 +chainlib-eth>=0.0.9rc4,<0.1.0 +chainlib==0.0.9rc1,<0.1.0 eth-erc20>=0.1.2a3,<0.2.0 erc20-demurrage-token>=0.0.5a2,<0.1.0 eth-address-index>=0.2.4a1,<0.3.0