From 23567905a1e43b5212fc3074c0e00a695ca2889a Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 11 Oct 2021 15:28:09 +0200 Subject: [PATCH] Run extensions processing from core processor --- cic/ext/eth/__init__.py | 54 +++++++++++++++++++++------------- cic/meta.py | 8 +++++ cic/output.py | 2 +- cic/processor.py | 42 +++++++++++++++++++------- tests/base_cic.py | 4 +-- tests/eth/base_eth.py | 11 ++++--- tests/eth/test_eth_full.py | 46 ++++++++++++++++++++++------- tests/eth/test_eth_offline.py | 17 ++++++----- tests/eth/test_eth_sign.py | 6 ++-- tests/test_meta.py | 12 ++++++-- tests/test_output.py | 31 +++++++++++++++++++ tests/test_processor.py | 53 +++++++++++++++++++++++++++++++++ tests/testdata/proof/meta.json | 7 +++++ 13 files changed, 233 insertions(+), 60 deletions(-) create mode 100644 tests/test_output.py create mode 100644 tests/test_processor.py create mode 100644 tests/testdata/proof/meta.json diff --git a/cic/ext/eth/__init__.py b/cic/ext/eth/__init__.py index c2cbd8d..3585ee6 100644 --- a/cic/ext/eth/__init__.py +++ b/cic/ext/eth/__init__.py @@ -27,7 +27,7 @@ logg = logging.getLogger(__name__) class CICEth: - def __init__(self, core_processor, chain_spec, resources, proof, signer=None, metadata=None, rpc=None, fee_oracle=None): + def __init__(self, chain_spec, resources, proof, signer=None, rpc=None, fee_oracle=None, outputs_writer=None): """resources will be modified """ self.resources = resources @@ -35,14 +35,13 @@ class CICEth: self.chain_spec = chain_spec self.signer = signer self.rpc = rpc - self.core_processor = core_processor self.fee_oracle = fee_oracle - self.metadata = metadata self.token_details = None self.token_address = None self.token_code = None self.outputs = [] self.tx_format = TxFormat.RAW_ARGS + self.outputs_writer = outputs_writer if self.rpc != None: self.tx_format = TxFormat.JSONRPC elif self.signer != None: @@ -122,10 +121,20 @@ class CICEth: return (args, args_types, positions) + def add_outputs(self, k, v): + logg.debug('adding outputs {} {}'.format(k, v)) + self.outputs.append((k, v)) + + + def get_outputs(self): + return self.outputs + + def process_token(self, writer=None): if writer == None: - writer = self.core_processor.writer() + writer = self.outputs_writer + logg.debug('ZZZZZZZZ token details {}'.format(self.token_details)) (args, args_types, positions) = self.__order_args() enc = ABIContractEncoder() @@ -158,13 +167,13 @@ class CICEth: if r == None: r = code - self.outputs.append(r) + self.add_outputs('token', r) return r def process_token_index(self, writer=None): if writer == None: - writer = self.core_processor.writer() + writer = self.outputs_writer signer_address = self.resources['token_index']['key_address'] contract_address = self.resources['token_index']['reference'] @@ -184,13 +193,13 @@ class CICEth: else: r = o - self.outputs.append(r) + self.add_outputs('token_index', r) return r def process_address_declarator(self, writer=None): if writer == None: - writer = self.core_processor.writer() + writer = self.outputs_writer signer_address = self.resources['address_declarator']['key_address'] contract_address = self.resources['address_declarator']['reference'] @@ -201,21 +210,26 @@ class CICEth: nonce_oracle = RPCNonceOracle(signer_address, conn=self.rpc) c = Declarator(self.chain_spec, signer=self.signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) - r = [] + results = [] for proof in self.proof.get(): o = c.add_declaration(contract_address, signer_address, self.token_address, proof, tx_format=self.tx_format) + r = None if self.rpc != None: - r.append(self.rpc.do(o[1])) + r = self.rpc.do(o[1]) elif self.signer != None: - r.append(o[1]) + r = o[1] else: - r.append(o) + r = o + self.add_outputs('address_declarator', r) + results.append(r) - self.outputs += r - return r + return results - def process(self): + def process(self, writer=None): + if writer == None: + writer = self.outputs_writer + tasks = [] self.token_address = self.resources['token']['reference'] @@ -230,12 +244,10 @@ class CICEth: for task in tasks: logg.debug('ciceth adapter process {}'.format(task)) - r = getattr(self, 'process_' + task)(writer=self.core_processor.writer()) + r = getattr(self, 'process_' + task)(writer=writer) - self.core_processor.process() - - return self.outputs + return self.token_address -def new(core_processor, resources, proof, signer_hint=None): - return CICEth(core_processor, resources, proof, signer=None) +def new(resources, proof, signer_hint=None): + return CICEth(resources, proof, signer=None) diff --git a/cic/meta.py b/cic/meta.py index 37f17bf..8457803 100644 --- a/cic/meta.py +++ b/cic/meta.py @@ -62,6 +62,14 @@ class Meta(Data): 'contact': self.contact, } + + def process(self, token_address=None, writer=None): + k = self.reference(token_address) + v = json.dumps(self.asdict()) + writer.write(k, v) + + return (k, v) + def __str__(self): s = "contact.name = {}\n".format(self.name) diff --git a/cic/output.py b/cic/output.py index 7334aa3..c74acba 100644 --- a/cic/output.py +++ b/cic/output.py @@ -10,7 +10,7 @@ class KVWriter: def write(self, k, v): - fp = open(self.path, str(k)) + fp = os.path.join(self.path, str(k)) f = open(fp, 'w') f.write(v) f.close() diff --git a/cic/processor.py b/cic/processor.py index 636dcd4..74afd93 100644 --- a/cic/processor.py +++ b/cic/processor.py @@ -1,22 +1,38 @@ +# standard imports +import logging + +logg = logging.getLogger(__name__) + + class Processor: - def __init__(self, token=None, metadata=None, outputs_writer=None): - self.token = token + def __init__(self, metadata=None, outputs_writer=None, extensions=[]): self.token_address = None self.metadata = metadata + self.extensions = extensions + self.outputs = [] self.__outputs_writer = outputs_writer - def set_token_address(address): - self.token_address = address - - def writer(self): return self.__outputs_writer + def can_process(self): + return self.token_address != None + + def process_metadata(self, writer=None): - pass + if not self.can_process(): + raise RuntimeError('incomplete processing state for metadata') + + + def get_outputs(self): + outputs = [] + for ext in self.extensions: + outputs += ext.outputs + outputs += self.outputs + return outputs def process(self): @@ -24,7 +40,13 @@ class Processor: 'metadata', ] - for task in tasks: - getattr(self, 'process_' + task)(writer=self.__outputs_writer) + for ext in self.extensions: + token_address = ext.process() - + for task in tasks: + a = getattr(self, task) + if a == None: + logg.debug('skipping missing task receiver "{}"'.format(task)) + continue + v = a.process(token_address=token_address, writer=self.__outputs_writer) + self.outputs.append(v) diff --git a/tests/base_cic.py b/tests/base_cic.py index 2ebb6a0..666ea9e 100644 --- a/tests/base_cic.py +++ b/tests/base_cic.py @@ -36,8 +36,8 @@ class TestCICBase(unittest.TestCase): self.token_index_address = add_0x(random.randbytes(32).hex()) self.address_declarator_address = add_0x(random.randbytes(32).hex()) - d = tempfile.mkdtemp() - self.outputs_writer = KVWriter(d) + self.outputs_dir = tempfile.mkdtemp() + self.outputs_writer = KVWriter(self.outputs_dir) self.core_processor = Processor(self.token_address, outputs_writer=self.outputs_writer) self.resources = { diff --git a/tests/eth/base_eth.py b/tests/eth/base_eth.py index d6f2e74..eb7fed0 100644 --- a/tests/eth/base_eth.py +++ b/tests/eth/base_eth.py @@ -42,9 +42,6 @@ class TestCICEthBase(EthTesterCase): random.seed(42) self.initial_description = add_0x(random.randbytes(32).hex()) self.token_address = add_0x(random.randbytes(20).hex()) - d = tempfile.mkdtemp() - self.outputs_writer = KVWriter(d) - self.core_processor = Processor(self.token_address, outputs_writer=self.outputs_writer) addresses = self.keystore.list() @@ -92,13 +89,19 @@ class TestCICEthBase(EthTesterCase): self.proofs = Proof(proof_dir, attachments=attach) self.proofs.load() + d = tempfile.mkdtemp() + self.outputs_writer = KVWriter(d) + class TestCICEthTokenBase(TestCICEthBase): def setUp(self): super(TestCICEthTokenBase, self).setUp() + self.resources['token']['reference'] = None - self.adapter = CICEth(self.core_processor, self.chain_spec, self.resources, self.proofs) + self.adapter = CICEth(self.chain_spec, self.resources, self.proofs, outputs_writer=self.outputs_writer) self.token_name = 'FOotoken' self.token_symbol = 'FOO' self.token_precision = 8 + + self.core_processor = Processor(outputs_writer=self.outputs_writer, extensions=[self.adapter]) diff --git a/tests/eth/test_eth_full.py b/tests/eth/test_eth_full.py index 0b34c1c..4a5ccd4 100644 --- a/tests/eth/test_eth_full.py +++ b/tests/eth/test_eth_full.py @@ -27,6 +27,7 @@ from giftable_erc20_token import GiftableToken # local imports from cic.ext.eth import CICEth +from cic.processor import Processor # test imports from tests.eth.base_eth import TestCICEthTokenBase @@ -42,7 +43,8 @@ class TestCICEthRPC(TestCICEthTokenBase): nonce_oracle = RPCNonceOracle(self.accounts[0], self.rpc) gas_oracle = RPCGasOracle(self.rpc) - self.adapter = CICEth(self.core_processor, self.chain_spec, self.resources, self.proofs, signer=self.signer, rpc=self.rpc, fee_oracle=gas_oracle) + self.adapter = CICEth(self.chain_spec, self.resources, self.proofs, signer=self.signer, rpc=self.rpc, fee_oracle=gas_oracle) + self.core_processor = Processor(outputs_writer=self.outputs_writer, extensions=[self.adapter]) def test_rpc_process_notoken(self): @@ -62,10 +64,10 @@ class TestCICEthRPC(TestCICEthTokenBase): self.token_index_address = r['contract_address'] logg.debug('token index deployed at {}'.format(self.token_index_address)) - results = self.adapter.process() - for hsh in results: - logg.debug('hsh {}'.format(hsh)) - o = receipt(hsh) + self.adapter.process() + results = self.adapter.get_outputs() + for v in results: + o = receipt(v[1]) r = self.rpc.do(o) self.assertEqual(r['status'], 1) @@ -73,14 +75,38 @@ class TestCICEthRPC(TestCICEthTokenBase): def test_rpc_process_withtoken(self): self.adapter.fee_oracle = OverrideGasOracle(limit=GiftableToken.gas(), conn=self.rpc) self.adapter.prepare_token(self.token_name, self.token_symbol, self.token_precision, GiftableToken.bytecode()) - results = self.adapter.process() - for hsh in results: - logg.debug('hsh {}'.format(hsh)) - o = receipt(hsh) + self.adapter.process() + + results = self.adapter.get_outputs() + for v in results: + o = receipt(v[1]) r = self.rpc.do(o) self.assertEqual(r['status'], 1) - o = receipt(results[0]) + o = receipt(results[0][1]) + r = self.rpc.do(o) + token_contract_address = r['contract_address'] + + c = TokenUniqueSymbolIndex(self.chain_spec) + o = c.address_of(self.token_index_address, 'FOO', sender_address=self.accounts[0]) + r = self.rpc.do(o) + token_contract_address_lookup = c.parse_address_of(r) + self.assertEqual(strip_0x(token_contract_address), strip_0x(token_contract_address_lookup)) + + + def test_rpc_process_top(self): + self.adapter.fee_oracle = OverrideGasOracle(limit=GiftableToken.gas(), conn=self.rpc) + self.adapter.prepare_token(self.token_name, self.token_symbol, self.token_precision, GiftableToken.bytecode()) + self.core_processor.process() + + results = self.adapter.get_outputs() + for v in results: + logg.debug('results {}'.format(v)) + o = receipt(v[1]) + r = self.rpc.do(o) + self.assertEqual(r['status'], 1) + + o = receipt(results[0][1]) r = self.rpc.do(o) token_contract_address = r['contract_address'] diff --git a/tests/eth/test_eth_offline.py b/tests/eth/test_eth_offline.py index 1c16108..cf32768 100644 --- a/tests/eth/test_eth_offline.py +++ b/tests/eth/test_eth_offline.py @@ -4,6 +4,7 @@ import logging # local imports from cic.ext.eth import CICEth +from cic.processor import Processor # tests imports from tests.eth.base_eth import TestCICEthBase @@ -16,21 +17,23 @@ class TestCICEthOffline(TestCICEthBase): def setUp(self): super(TestCICEthOffline, self).setUp() - self.adapter = CICEth(self.core_processor, self.chain_spec, self.resources, self.proofs) + self.adapter = CICEth(self.chain_spec, self.resources, self.proofs) + self.core_processor = Processor(outputs_writer=self.outputs_writer, extensions=[self.adapter]) def test_offline_token_index(self): self.adapter.token_address = self.token_address - v = self.adapter.process_token_index() - self.assertEqual(self.adapter.outputs[0], v) - self.assertEqual(self.adapter.outputs[0][:8], '4420e486') + self.adapter.process_token_index() + self.assertEqual(self.adapter.outputs[0][0], 'token_index') + self.assertEqual(self.adapter.outputs[0][1][:8], '4420e486') def test_offline_address_declarator(self): self.adapter.token_address = self.token_address - v = self.adapter.process_address_declarator() - self.assertEqual(v[0][:8], 'ae47ece0') - self.assertEqual(len(v), 3) + self.adapter.process_address_declarator() + self.assertEqual(self.adapter.outputs[0][0], 'address_declarator') + self.assertEqual(self.adapter.outputs[0][1][:8], 'ae47ece0') + self.assertEqual(len(self.adapter.outputs), 3) diff --git a/tests/eth/test_eth_sign.py b/tests/eth/test_eth_sign.py index d080636..054a6bf 100644 --- a/tests/eth/test_eth_sign.py +++ b/tests/eth/test_eth_sign.py @@ -11,6 +11,7 @@ from hexathon import ( # local imports from cic.ext.eth import CICEth +from cic.processor import Processor # tests imports from tests.eth.base_eth import TestCICEthBase @@ -23,13 +24,14 @@ class TestCICEthSign(TestCICEthBase): def setUp(self): super(TestCICEthSign, self).setUp() - self.adapter = CICEth(self.core_processor, self.chain_spec, self.resources, self.proofs, signer=self.signer) + self.adapter = CICEth(self.chain_spec, self.resources, self.proofs, signer=self.signer) + self.core_processor = Processor(outputs_writer=self.outputs_writer, extensions=[self.adapter]) def test_sign_token_index(self): self.adapter.token_address = self.token_address v = self.adapter.process_token_index() - tx_raw = bytes.fromhex(strip_0x(self.adapter.outputs[0])) + tx_raw = bytes.fromhex(strip_0x(self.adapter.outputs[0][1])) tx = unpack(tx_raw, self.chain_spec) self.assertEqual(strip_0x(tx['data'])[:8], '4420e486') diff --git a/tests/test_meta.py b/tests/test_meta.py index d6e1e63..6858f0b 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,15 +1,19 @@ # standard imports import unittest import logging +import os # external imports from hexathon import strip_0x # local imports -from cic.output import KVWriter +from cic.meta import Meta # test imports -from tests.base_cic import TestCICBase +from tests.base_cic import ( + TestCICBase, + test_data_dir, + ) logging.basicConfig(level=logging.DEBUG) logg = logging.getLogger() @@ -18,7 +22,9 @@ logg = logging.getLogger() class TestCICMeta(TestCICBase): def test_meta(self): - pass + fp = os.path.join(test_data_dir, 'proof') + m = Meta(fp) + m.load() if __name__ == '__main__': diff --git a/tests/test_output.py b/tests/test_output.py new file mode 100644 index 0000000..6d7db69 --- /dev/null +++ b/tests/test_output.py @@ -0,0 +1,31 @@ +# standard imports +import os +import unittest +import logging + +# external imports +from hexathon import strip_0x + +# local imports +from cic.output import KVWriter + +# test imports +from tests.base_cic import TestCICBase + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + + +class TestCICOutput(TestCICBase): + + def test_output_file(self): + self.outputs_writer.write('foo', 'bar') + fp = os.path.join(self.outputs_dir, 'foo') + f = open(fp, 'r') + v = f.read() + f.close() + self.assertEqual(v, 'bar') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_processor.py b/tests/test_processor.py new file mode 100644 index 0000000..146e2ae --- /dev/null +++ b/tests/test_processor.py @@ -0,0 +1,53 @@ +# standard imports +import unittest +import logging +import os +import json + +# external imports +from hexathon import strip_0x + +# local imports +from cic.processor import Processor +from cic.meta import Meta + +# test imports +from tests.base_cic import ( + TestCICBase, + test_data_dir, + ) + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + +class MockExt: + + def __init__(self, address): + self.address = address + + def process(self): + return self.address + + +class TestCICProcessor(TestCICBase): + + def test_processor(self): + fp = os.path.join(test_data_dir, 'proof') + m = Meta(fp) + m.load() + + mock_ext = MockExt(self.token_address) + p = Processor(metadata=m, outputs_writer=self.outputs_writer, extensions=[mock_ext]) + p.process() + + meta_reference = m.reference(self.token_address) + + fp = os.path.join(self.outputs_dir, meta_reference) + f = open(fp, 'r') + o = json.load(f) + f.close() + self.assertEqual(m.asdict(), o) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/testdata/proof/meta.json b/tests/testdata/proof/meta.json new file mode 100644 index 0000000..0023838 --- /dev/null +++ b/tests/testdata/proof/meta.json @@ -0,0 +1,7 @@ +{ + "name": "", + "location": "", + "country_code": "", + "contact": { + } +}