From eb42095a74d3f3fc5c7056120b89628855feaeea Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 22 Oct 2021 16:42:38 +0200 Subject: [PATCH] Add dual writer for key and immutable storage --- cic/cmd/export.py | 9 ++-- cic/data/proof_template_v0.json | 3 +- cic/ext/eth/__init__.py | 40 ++++++++++-------- cic/meta.py | 17 +++++++- cic/output.py | 46 +++++++++++++++++++- cic/proof.py | 75 +++++++++++++++++++++------------ tests/base_cic.py | 4 +- tests/test_proof.py | 11 +---- 8 files changed, 139 insertions(+), 66 deletions(-) diff --git a/cic/cmd/export.py b/cic/cmd/export.py index a4f0307..21ab9c2 100644 --- a/cic/cmd/export.py +++ b/cic/cmd/export.py @@ -12,7 +12,10 @@ from cic import ( Proof, Processor, ) -from cic.output import HTTPWriter +from cic.output import ( + HTTPWriter, + KeyedWriterFactory, + ) from cic.meta import ( Meta, MetadataWriter, @@ -66,8 +69,8 @@ def execute(config, eargs): MetadataSigner.gpg_path = os.path.join('/tmp') MetadataSigner.key_file_path = '/home/lash/src/client/cic/grassrootseconomics/cic-internal-integration/apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc' MetadataSigner.gpg_passphrase = 'merman' - writers['proof'] = HTTPWriter - writers['attachment'] = HTTPWriter + writers['proof'] = KeyedWriterFactory(MetadataWriter, HTTPWriter).new + writers['attachment'] = KeyedWriterFactory(None, HTTPWriter).new writers['meta'] = MetadataWriter output_writer_path_meta = eargs.metadata_endpoint diff --git a/cic/data/proof_template_v0.json b/cic/data/proof_template_v0.json index 5800c5a..2f1b58f 100644 --- a/cic/data/proof_template_v0.json +++ b/cic/data/proof_template_v0.json @@ -2,5 +2,6 @@ "version": 0, "namespace": "ge", "issuer": "", - "description": "" + "description": null, + "proofs": [] } diff --git a/cic/ext/eth/__init__.py b/cic/ext/eth/__init__.py index 0d8a393..fe256ca 100644 --- a/cic/ext/eth/__init__.py +++ b/cic/ext/eth/__init__.py @@ -224,24 +224,28 @@ class CICEth(Extension): c = Declarator(self.chain_spec, signer=self.signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) results = [] - (main_proof, all_proofs) = self.proof.get() - for proof in all_proofs: - logg.debug('proof {} '.format(proof)) - k = 'address_declarator_' + proof - o = c.add_declaration(contract_address, signer_address, self.token_address, proof, tx_format=self.tx_format) - r = None - if self.rpc != None: - r = self.rpc.do(o[1]) - self.rpc.wait(r) - elif self.signer != None: - r = o[1] - else: - r = o - self.add_outputs(k, r) - results.append(r) - v = r.encode('utf-8') - if writer != None: - writer.write(k, v) + #(main_proof, all_proofs) = self.proof.get() + + #for proof in all_proofs: + #logg.debug('proof {} '.format(proof)) + + (k, v) = self.proof.root() + + fk = 'address_declarator_' + k + o = c.add_declaration(contract_address, signer_address, self.token_address, k, tx_format=self.tx_format) + r = None + if self.rpc != None: + r = self.rpc.do(o[1]) + self.rpc.wait(r) + elif self.signer != None: + r = o[1] + else: + r = o + self.add_outputs(fk, r) + results.append(r) + v = r.encode('utf-8') + if writer != None: + writer.write(fk, v) return results diff --git a/cic/meta.py b/cic/meta.py index b9c1c67..f80bf33 100644 --- a/cic/meta.py +++ b/cic/meta.py @@ -1,6 +1,8 @@ # standard imports import os import json +import logging +import base64 # external imports from cic_types import MetadataPointer @@ -14,6 +16,7 @@ from .base import ( data_dir, ) from cic.output import OutputWriter +logg = logging.getLogger(__name__) class Meta(Data): @@ -98,5 +101,15 @@ class MetadataWriter(OutputWriter): def write(self, k, v): rq = MetadataRequestsHandler(MetadataPointer.NONE, bytes.fromhex(k)) - return rq.create(json.loads(v.decode('utf-8'))) - rq = urllib.request.Request(self.path) + try: + v = v.decode('utf-8') + v = json.loads(v) + logg.debug('metadatawriter bindecode {} {}'.format(k, v)) + except UnicodeDecodeError: + v = base64.b64encode(v).decode('utf-8') + v = json.loads(json.dumps(v)) + logg.debug('metadatawriter b64encode {} {}'.format(k, v)) + r = rq.create(v) + logg.info('metadata submitted at {}'.format(k)) + return r + diff --git a/cic/output.py b/cic/output.py index 52bbe22..253d7fd 100644 --- a/cic/output.py +++ b/cic/output.py @@ -48,6 +48,48 @@ class HTTPWriter(OutputWriter): def write(self, k, v): - rq = urllib.request.Request(self.path, method='POST', data=v) + path = self.path + if k != None: + path = os.path.join(path, k) + logg.debug('http writer post {}'.format(path)) + rq = urllib.request.Request(path, method='POST', data=v) r = urllib.request.urlopen(rq) - logg.info('proof submited at {}'.format(r.read)) + logg.info('proof submited at {}'.format(r.read())) + + +class KeyedWriter(OutputWriter): + + def __init__(self, writer_keyed, writer_immutable): + self.writer_keyed = writer_keyed + self.writer_immutable = writer_immutable + + + def write(self, k, v): + logg.debug('writing keywriter {} {}'.format(k, v)) + if isinstance(v, str): + v = v.encode('utf-8') + if self.writer_keyed != None: + self.writer_keyed.write(k, v) + if self.writer_immutable != None: + self.writer_immutable.write(None, v) + + +class KeyedWriterFactory: + + def __init__(self, key_writer_constructor, immutable_writer_constructor, *args, **kwargs): + self.key_writer_constructor = key_writer_constructor + self.immutable_writer_constructor = immutable_writer_constructor + self.x = {} + for k in kwargs.keys(): + logg.debug('adding key {} t keyed writer factory'.format(k)) + self.x[k] = kwargs[k] + + + def new(self, path=None, *args, **kwargs): + writer_keyed = None + writer_immutable = None + if self.key_writer_constructor != None: + writer_keyed = self.key_writer_constructor(path, **self.x) + if self.immutable_writer_constructor != None: + writer_immutable = self.immutable_writer_constructor(path, **self.x) + return KeyedWriter(writer_keyed, writer_immutable) diff --git a/cic/proof.py b/cic/proof.py index 887dd00..d01d2e9 100644 --- a/cic/proof.py +++ b/cic/proof.py @@ -22,6 +22,7 @@ class Proof(Data): def __init__(self, path='.', attachments=None, writer=None): super(Proof, self).__init__() + self.proofs = [] self.namespace = 'ge' self.description = None self.path = path @@ -44,6 +45,14 @@ class Proof(Data): self.namespace = o['namespace'] self.issuer = o['issuer'] + if self.extra_attachments != None: + a = self.extra_attachments.asdict() + for k in a.keys(): + self.attachments[k] = a[k] + + hshs = self.__get_ordered_hashes() + self.proofs = list(map(strip_0x, hshs)) + self.inited = True @@ -67,6 +76,7 @@ class Proof(Data): 'namespace': self.namespace, 'description': self.description, 'issuer': self.issuer, + 'proofs': self.proofs, } @@ -77,50 +87,59 @@ class Proof(Data): return ks - def get(self): - v = self.asdict() - b = cbor2.dumps(v) +# def get(self): +# hsh = self.hash(b).hex() +# self.attachments[hsh] = self.temp_proof_path +# logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh)) - f = open(self.temp_proof_path, 'wb') + + def root(self): + v = self.asdict() + #b = cbor2.dumps(v) + b = json.dumps(v) + + f = open(self.temp_proof_path, 'w') f.write(b) f.close() - hsh = self.hash(b).hex() - self.attachments[hsh] = self.temp_proof_path - logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh)) + k = self.hash(b.encode('utf-8')) - if self.extra_attachments != None: - a = self.extra_attachments.asdict() - for k in a.keys(): - self.attachments[k] = a[k] - - hshs = self.__get_ordered_hashes() - - return (hsh, hshs) + return (k.hex(), b) def process(self, token_address=None, token_symbol=None, writer=None): if writer == None: writer = self.writer - (hsh, hshs) = self.get() - hshs = list(map(strip_0x, hshs)) - hshs_bin = list(map(bytes.fromhex, hshs)) - hshs_cat = b''.join(hshs_bin) + (k, v) = self.root() + writer.write(k, v) - f = open(self.temp_proof_path, 'rb') - v = f.read() - f.close() - writer.write(hsh, v) + token_symbol_bytes = token_symbol.encode('utf-8') + k = generate_metadata_pointer(token_symbol_bytes, MetadataPointer.TOKEN_PROOF_SYMBOL) + writer.write(k, v) - r = self.hash(hshs_cat) - r_hex = r.hex() + token_address_bytes = bytes.fromhex(strip_0x(token_address)) + k = generate_metadata_pointer(token_address_bytes, MetadataPointer.TOKEN_PROOF) + writer.write(k, v) - logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs)) +# (hsh, hshs) = self.get() + #hshs = list(map(strip_0x, hshs)) +# hshs_bin = list(map(bytes.fromhex, hshs)) +# hshs_cat = b''.join(hshs_bin) - writer.write(r_hex, hshs_cat) +# f = open(self.temp_proof_path, 'rb') +# v = f.read() +# f.close() +# writer.write(hsh, v) + +# r = self.hash(hshs_cat) +# r_hex = r.hex() + + #logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs)) + + #writer.write(r_hex, hshs_cat) - return r_hex + return k def __str__(self): diff --git a/tests/base_cic.py b/tests/base_cic.py index b996588..2819c6f 100644 --- a/tests/base_cic.py +++ b/tests/base_cic.py @@ -22,8 +22,7 @@ test_data_dir = os.path.join(test_base_dir, 'testdata') proof_hash = '0f6fc017f29caf512c0feaaf83bc10614b488311cace2973dc248dc24b01e04f' foo_hash = '2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae' bar_hash = 'fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9' -root_merged_hash = 'ba135f8518d36af5fa65c59317ea9602c4b654d998ea9097ecf81e0638a03441' -root_unmerged_hash = '68ccfe99fd905be439b09dcd780993865598605b8492462a6fc4f127b688fef6' +root_merged_hash = '4bd0ad4305a5fee20fb80e179a437c296f6a769ca376d746a3848a80e9b7a1a6' class TestCICBase(unittest.TestCase): @@ -37,6 +36,7 @@ class TestCICBase(unittest.TestCase): add_0x(random.randbytes(20).hex()), add_0x(random.randbytes(20).hex()), ] + self.token_symbol = 'FOO' self.token_address = add_0x(random.randbytes(32).hex()) self.token_index_address = add_0x(random.randbytes(32).hex()) self.address_declarator_address = add_0x(random.randbytes(32).hex()) diff --git a/tests/test_proof.py b/tests/test_proof.py index 8e28179..4e31018 100644 --- a/tests/test_proof.py +++ b/tests/test_proof.py @@ -12,7 +12,6 @@ from tests.base_cic import ( test_data_dir, TestCICBase, root_merged_hash, - root_unmerged_hash, ) logging.basicConfig(level=logging.DEBUG) @@ -20,14 +19,6 @@ logg = logging.getLogger() class TestProof(TestCICBase): - def test_proof_serialize(self): - proof_path = os.path.join(test_data_dir, 'proof') - c = Proof(path=proof_path, writer=self.outputs_writer) - c.load() - v = c.process() - self.assertEqual(v, root_unmerged_hash) - - def test_proof_serialize_merge(self): proof_path = os.path.join(test_data_dir, 'proof') @@ -36,7 +27,7 @@ class TestProof(TestCICBase): c = Proof(path=proof_path, attachments=attach, writer=self.outputs_writer) c.load() - v = c.process() + v = c.process(token_address=self.token_address, token_symbol=self.token_symbol) self.assertEqual(v, root_merged_hash)