From a707f120865186c8e4a7840d53c9dcf5f4257ab3 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Mon, 25 Oct 2021 18:55:42 +0000 Subject: [PATCH] feat: Add token symbol proof metadata references --- cic/attachment.py | 3 +- cic/cmd/export.py | 11 +++-- cic/data/proof_template_v0.json | 3 +- cic/ext/eth/__init__.py | 40 ++++++++------- cic/extension.py | 5 +- cic/meta.py | 30 +++++++++-- cic/output.py | 46 ++++++++++++++++- cic/processor.py | 4 +- cic/proof.py | 88 +++++++++++++++++++++------------ requirements.txt | 2 +- tests/base_cic.py | 4 +- tests/test_proof.py | 11 +---- 12 files changed, 167 insertions(+), 80 deletions(-) diff --git a/cic/attachment.py b/cic/attachment.py index aa52ba3..014c0fc 100644 --- a/cic/attachment.py +++ b/cic/attachment.py @@ -44,7 +44,7 @@ class Attachment(Data): return self.contents - def process(self, token_address=None, writer=None): + def process(self, token_address=None, token_symbol=None, writer=None): if writer == None: writer = self.writer @@ -59,7 +59,6 @@ class Attachment(Data): def __str__(self): s = '' - #for i in range(len(self.contents)): for k in self.contents.keys(): s += '{} = {}\n'.format(k, self.contents[k]) #self.digests[i].hex(), self.contents[i]) diff --git a/cic/cmd/export.py b/cic/cmd/export.py index a4f0307..a873b41 100644 --- a/cic/cmd/export.py +++ b/cic/cmd/export.py @@ -12,7 +12,10 @@ from cic import ( Proof, Processor, ) -from cic.output import HTTPWriter +from cic.output import ( + HTTPWriter, + KeyedWriterFactory, + ) from cic.meta import ( Meta, MetadataWriter, @@ -66,8 +69,8 @@ def execute(config, eargs): MetadataSigner.gpg_path = os.path.join('/tmp') MetadataSigner.key_file_path = '/home/lash/src/client/cic/grassrootseconomics/cic-internal-integration/apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc' MetadataSigner.gpg_passphrase = 'merman' - writers['proof'] = HTTPWriter - writers['attachment'] = HTTPWriter + writers['proof'] = KeyedWriterFactory(MetadataWriter, HTTPWriter).new + writers['attachment'] = KeyedWriterFactory(None, HTTPWriter).new writers['meta'] = MetadataWriter output_writer_path_meta = eargs.metadata_endpoint @@ -77,10 +80,10 @@ def execute(config, eargs): cp = Proof(path=eargs.directory, attachments=ca, writer=writers['proof'](path=output_writer_path_meta)) cn = Network(path=eargs.directory) + ca.load() ct.load() cp.load() cm.load() - ca.load() cn.load() chain_spec = None diff --git a/cic/data/proof_template_v0.json b/cic/data/proof_template_v0.json index 5800c5a..2f1b58f 100644 --- a/cic/data/proof_template_v0.json +++ b/cic/data/proof_template_v0.json @@ -2,5 +2,6 @@ "version": 0, "namespace": "ge", "issuer": "", - "description": "" + "description": null, + "proofs": [] } diff --git a/cic/ext/eth/__init__.py b/cic/ext/eth/__init__.py index 0d8a393..fe256ca 100644 --- a/cic/ext/eth/__init__.py +++ b/cic/ext/eth/__init__.py @@ -224,24 +224,28 @@ class CICEth(Extension): c = Declarator(self.chain_spec, signer=self.signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) results = [] - (main_proof, all_proofs) = self.proof.get() - for proof in all_proofs: - logg.debug('proof {} '.format(proof)) - k = 'address_declarator_' + proof - o = c.add_declaration(contract_address, signer_address, self.token_address, proof, tx_format=self.tx_format) - r = None - if self.rpc != None: - r = self.rpc.do(o[1]) - self.rpc.wait(r) - elif self.signer != None: - r = o[1] - else: - r = o - self.add_outputs(k, r) - results.append(r) - v = r.encode('utf-8') - if writer != None: - writer.write(k, v) + #(main_proof, all_proofs) = self.proof.get() + + #for proof in all_proofs: + #logg.debug('proof {} '.format(proof)) + + (k, v) = self.proof.root() + + fk = 'address_declarator_' + k + o = c.add_declaration(contract_address, signer_address, self.token_address, k, tx_format=self.tx_format) + r = None + if self.rpc != None: + r = self.rpc.do(o[1]) + self.rpc.wait(r) + elif self.signer != None: + r = o[1] + else: + r = o + self.add_outputs(fk, r) + results.append(r) + v = r.encode('utf-8') + if writer != None: + writer.write(fk, v) return results diff --git a/cic/extension.py b/cic/extension.py index 61f4393..5bcc932 100644 --- a/cic/extension.py +++ b/cic/extension.py @@ -94,7 +94,8 @@ class Extension: tasks = [] self.token_address = self.resources['token']['reference'] - + + # TODO: get token details when token address is not none if self.token_address == None: if self.token_details['code'] == None: raise RuntimeError('neither token address nor token code has been set') @@ -114,4 +115,4 @@ class Extension: logg.debug('ciceth adapter process {}'.format(task)) r = getattr(self, 'process_' + task)(writer=writer) - return self.token_address + return (self.token_address, self.token_details['symbol']) diff --git a/cic/meta.py b/cic/meta.py index fa08415..f80bf33 100644 --- a/cic/meta.py +++ b/cic/meta.py @@ -1,6 +1,8 @@ # standard imports import os import json +import logging +import base64 # external imports from cic_types import MetadataPointer @@ -14,6 +16,7 @@ from .base import ( data_dir, ) from cic.output import OutputWriter +logg = logging.getLogger(__name__) class Meta(Data): @@ -66,12 +69,18 @@ class Meta(Data): } - def process(self, token_address=None, writer=None): + def process(self, token_address=None, token_symbol=None, writer=None): if writer == None: writer = self.writer - k = self.reference(token_address) v = json.dumps(self.asdict()) + + token_address_bytes = bytes.fromhex(strip_0x(token_address)) + k = generate_metadata_pointer(token_address_bytes, MetadataPointer.TOKEN_META) + writer.write(k, v.encode('utf-8')) + + token_symbol_bytes = token_symbol.encode('utf-8') + k = generate_metadata_pointer(token_symbol_bytes, MetadataPointer.TOKEN_META_SYMBOL) writer.write(k, v.encode('utf-8')) return (k, v) @@ -89,7 +98,18 @@ class Meta(Data): class MetadataWriter(OutputWriter): - + def write(self, k, v): - rq = MetadataRequestsHandler(MetadataPointer.TOKEN_META, bytes.fromhex(k)) - return rq.create(json.loads(v.decode('utf-8'))) + rq = MetadataRequestsHandler(MetadataPointer.NONE, bytes.fromhex(k)) + try: + v = v.decode('utf-8') + v = json.loads(v) + logg.debug('metadatawriter bindecode {} {}'.format(k, v)) + except UnicodeDecodeError: + v = base64.b64encode(v).decode('utf-8') + v = json.loads(json.dumps(v)) + logg.debug('metadatawriter b64encode {} {}'.format(k, v)) + r = rq.create(v) + logg.info('metadata submitted at {}'.format(k)) + return r + diff --git a/cic/output.py b/cic/output.py index 52bbe22..5d514d0 100644 --- a/cic/output.py +++ b/cic/output.py @@ -48,6 +48,48 @@ class HTTPWriter(OutputWriter): def write(self, k, v): - rq = urllib.request.Request(self.path, method='POST', data=v) + path = self.path + if k != None: + path = os.path.join(path, k) + logg.debug('http writer post {}'.format(path)) + rq = urllib.request.Request(path, method='POST', data=v) r = urllib.request.urlopen(rq) - logg.info('proof submited at {}'.format(r.read)) + logg.info('http writer submitted at {}'.format(r.read())) + + +class KeyedWriter(OutputWriter): + + def __init__(self, writer_keyed, writer_immutable): + self.writer_keyed = writer_keyed + self.writer_immutable = writer_immutable + + + def write(self, k, v): + logg.debug('writing keywriter {} {}'.format(k, v)) + if isinstance(v, str): + v = v.encode('utf-8') + if self.writer_keyed != None: + self.writer_keyed.write(k, v) + if self.writer_immutable != None: + self.writer_immutable.write(None, v) + + +class KeyedWriterFactory: + + def __init__(self, key_writer_constructor, immutable_writer_constructor, *args, **kwargs): + self.key_writer_constructor = key_writer_constructor + self.immutable_writer_constructor = immutable_writer_constructor + self.x = {} + for k in kwargs.keys(): + logg.debug('adding key {} t keyed writer factory'.format(k)) + self.x[k] = kwargs[k] + + + def new(self, path=None, *args, **kwargs): + writer_keyed = None + writer_immutable = None + if self.key_writer_constructor != None: + writer_keyed = self.key_writer_constructor(path, **self.x) + if self.immutable_writer_constructor != None: + writer_immutable = self.immutable_writer_constructor(path, **self.x) + return KeyedWriter(writer_keyed, writer_immutable) diff --git a/cic/processor.py b/cic/processor.py index e3f164d..a55ebf2 100644 --- a/cic/processor.py +++ b/cic/processor.py @@ -39,7 +39,7 @@ class Processor: ] for ext in self.extensions: - token_address = ext.process() + (token_address, token_symbol) = ext.process() for task in tasks: a = self.cores.get(task) @@ -47,5 +47,5 @@ class Processor: if a == None: logg.debug('skipping missing task receiver "{}"'.format(task)) continue - v = a.process(token_address=token_address, writer=self.__outputs_writer) + v = a.process(token_address=token_address, token_symbol=token_symbol, writer=self.__outputs_writer) self.outputs.append(v) diff --git a/cic/proof.py b/cic/proof.py index 9d84231..8c74ec4 100644 --- a/cic/proof.py +++ b/cic/proof.py @@ -22,6 +22,7 @@ class Proof(Data): def __init__(self, path='.', attachments=None, writer=None): super(Proof, self).__init__() + self.proofs = [] self.namespace = 'ge' self.description = None self.path = path @@ -43,6 +44,15 @@ class Proof(Data): self.description = o['description'] self.namespace = o['namespace'] self.issuer = o['issuer'] + self.proofs = o['proofs'] + + if self.extra_attachments != None: + a = self.extra_attachments.asdict() + for k in a.keys(): + self.attachments[k] = a[k] + + hshs = self.__get_ordered_hashes() + self.proofs = list(map(strip_0x, hshs)) self.inited = True @@ -67,6 +77,7 @@ class Proof(Data): 'namespace': self.namespace, 'description': self.description, 'issuer': self.issuer, + 'proofs': self.proofs, } @@ -77,50 +88,65 @@ class Proof(Data): return ks - def get(self): - v = self.asdict() - b = cbor2.dumps(v) +# def get(self): +# hsh = self.hash(b).hex() +# self.attachments[hsh] = self.temp_proof_path +# logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh)) - f = open(self.temp_proof_path, 'wb') + + def root(self): + v = self.asdict() + #b = cbor2.dumps(v) + b = json.dumps(v) + + f = open(self.temp_proof_path, 'w') f.write(b) f.close() - hsh = self.hash(b).hex() - self.attachments[hsh] = self.temp_proof_path - logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh)) + k = self.hash(b.encode('utf-8')) - if self.extra_attachments != None: - a = self.extra_attachments.asdict() - for k in a.keys(): - self.attachments[k] = a[k] - - hshs = self.__get_ordered_hashes() - - return (hsh, hshs) + return (k.hex(), b) - def process(self, token_address=None, writer=None): + def process(self, token_address=None, token_symbol=None, writer=None): if writer == None: writer = self.writer - (hsh, hshs) = self.get() - hshs = list(map(strip_0x, hshs)) - hshs_bin = list(map(bytes.fromhex, hshs)) - hshs_cat = b''.join(hshs_bin) - f = open(self.temp_proof_path, 'rb') - v = f.read() + (k, v) = self.root() + writer.write(k, v) + + token_symbol_bytes = token_symbol.encode('utf-8') + k = generate_metadata_pointer(token_symbol_bytes, MetadataPointer.TOKEN_PROOF_SYMBOL) + writer.write(k, v) + + token_address_bytes = bytes.fromhex(strip_0x(token_address)) + k = generate_metadata_pointer(token_address_bytes, MetadataPointer.TOKEN_PROOF) + writer.write(k, v) + +# (hsh, hshs) = self.get() + #hshs = list(map(strip_0x, hshs)) +# hshs_bin = list(map(bytes.fromhex, hshs)) +# hshs_cat = b''.join(hshs_bin) + +# f = open(self.temp_proof_path, 'rb') +# v = f.read() +# f.close() +# writer.write(hsh, v) + +# r = self.hash(hshs_cat) +# r_hex = r.hex() + + #logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs)) + + #writer.write(r_hex, hshs_cat) + + o = self.asdict() + f = open(self.proof_path, 'w') + json.dump(o, f, sort_keys=True, indent="\t") f.close() - writer.write(hsh, v) - r = self.hash(hshs_cat) - r_hex = r.hex() - - logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs)) - - writer.write(r_hex, hshs_cat) - - return r_hex + return k def __str__(self): diff --git a/requirements.txt b/requirements.txt index 94b1a28..87eaac1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ funga-eth>=0.5.0a1,<0.6.0 -cic-types>=0.2.0a4,<=0.2.0 +cic-types>=0.2.1a1,<=0.2.1 confini>=0.4.2rc3,<0.5.0 chainlib>=0.0.10a3,<0.1.0 cbor2==5.4.1 diff --git a/tests/base_cic.py b/tests/base_cic.py index b996588..2819c6f 100644 --- a/tests/base_cic.py +++ b/tests/base_cic.py @@ -22,8 +22,7 @@ test_data_dir = os.path.join(test_base_dir, 'testdata') proof_hash = '0f6fc017f29caf512c0feaaf83bc10614b488311cace2973dc248dc24b01e04f' foo_hash = '2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae' bar_hash = 'fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9' -root_merged_hash = 'ba135f8518d36af5fa65c59317ea9602c4b654d998ea9097ecf81e0638a03441' -root_unmerged_hash = '68ccfe99fd905be439b09dcd780993865598605b8492462a6fc4f127b688fef6' +root_merged_hash = '4bd0ad4305a5fee20fb80e179a437c296f6a769ca376d746a3848a80e9b7a1a6' class TestCICBase(unittest.TestCase): @@ -37,6 +36,7 @@ class TestCICBase(unittest.TestCase): add_0x(random.randbytes(20).hex()), add_0x(random.randbytes(20).hex()), ] + self.token_symbol = 'FOO' self.token_address = add_0x(random.randbytes(32).hex()) self.token_index_address = add_0x(random.randbytes(32).hex()) self.address_declarator_address = add_0x(random.randbytes(32).hex()) diff --git a/tests/test_proof.py b/tests/test_proof.py index 8e28179..4e31018 100644 --- a/tests/test_proof.py +++ b/tests/test_proof.py @@ -12,7 +12,6 @@ from tests.base_cic import ( test_data_dir, TestCICBase, root_merged_hash, - root_unmerged_hash, ) logging.basicConfig(level=logging.DEBUG) @@ -20,14 +19,6 @@ logg = logging.getLogger() class TestProof(TestCICBase): - def test_proof_serialize(self): - proof_path = os.path.join(test_data_dir, 'proof') - c = Proof(path=proof_path, writer=self.outputs_writer) - c.load() - v = c.process() - self.assertEqual(v, root_unmerged_hash) - - def test_proof_serialize_merge(self): proof_path = os.path.join(test_data_dir, 'proof') @@ -36,7 +27,7 @@ class TestProof(TestCICBase): c = Proof(path=proof_path, attachments=attach, writer=self.outputs_writer) c.load() - v = c.process() + v = c.process(token_address=self.token_address, token_symbol=self.token_symbol) self.assertEqual(v, root_merged_hash)