Add dual writer for key and immutable storage

cic/cmd/export.py

@@ -12,7 +12,10 @@ from cic import (
         Proof,
         Processor,
         )
-from cic.output import HTTPWriter
+from cic.output import (
+        HTTPWriter,
+        KeyedWriterFactory,
+        )
 from cic.meta import (
         Meta,
         MetadataWriter,
@@ -66,8 +69,8 @@ def execute(config, eargs):
         MetadataSigner.gpg_path = os.path.join('/tmp')
         MetadataSigner.key_file_path = '/home/lash/src/client/cic/grassrootseconomics/cic-internal-integration/apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc'
         MetadataSigner.gpg_passphrase = 'merman'
-        writers['proof'] = HTTPWriter
+        writers['proof'] = KeyedWriterFactory(MetadataWriter, HTTPWriter).new
-        writers['attachment'] = HTTPWriter
+        writers['attachment'] = KeyedWriterFactory(None, HTTPWriter).new
         writers['meta'] = MetadataWriter
         output_writer_path_meta = eargs.metadata_endpoint
     

cic/data/proof_template_v0.json

@@ -2,5 +2,6 @@
 	"version": 0,
 	"namespace": "ge",
 	"issuer": "",
-	"description": ""
+	"description": null,
+	"proofs": []
 }

cic/ext/eth/__init__.py

@@ -224,24 +224,28 @@ class CICEth(Extension):
         c = Declarator(self.chain_spec, signer=self.signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
 
         results = []
-        (main_proof, all_proofs) = self.proof.get()
+        #(main_proof, all_proofs) = self.proof.get()
-        for proof in all_proofs:
+
-            logg.debug('proof {} '.format(proof))
+        #for proof in all_proofs:
-            k = 'address_declarator_' + proof
+        #logg.debug('proof {} '.format(proof))
-            o = c.add_declaration(contract_address, signer_address, self.token_address, proof, tx_format=self.tx_format)
+
-            r = None
+        (k, v) = self.proof.root()
-            if self.rpc != None:
+
-                r = self.rpc.do(o[1])
+        fk = 'address_declarator_' + k
-                self.rpc.wait(r)
+        o = c.add_declaration(contract_address, signer_address, self.token_address, k, tx_format=self.tx_format)
-            elif self.signer != None:
+        r = None
-                r = o[1]
+        if self.rpc != None:
-            else:
+            r = self.rpc.do(o[1])
-                r = o
+            self.rpc.wait(r)
-            self.add_outputs(k, r)
+        elif self.signer != None:
-            results.append(r)
+            r = o[1]
-            v = r.encode('utf-8')
+        else:
-            if writer != None:
+            r = o
-                writer.write(k, v)
+        self.add_outputs(fk, r)
+        results.append(r)
+        v = r.encode('utf-8')
+        if writer != None:
+            writer.write(fk, v)
 
         return results
 

cic/meta.py

@@ -1,6 +1,8 @@
 # standard imports
 import os
 import json
+import logging
+import base64
 
 # external imports
 from cic_types import MetadataPointer
@@ -14,6 +16,7 @@ from .base import (
         data_dir,
         )
 from cic.output import OutputWriter
+logg = logging.getLogger(__name__)
 
 
 class Meta(Data):
@@ -98,5 +101,15 @@ class MetadataWriter(OutputWriter):
 
     def write(self, k, v):
         rq = MetadataRequestsHandler(MetadataPointer.NONE, bytes.fromhex(k))
-        return rq.create(json.loads(v.decode('utf-8')))
+        try:
-        rq = urllib.request.Request(self.path)
+            v = v.decode('utf-8')
+            v = json.loads(v)
+            logg.debug('metadatawriter bindecode {} {}'.format(k, v))
+        except UnicodeDecodeError:
+            v = base64.b64encode(v).decode('utf-8')
+            v = json.loads(json.dumps(v))
+            logg.debug('metadatawriter b64encode {} {}'.format(k, v))
+        r = rq.create(v)
+        logg.info('metadata submitted at {}'.format(k))
+        return r
+

cic/output.py

@@ -48,6 +48,48 @@ class HTTPWriter(OutputWriter):
 
   
     def write(self, k, v):
-        rq = urllib.request.Request(self.path, method='POST', data=v)
+        path = self.path
+        if k != None:
+            path = os.path.join(path, k)
+        logg.debug('http writer post {}'.format(path))
+        rq = urllib.request.Request(path, method='POST', data=v)
         r = urllib.request.urlopen(rq)
-        logg.info('proof submited at {}'.format(r.read))
+        logg.info('proof submited at {}'.format(r.read()))
+
+
+class KeyedWriter(OutputWriter):
+
+    def __init__(self, writer_keyed, writer_immutable):
+        self.writer_keyed = writer_keyed
+        self.writer_immutable = writer_immutable
+
+
+    def write(self, k, v):
+        logg.debug('writing keywriter {} {}'.format(k, v))
+        if isinstance(v, str):
+            v = v.encode('utf-8')
+        if self.writer_keyed != None:
+            self.writer_keyed.write(k, v)
+        if self.writer_immutable != None:
+            self.writer_immutable.write(None, v)
+
+
+class KeyedWriterFactory:
+
+    def __init__(self, key_writer_constructor, immutable_writer_constructor, *args, **kwargs):
+        self.key_writer_constructor = key_writer_constructor
+        self.immutable_writer_constructor = immutable_writer_constructor
+        self.x = {}
+        for k in kwargs.keys():
+            logg.debug('adding key {} t keyed writer factory'.format(k))
+            self.x[k] = kwargs[k] 
+       
+
+    def new(self, path=None, *args, **kwargs):
+        writer_keyed = None
+        writer_immutable = None
+        if self.key_writer_constructor != None:
+            writer_keyed = self.key_writer_constructor(path, **self.x)
+        if self.immutable_writer_constructor != None:
+            writer_immutable = self.immutable_writer_constructor(path, **self.x)
+        return KeyedWriter(writer_keyed, writer_immutable)

cic/proof.py

@@ -22,6 +22,7 @@ class Proof(Data):
 
     def __init__(self, path='.', attachments=None, writer=None):
         super(Proof, self).__init__()
+        self.proofs = []
         self.namespace = 'ge'
         self.description = None
         self.path = path
@@ -44,6 +45,14 @@ class Proof(Data):
         self.namespace = o['namespace']
         self.issuer = o['issuer']
 
+        if self.extra_attachments != None:
+            a = self.extra_attachments.asdict()
+            for k in a.keys():
+                self.attachments[k] = a[k]
+
+        hshs = self.__get_ordered_hashes()
+        self.proofs = list(map(strip_0x, hshs))
+
         self.inited = True
 
 
@@ -67,6 +76,7 @@ class Proof(Data):
             'namespace': self.namespace,
             'description': self.description,
             'issuer': self.issuer,
+            'proofs': self.proofs,
                 }
 
 
@@ -77,50 +87,59 @@ class Proof(Data):
         return ks
 
 
-    def get(self):
+#    def get(self):
-        v = self.asdict() 
+#        hsh = self.hash(b).hex()
-        b = cbor2.dumps(v)
+#        self.attachments[hsh] = self.temp_proof_path
+#        logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh))
 
-        f = open(self.temp_proof_path, 'wb')
+
+    def root(self):
+        v = self.asdict() 
+        #b = cbor2.dumps(v)
+        b = json.dumps(v)
+
+        f = open(self.temp_proof_path, 'w')
         f.write(b)
         f.close()
 
-        hsh = self.hash(b).hex()
+        k = self.hash(b.encode('utf-8'))
-        self.attachments[hsh] = self.temp_proof_path
-        logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh))
 
-        if self.extra_attachments != None:
+        return (k.hex(), b)
-            a = self.extra_attachments.asdict()
-            for k in a.keys():
-                self.attachments[k] = a[k]
-
-        hshs = self.__get_ordered_hashes()
-
-        return (hsh, hshs)
 
 
     def process(self, token_address=None, token_symbol=None, writer=None):
         if writer == None:
             writer = self.writer
 
-        (hsh, hshs) = self.get()
+        (k, v) = self.root()
-        hshs = list(map(strip_0x, hshs))
+        writer.write(k, v)
-        hshs_bin = list(map(bytes.fromhex, hshs))
-        hshs_cat = b''.join(hshs_bin)
 
-        f = open(self.temp_proof_path, 'rb')
+        token_symbol_bytes = token_symbol.encode('utf-8')
-        v = f.read()
+        k = generate_metadata_pointer(token_symbol_bytes, MetadataPointer.TOKEN_PROOF_SYMBOL)
-        f.close()
+        writer.write(k, v)
-        writer.write(hsh, v)
 
-        r = self.hash(hshs_cat)
+        token_address_bytes = bytes.fromhex(strip_0x(token_address))
-        r_hex = r.hex()
+        k = generate_metadata_pointer(token_address_bytes, MetadataPointer.TOKEN_PROOF)
+        writer.write(k, v)
 
-        logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs))
+#        (hsh, hshs) = self.get()
+        #hshs = list(map(strip_0x, hshs))
+#        hshs_bin = list(map(bytes.fromhex, hshs))
+#        hshs_cat = b''.join(hshs_bin)
 
-        writer.write(r_hex, hshs_cat)
+#        f = open(self.temp_proof_path, 'rb')
+#        v = f.read()
+#        f.close()
+#        writer.write(hsh, v)
 
-        return r_hex
+#        r = self.hash(hshs_cat)
+#        r_hex = r.hex()
+
+        #logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs))
+
+        #writer.write(r_hex, hshs_cat)
+        
+        return k
 
 
     def __str__(self):

tests/base_cic.py

@@ -22,8 +22,7 @@ test_data_dir = os.path.join(test_base_dir, 'testdata')
 proof_hash = '0f6fc017f29caf512c0feaaf83bc10614b488311cace2973dc248dc24b01e04f'
 foo_hash = '2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae'
 bar_hash = 'fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9'
-root_merged_hash = 'ba135f8518d36af5fa65c59317ea9602c4b654d998ea9097ecf81e0638a03441'
+root_merged_hash = '4bd0ad4305a5fee20fb80e179a437c296f6a769ca376d746a3848a80e9b7a1a6'
-root_unmerged_hash = '68ccfe99fd905be439b09dcd780993865598605b8492462a6fc4f127b688fef6'
 
 
 class TestCICBase(unittest.TestCase):
@@ -37,6 +36,7 @@ class TestCICBase(unittest.TestCase):
             add_0x(random.randbytes(20).hex()),
             add_0x(random.randbytes(20).hex()),
                 ]
+        self.token_symbol = 'FOO'
         self.token_address = add_0x(random.randbytes(32).hex())
         self.token_index_address = add_0x(random.randbytes(32).hex())
         self.address_declarator_address = add_0x(random.randbytes(32).hex())

tests/test_proof.py

@@ -12,7 +12,6 @@ from tests.base_cic import (
         test_data_dir,
         TestCICBase,
         root_merged_hash,
-        root_unmerged_hash,
         )
 
 logging.basicConfig(level=logging.DEBUG)
@@ -20,14 +19,6 @@ logg = logging.getLogger()
 
 class TestProof(TestCICBase):
 
-    def test_proof_serialize(self):
-        proof_path = os.path.join(test_data_dir, 'proof')
-        c = Proof(path=proof_path, writer=self.outputs_writer)
-        c.load()
-        v = c.process()
-        self.assertEqual(v, root_unmerged_hash)
-
-
     def test_proof_serialize_merge(self):
         proof_path = os.path.join(test_data_dir, 'proof')
 
@@ -36,7 +27,7 @@ class TestProof(TestCICBase):
 
         c = Proof(path=proof_path, attachments=attach, writer=self.outputs_writer)
         c.load()
-        v = c.process()
+        v = c.process(token_address=self.token_address, token_symbol=self.token_symbol)
         self.assertEqual(v, root_merged_hash)