cicnet/cic-cli
5
1
Fork 0
Code Issues Pull Requests Projects Releases 18 Wiki Activity

Add dual writer for key and immutable storage

Browse Source
This commit is contained in:
nolash 2021-10-22 16:42:38 +02:00
parent fd6e9c23a7
commit eb42095a74
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
8 changed files with 139 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cic/cmd/export.py
View File

@ -12,7 +12,10 @@ from cic import (
Proof,
Processor,
)
from cic.output import HTTPWriter
from cic.output import (
HTTPWriter,
KeyedWriterFactory,
)
from cic.meta import (
Meta,
MetadataWriter,
@ -66,8 +69,8 @@ def execute(config, eargs):
MetadataSigner.gpg_path = os.path.join('/tmp')
MetadataSigner.key_file_path = '/home/lash/src/client/cic/grassrootseconomics/cic-internal-integration/apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc'
MetadataSigner.gpg_passphrase = 'merman'
writers['proof'] = HTTPWriter
writers['attachment'] = HTTPWriter
writers['proof'] = KeyedWriterFactory(MetadataWriter, HTTPWriter).new
writers['attachment'] = KeyedWriterFactory(None, HTTPWriter).new
writers['meta'] = MetadataWriter
output_writer_path_meta = eargs.metadata_endpoint

3
cic/data/proof_template_v0.json
View File

@ -2,5 +2,6 @@
"version": 0,
"namespace": "ge",
"issuer": "",
"description": ""
"description": null,
"proofs": []
}

18
cic/ext/eth/__init__.py
View File

@ -224,11 +224,15 @@ class CICEth(Extension):
c = Declarator(self.chain_spec, signer=self.signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
results = []
(main_proof, all_proofs) = self.proof.get()
for proof in all_proofs:
logg.debug('proof {} '.format(proof))
k = 'address_declarator_' + proof
o = c.add_declaration(contract_address, signer_address, self.token_address, proof, tx_format=self.tx_format)
#(main_proof, all_proofs) = self.proof.get()
#for proof in all_proofs:
#logg.debug('proof {} '.format(proof))
(k, v) = self.proof.root()
fk = 'address_declarator_' + k
o = c.add_declaration(contract_address, signer_address, self.token_address, k, tx_format=self.tx_format)
r = None
if self.rpc != None:
r = self.rpc.do(o[1])
@ -237,11 +241,11 @@ class CICEth(Extension):
r = o[1]
else:
r = o
self.add_outputs(k, r)
self.add_outputs(fk, r)
results.append(r)
v = r.encode('utf-8')
if writer != None:
writer.write(k, v)
writer.write(fk, v)
return results

17
cic/meta.py
View File

@ -1,6 +1,8 @@
# standard imports
import os
import json
import logging
import base64
# external imports
from cic_types import MetadataPointer
@ -14,6 +16,7 @@ from .base import (
data_dir,
)
from cic.output import OutputWriter
logg = logging.getLogger(__name__)
class Meta(Data):
@ -98,5 +101,15 @@ class MetadataWriter(OutputWriter):
def write(self, k, v):
rq = MetadataRequestsHandler(MetadataPointer.NONE, bytes.fromhex(k))
return rq.create(json.loads(v.decode('utf-8')))
rq = urllib.request.Request(self.path)
try:
v = v.decode('utf-8')
v = json.loads(v)
logg.debug('metadatawriter bindecode {} {}'.format(k, v))
except UnicodeDecodeError:
v = base64.b64encode(v).decode('utf-8')
v = json.loads(json.dumps(v))
logg.debug('metadatawriter b64encode {} {}'.format(k, v))
r = rq.create(v)
logg.info('metadata submitted at {}'.format(k))
return r

46
cic/output.py
View File

@ -48,6 +48,48 @@ class HTTPWriter(OutputWriter):
def write(self, k, v):
rq = urllib.request.Request(self.path, method='POST', data=v)
path = self.path
if k != None:
path = os.path.join(path, k)
logg.debug('http writer post {}'.format(path))
rq = urllib.request.Request(path, method='POST', data=v)
r = urllib.request.urlopen(rq)
logg.info('proof submited at {}'.format(r.read))
logg.info('proof submited at {}'.format(r.read()))
class KeyedWriter(OutputWriter):
def __init__(self, writer_keyed, writer_immutable):
self.writer_keyed = writer_keyed
self.writer_immutable = writer_immutable
def write(self, k, v):
logg.debug('writing keywriter {} {}'.format(k, v))
if isinstance(v, str):
v = v.encode('utf-8')
if self.writer_keyed != None:
self.writer_keyed.write(k, v)
if self.writer_immutable != None:
self.writer_immutable.write(None, v)
class KeyedWriterFactory:
def __init__(self, key_writer_constructor, immutable_writer_constructor, *args, **kwargs):
self.key_writer_constructor = key_writer_constructor
self.immutable_writer_constructor = immutable_writer_constructor
self.x = {}
for k in kwargs.keys():
logg.debug('adding key {} t keyed writer factory'.format(k))
self.x[k] = kwargs[k]
def new(self, path=None, *args, **kwargs):
writer_keyed = None
writer_immutable = None
if self.key_writer_constructor != None:
writer_keyed = self.key_writer_constructor(path, **self.x)
if self.immutable_writer_constructor != None:
writer_immutable = self.immutable_writer_constructor(path, **self.x)
return KeyedWriter(writer_keyed, writer_immutable)

75
cic/proof.py
View File

@ -22,6 +22,7 @@ class Proof(Data):
def __init__(self, path='.', attachments=None, writer=None):
super(Proof, self).__init__()
self.proofs = []
self.namespace = 'ge'
self.description = None
self.path = path
@ -44,6 +45,14 @@ class Proof(Data):
self.namespace = o['namespace']
self.issuer = o['issuer']
if self.extra_attachments != None:
a = self.extra_attachments.asdict()
for k in a.keys():
self.attachments[k] = a[k]
hshs = self.__get_ordered_hashes()
self.proofs = list(map(strip_0x, hshs))
self.inited = True
@ -67,6 +76,7 @@ class Proof(Data):
'namespace': self.namespace,
'description': self.description,
'issuer': self.issuer,
'proofs': self.proofs,
}
@ -77,50 +87,59 @@ class Proof(Data):
return ks
def get(self):
v = self.asdict()
b = cbor2.dumps(v)
# def get(self):
# hsh = self.hash(b).hex()
# self.attachments[hsh] = self.temp_proof_path
# logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh))
f = open(self.temp_proof_path, 'wb')
def root(self):
v = self.asdict()
#b = cbor2.dumps(v)
b = json.dumps(v)
f = open(self.temp_proof_path, 'w')
f.write(b)
f.close()
hsh = self.hash(b).hex()
self.attachments[hsh] = self.temp_proof_path
logg.debug('cbor of {} is {} hashes to {}'.format(v, b.hex(), hsh))
k = self.hash(b.encode('utf-8'))
if self.extra_attachments != None:
a = self.extra_attachments.asdict()
for k in a.keys():
self.attachments[k] = a[k]
hshs = self.__get_ordered_hashes()
return (hsh, hshs)
return (k.hex(), b)
def process(self, token_address=None, token_symbol=None, writer=None):
if writer == None:
writer = self.writer
(hsh, hshs) = self.get()
hshs = list(map(strip_0x, hshs))
hshs_bin = list(map(bytes.fromhex, hshs))
hshs_cat = b''.join(hshs_bin)
(k, v) = self.root()
writer.write(k, v)
f = open(self.temp_proof_path, 'rb')
v = f.read()
f.close()
writer.write(hsh, v)
token_symbol_bytes = token_symbol.encode('utf-8')
k = generate_metadata_pointer(token_symbol_bytes, MetadataPointer.TOKEN_PROOF_SYMBOL)
writer.write(k, v)
r = self.hash(hshs_cat)
r_hex = r.hex()
token_address_bytes = bytes.fromhex(strip_0x(token_address))
k = generate_metadata_pointer(token_address_bytes, MetadataPointer.TOKEN_PROOF)
writer.write(k, v)
logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs))
# (hsh, hshs) = self.get()
#hshs = list(map(strip_0x, hshs))
# hshs_bin = list(map(bytes.fromhex, hshs))
# hshs_cat = b''.join(hshs_bin)
writer.write(r_hex, hshs_cat)
# f = open(self.temp_proof_path, 'rb')
# v = f.read()
# f.close()
# writer.write(hsh, v)
return r_hex
# r = self.hash(hshs_cat)
# r_hex = r.hex()
#logg.debug('generated proof {} for hashes {}'.format(r_hex, hshs))
#writer.write(r_hex, hshs_cat)
return k
def __str__(self):

4
tests/base_cic.py
View File

@ -22,8 +22,7 @@ test_data_dir = os.path.join(test_base_dir, 'testdata')
proof_hash = '0f6fc017f29caf512c0feaaf83bc10614b488311cace2973dc248dc24b01e04f'
foo_hash = '2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae'
bar_hash = 'fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9'
root_merged_hash = 'ba135f8518d36af5fa65c59317ea9602c4b654d998ea9097ecf81e0638a03441'
root_unmerged_hash = '68ccfe99fd905be439b09dcd780993865598605b8492462a6fc4f127b688fef6'
root_merged_hash = '4bd0ad4305a5fee20fb80e179a437c296f6a769ca376d746a3848a80e9b7a1a6'
class TestCICBase(unittest.TestCase):
@ -37,6 +36,7 @@ class TestCICBase(unittest.TestCase):
add_0x(random.randbytes(20).hex()),
add_0x(random.randbytes(20).hex()),
]
self.token_symbol = 'FOO'
self.token_address = add_0x(random.randbytes(32).hex())
self.token_index_address = add_0x(random.randbytes(32).hex())
self.address_declarator_address = add_0x(random.randbytes(32).hex())

11
tests/test_proof.py
View File

@ -12,7 +12,6 @@ from tests.base_cic import (
test_data_dir,
TestCICBase,
root_merged_hash,
root_unmerged_hash,
)
logging.basicConfig(level=logging.DEBUG)
@ -20,14 +19,6 @@ logg = logging.getLogger()
class TestProof(TestCICBase):
def test_proof_serialize(self):
proof_path = os.path.join(test_data_dir, 'proof')
c = Proof(path=proof_path, writer=self.outputs_writer)
c.load()
v = c.process()
self.assertEqual(v, root_unmerged_hash)
def test_proof_serialize_merge(self):
proof_path = os.path.join(test_data_dir, 'proof')
@ -36,7 +27,7 @@ class TestProof(TestCICBase):
c = Proof(path=proof_path, attachments=attach, writer=self.outputs_writer)
c.load()
v = c.process()
v = c.process(token_address=self.token_address, token_symbol=self.token_symbol)
self.assertEqual(v, root_merged_hash)