kamikazechaser
/
cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rehabilitate automerge storage after adding immutable option

This commit is contained in:
nolash 2021-10-19 08:53:53 +02:00
parent fe2a88a4e1
commit b05fc43086
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
4 changed files with 5765 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5650
apps/cic-meta/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

53
apps/cic-meta/scripts/server/handlers.ts
View File

@ -37,14 +37,14 @@ function handleNoMergeGet(db, digest, keystore) {
format: 'binary',
};
pgp.decrypt(opts).then((plainText) => {
console.debug('immutable ', rs.rows[0]['owner_fingerprint']);
console.debug('immutable ', rs.rows[0]['owner_fingerprint'], immutable);
let r;
if (immutable) {
r = plainText.data;
console.debug('data ', r, r.length);
} else {
mimeType = 'application/json';
const o = Syncable.fromJSON(plainText.data);
const d = new TextDecoder().decode(plainText.data);
const o = Syncable.fromJSON(d);
r = JSON.stringify(o.m['data']);
}
whohoo([r, mimeType]);
@ -161,7 +161,13 @@ function handleClientMergeGet(db, digest, keystore) {
privateKeys: [keystore.getPrivateKey()],
};
pgp.decrypt(opts).then((plainText) => {
const o = Syncable.fromJSON(plainText.data);
let d;
if (typeof(plainText.data) == 'string') {
d = plainText.data;
} else {
d = new TextDecoder().decode(plainText.data);
}
const o = Syncable.fromJSON(d);
const e = new Envelope(o);
whohoo(e.toJSON());
}).catch((e) => {
@ -223,14 +229,30 @@ function handleClientMergePut(data, db, digest, keystore, signer) {
});
}
function handleImmutablePost(data, db, digest, keystore, contentType) {
return new Promise<boolean>((whohoo, doh) => {
handleNoMergeGet(db, digest, keystore).then((haveDigest) => {
if (haveDigest !== false) {
whohoo(false);
return new Promise<Array<string|boolean>>((whohoo, doh) => {
let data_binary = data;
const h = crypto.createHash('sha256');
h.update(data_binary);
const z = h.digest();
const r = bytesToHex(z);
if (digest) {
if (r != digest) {
doh('hash mismatch: ' + r + ' != ' + digest);
return;
}
} else {
digest = r;
console.debug('calculated digest ' + digest);
}
handleNoMergeGet(db, digest, keystore).then((haveDigest) => {
if (haveDigest !== false) {
whohoo([false, digest]);
return;
}
let data_binary = data;
let message;
if (typeof(data) == 'string') {
data_binary = new TextEncoder().encode(data);
@ -239,16 +261,7 @@ function handleImmutablePost(data, db, digest, keystore, contentType) {
message = pgp.message.fromBinary(data);
}
const h = crypto.createHash('sha256');
h.update(data_binary);
const z = h.digest();
const r = bytesToHex(z);
if (r != digest) {
doh('hash mismatch: ' + r + ' != ' + digest);
return;
}
const opts = {
const opts = {
message: message,
publicKeys: keystore.getEncryptKeys(),
};
@ -259,7 +272,7 @@ function handleImmutablePost(data, db, digest, keystore, contentType) {
doh(e);
return;
}
whohoo(true);
whohoo([true, digest]);
});
}).catch((e) => {
doh(e);

63
apps/cic-meta/scripts/server/server.ts
View File

@ -118,32 +118,51 @@ async function processRequest(req, res) {
return;
}
let mod = req.method.toLowerCase() + ":automerge:";
let modDetail = undefined;
let immutablePost = false;
try {
digest = parseDigest(req.url);
} catch(e) {
console.error('digest error: ' + e)
res.writeHead(400, {"Content-Type": "text/plain"});
res.end();
return;
if (req.url == '/') {
immutablePost = true;
modDetail = 'immutable';
} else {
console.error('url is not empty (' + req.url + ') and not valid digest error: ' + e)
res.writeHead(400, {"Content-Type": "text/plain"});
res.end();
return;
}
}
let mod = req.method.toLowerCase() + ":automerge:";
const mergeHeader = req.headers['x-cic-automerge'];
switch (mergeHeader) {
case "client":
mod += "client"; // client handles merges
break;
case "server":
mod += "server"; // server handles merges
break;
case "immutable":
mod += "immutable"; // server handles merges
break;
default:
mod += "none"; // merged object only (get only)
if (modDetail === undefined) {
const mergeHeader = req.headers['x-cic-automerge'];
switch (mergeHeader) {
case "client":
if (immutablePost) {
res.writeHead(400, 'Valid digest missing', {"Content-Type": "text/plain"});
res.end();
return;
}
modDetail = "client"; // client handles merges
break;
case "server":
if (immutablePost) {
res.writeHead(400, 'Valid digest missing', {"Content-Type": "text/plain"});
res.end();
return;
}
modDetail = "server"; // server handles merges
break;
case "immutable":
modDetail = "immutable"; // no merging, literal immutable content with content-addressing
break;
default:
modDetail = "none"; // merged object only (get only)
}
}
mod += modDetail;
// handle bigger chunks of data
let data;
@ -213,10 +232,10 @@ async function processRequest(req, res) {
inputContentType = 'application/octet-stream';
}
r = await handlers.handleImmutablePost(data, db, digest, keystore, inputContentType);
if (r) {
if (r[0]) {
statusCode = 201;
}
content = '';
content = r[1];
break;
default:

119
apps/cic-meta/tests/server.ts
View File

@ -44,11 +44,13 @@ function createDatabase(sqlite_file:string):Promise<any> {
// doh(e);
// return;
// }
// get this from real sql files sources
const sql = `CREATE TABLE store (
id integer primary key autoincrement,
owner_fingerprint text not null,
owner_fingerprint text default null,
hash char(64) not null unique,
content text not null
content text not null,
mime_type text default null
);
`
@ -111,15 +113,18 @@ describe('server', async () => {
let j = env.toJSON();
const content = await handlers.handleClientMergePut(j, db, digest, keystore, signer);
assert(content); // true-ish
console.debug('content', content);
let v = await handlers.handleNoMergeGet(db, digest, keystore);
if (v === undefined) {
if (v === false) {
db.close();
assert.fail('');
}
db.close();
return;
v = await handlers.handleClientMergeGet(db, digest, keystore);
if (v === undefined) {
if (v === false) {
db.close();
assert.fail('');
}
@ -187,7 +192,7 @@ describe('server', async () => {
j = await handlers.handleNoMergeGet(db, digest, keystore);
assert(v); // true-ish
let o = JSON.parse(j);
let o = JSON.parse(j[0]);
o.bar = 'xyzzy';
j = JSON.stringify(o);
@ -212,63 +217,63 @@ describe('server', async () => {
j = await handlers.handleNoMergeGet(db, digest, keystore);
assert(j); // true-ish
o = JSON.parse(j);
console.log(o);
db.close();
});
await it('server_merge', async () => {
const keystore = await createKeystore();
const signer = new PGPSigner(keystore);
const db = await createDatabase(__dirname + '/db.three.sqlite');
const digest = 'deadbeef';
let s = new Syncable(digest, {
bar: 'baz',
});
let env = await wrap(s, signer)
let j:any = env.toJSON();
let v = await handlers.handleClientMergePut(j, db, digest, keystore, signer);
assert(v); // true-ish
j = await handlers.handleNoMergeGet(db, digest, keystore);
assert(v); // true-ish
let o = JSON.parse(j);
o.bar = 'xyzzy';
j = JSON.stringify(o);
let signMaterial = await handlers.handleServerMergePost(j, db, digest, keystore, signer);
assert(signMaterial)
env = Envelope.fromJSON(signMaterial);
console.log('envvvv', env);
const signedData = await signData(env.o['digest'], keystore);
console.log('signed', signedData);
o = {
'm': env,
's': signedData,
}
j = JSON.stringify(o);
console.log(j);
v = await handlers.handleServerMergePut(j, db, digest, keystore, signer);
assert(v);
j = await handlers.handleNoMergeGet(db, digest, keystore);
assert(j); // true-ish
o = JSON.parse(j);
o = JSON.parse(j[0]);
console.log(o);
db.close();
});
// await it('server_merge', async () => {
// const keystore = await createKeystore();
// const signer = new PGPSigner(keystore);
//
// const db = await createDatabase(__dirname + '/db.three.sqlite');
//
// const digest = 'deadbeef';
// let s = new Syncable(digest, {
// bar: 'baz',
// });
// let env = await wrap(s, signer)
// let j:any = env.toJSON();
//
// let v = await handlers.handleClientMergePut(j, db, digest, keystore, signer);
// assert(v); // true-ish
//
// j = await handlers.handleNoMergeGet(db, digest, keystore);
// assert(v); // true-ish
//
// let o = JSON.parse(j);
// o.bar = 'xyzzy';
// j = JSON.stringify(o);
//
// let signMaterial = await handlers.handleServerMergePost(j, db, digest, keystore, signer);
// assert(signMaterial)
//
// env = Envelope.fromJSON(signMaterial);
//
// console.log('envvvv', env);
//
// const signedData = await signData(env.o['digest'], keystore);
// console.log('signed', signedData);
//
// o = {
// 'm': env,
// 's': signedData,
// }
// j = JSON.stringify(o);
// console.log(j);
//
// v = await handlers.handleServerMergePut(j, db, digest, keystore, signer);
// assert(v);
//
// j = await handlers.handleNoMergeGet(db, digest, keystore);
// assert(j); // true-ish
// o = JSON.parse(j);
// console.log(o);
//
// db.close();
// });
//
// await it('server_merge_empty', async () => {