Add immutable content submissions

apps/cic-meta/docker/Dockerfile

@@ -7,13 +7,13 @@ WORKDIR /root
 RUN apk add --no-cache postgresql bash
 
 # copy the dependencies
-COPY package.json package-lock.json .
+COPY package.json package-lock.json ./
 RUN --mount=type=cache,mode=0755,target=/root/.npm \
     npm set cache /root/.npm && \
     npm ci
 
-COPY webpack.config.js .
+COPY webpack.config.js ./
-COPY tsconfig.json . 
+COPY tsconfig.json ./
 ## required to build the cic-client-meta module
 COPY . .
 COPY tests/*.asc /root/pgp/

apps/cic-meta/scripts/initdb/server.postgres.sql

@@ -1,8 +1,9 @@
 create table if not exists store (
 	id serial primary key not null,
-	owner_fingerprint text not null,
+	owner_fingerprint text default null,
 	hash char(64) not null unique,
-	content text not null
+	content text not null,
+	mime_type text
 );
 
 create index if not exists idx_fp on store ((lower(owner_fingerprint)));

apps/cic-meta/scripts/initdb/server.sqlite.sql

@@ -1,9 +1,10 @@
 create table if not exists store (
 	/*id serial primary key not null,*/
 	id integer primary key autoincrement,
-	owner_fingerprint text not null,
+	owner_fingerprint text default null,
 	hash char(64) not null unique,
-	content text not null
+	content text not null,
+	mime_type text
 );
 
 create index if not exists idx_fp on store ((lower(owner_fingerprint)));

apps/cic-meta/scripts/server/handlers.ts

@@ -1,12 +1,13 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
+import * as crypto from 'crypto';
 
-import { Envelope, Syncable } from '@cicnet/crdt-meta';
+import { Envelope, Syncable, bytesToHex } from '@cicnet/crdt-meta';
 
 
 function handleNoMergeGet(db, digest, keystore) {
-	const sql = "SELECT content FROM store WHERE hash = '" + digest + "'";
+	const sql = "SELECT owner_fingerprint, content, mime_type FROM store WHERE hash = '" + digest + "'";
-	return new Promise<string|boolean>((whohoo, doh) => {
+	return new Promise<any>((whohoo, doh) => {
 		db.query(sql, (e, rs) => {
 			if (e !== null && e !== undefined) {
 				doh(e);
@@ -16,16 +17,37 @@ function handleNoMergeGet(db, digest, keystore) {
 				return;
 			}
 
+			const immutable = rs.rows[0]['owner_fingerprint'] == undefined;
+			let mimeType;
+			if (immutable) {
+				if (rs.rows[0]['mime_type'] === undefined) {
+					mimeType = 'application/octet-stream';
+				} else {
+					mimeType = rs.rows[0]['mime_type'];
+				}
+			} else {
+				mimeType = 'application/json';
+			}
+
 			const cipherText = rs.rows[0]['content'];
 			pgp.message.readArmored(cipherText).then((m) => {
 				const opts = {
 					message: m,
 					privateKeys: [keystore.getPrivateKey()],
+					format: 'binary',
 				};
 				pgp.decrypt(opts).then((plainText) => {
-					const o = Syncable.fromJSON(plainText.data);
+					console.debug('immutable ', rs.rows[0]['owner_fingerprint']);
-					const r = JSON.stringify(o.m['data']);
+					let r;
-					whohoo(r);
+				     	if (immutable) {
+						r = plainText.data;
+						console.debug('data ', r, r.length);
+					} else {
+						mimeType = 'application/json';
+						const o = Syncable.fromJSON(plainText.data);
+						r = JSON.stringify(o.m['data']);
+					}
+					whohoo([r, mimeType]);
 				}).catch((e) => {
 					console.error('decrypt', e);
 					doh(e);
@@ -201,10 +223,58 @@ function handleClientMergePut(data, db, digest, keystore, signer) {
 	});
 }
 
+function handleImmutablePost(data, db, digest, keystore, contentType) {
+	return new Promise<boolean>((whohoo, doh) => {
+		handleNoMergeGet(db, digest, keystore).then((haveDigest) => {
+			if (haveDigest !== false) {
+				whohoo(false);
+				return;
+			}
+			let data_binary = data;
+			let message;
+			if (typeof(data) == 'string') {
+				data_binary = new TextEncoder().encode(data);
+				message = pgp.message.fromText(data);
+			} else {
+				message = pgp.message.fromBinary(data);
+			}
+
+			const h = crypto.createHash('sha256');
+			h.update(data_binary);
+			const z = h.digest();
+			const r = bytesToHex(z);
+			if (r != digest) {
+				doh('hash mismatch: ' + r + ' != ' +  digest);
+				return;
+			}
+			
+			const opts = {
+				message: message,
+				publicKeys: keystore.getEncryptKeys(),
+			};
+			pgp.encrypt(opts).then((cipherText) => {
+				const sql = "INSERT INTO store (hash, content, mime_type) VALUES ('" + digest + "', '" + cipherText.data + "', '" + contentType + "') ON CONFLICT (hash) DO UPDATE SET content = EXCLUDED.content;";
+				db.query(sql, (e, rs) => {
+					if (e !== null && e !== undefined) {
+						doh(e);
+						return;
+					}
+					whohoo(true);
+				});
+			}).catch((e) => {
+				doh(e);	
+			});
+		}).catch((e) => {
+			doh(e);	
+		});
+	});
+}
+
 export {
 	handleClientMergePut,
 	handleClientMergeGet,
 	handleServerMergePost,
 	handleServerMergePut,
 	handleNoMergeGet,
+	handleImmutablePost,
 };

apps/cic-meta/scripts/server/server.ts

@@ -127,8 +127,10 @@ async function processRequest(req, res) {
 		return;
 	}
 
-	const mergeHeader = req.headers['x-cic-automerge'];
+
 	let mod = req.method.toLowerCase() + ":automerge:";
+
+	const mergeHeader = req.headers['x-cic-automerge'];
 	switch (mergeHeader) {
 		case "client":
 			mod += "client"; // client handles merges
@@ -136,19 +138,33 @@ async function processRequest(req, res) {
 		case "server":
 			mod += "server"; // server handles merges
 			break;
+		case "immutable":
+			mod += "immutable"; // server handles merges
+			break;
 		default:
 			mod += "none"; // merged object only (get only)
 	}
 
-	let data = '';
+	// handle bigger chunks of data
+	let data;
 	req.on('data', (d) => {
-		data += d;
+		if (data === undefined) {
+			data = d;
+		} else {
+			data += d;
+		}
 	});
-	req.on('end', async () => {
+	req.on('end', async (d) => {
-		console.debug('mode', mod);
+		console.debug('hedaers ', req.headers);
-		let content = '';
+		let inputContentType = req.headers['content-type'];
+		let debugString = 'executing mode ' + mod ;
+		if (data !== undefined) {
+			debugString += ' for content type ' + inputContentType + ' length ' + data.length;
+		}
+		console.debug(debugString);
+		let content;
 		let contentType = 'application/json';
-		console.debug('handling data', data);
+		let statusCode = 200;
 		let r:any = undefined;
 		try {
 			switch (mod) {
@@ -183,12 +199,24 @@ async function processRequest(req, res) {
 
 				case 'get:automerge:none':
 					r = await handlers.handleNoMergeGet(db, digest, keystore);	
-					if (r == false) {
+					if (r === false) {
 						res.writeHead(404, {"Content-Type": "text/plain"});
 						res.end();
 						return;
 					}
-					content = r;
+					content = r[0];
+					contentType = r[1];
+					break;
+
+				case 'post:automerge:immutable':
+					if (inputContentType === undefined) {
+						inputContentType = 'application/octet-stream';
+					}
+					r = await handlers.handleImmutablePost(data, db, digest, keystore, inputContentType);
+					if (r) {
+						statusCode = 201;
+					}
+					content = '';
 					break;
 
 				default:
@@ -210,8 +238,12 @@ async function processRequest(req, res) {
 			return;
 		}
 
-		const responseContentLength = (new TextEncoder().encode(content)).length;
+		//let responseContentLength;
-		res.writeHead(200, {
+		//if (typeof(content) == 'string') {
+		//	(new TextEncoder().encode(content)).length;
+		//}
+		const responseContentLength = content.length;
+		res.writeHead(statusCode, {
 			"Access-Control-Allow-Origin": "*",
 			"Content-Type": contentType,
 			"Content-Length": responseContentLength,