Cic meta builds

apps/cic-meta/scripts/server/args.ts

@@ -0,0 +1,28 @@
+const args = require('yargs');
+
+const standardArgs = args.option('config', {
+	alias: 'c',
+	type: 'string',
+	description: 'absolute path to configuation files directory', 
+
+}).option('env-prefix', {
+	type: 'string',
+	description: 'prefix to add to environment variables to match configuration directives',
+
+}).option('database-engine', {
+	type: 'string',
+	description: 'database engines to use', 
+
+}).option('address', {
+	alias: 'a',
+	type: 'string',
+	description: 'ip address to bind server to',
+
+}).option('server-address', {
+	alias: 'p',
+	type: 'number',
+	description: 'port to bind server to',
+
+});
+
+export { standardArgs };

apps/cic-meta/scripts/server/handlers.ts

@@ -0,0 +1,211 @@
+import * as Automerge from 'automerge';
+import * as pgp from 'openpgp';
+import * as pg from 'pg';
+
+import { Envelope, Syncable } from '../../src/sync';
+
+
+function handleNoMergeGet(db, digest, keystore) {
+	const sql = "SELECT content FROM store WHERE hash = '" + digest + "'";
+	return new Promise<string|boolean>((whohoo, doh) => {
+		db.query(sql, (e, rs) => {
+			if (e !== null && e !== undefined) {
+				doh(e);
+				return;
+			} else if (rs.rowCount == 0) {
+				whohoo(false);
+				return;
+			}
+
+			const cipherText = rs.rows[0]['content'];
+			pgp.message.readArmored(cipherText).then((m) => {
+				const opts = {
+					message: m,
+					privateKeys: [keystore.getPrivateKey()],
+				};
+				pgp.decrypt(opts).then((plainText) => {
+					const o = Syncable.fromJSON(plainText.data);
+					const r = JSON.stringify(o.m['data']);
+					whohoo(r);
+				}).catch((e) => {
+					console.error('decrypt', e);
+					doh(e);
+				});
+			}).catch((e) => {
+				console.error('mesage', e);
+				doh(e);
+			});
+		})
+	});
+}
+
+// TODO: add input for change description
+function handleServerMergePost(data, db, digest, keystore, signer) {
+	return new Promise<string>((whohoo, doh) => {
+		const o = JSON.parse(data);
+		const cipherText = handleClientMergeGet(db, digest, keystore).then(async (v) => {
+			let e = undefined;
+			let s = undefined;
+			if (v === undefined) {
+				s = new Syncable(digest, data);
+				s.onwrap = (e) => {
+					whohoo(e.toJSON());
+				};
+				digest = s.digest();
+				s.wrap({
+					digest: digest,
+				});
+			} else {
+				e = Envelope.fromJSON(v);
+				s = e.unwrap();
+				s.replace(o, 'server merge');
+				e.set(s);
+				s.onwrap = (e) => {
+					whohoo(e.toJSON());
+				}
+				digest = s.digest();
+				s.wrap({
+					digest: digest,
+				});
+			}
+		});
+	});
+}
+
+// TODO: this still needs to merge with the stored version
+function handleServerMergePut(data, db, digest, keystore, signer) {
+	return new Promise<boolean>((whohoo, doh) => {
+		const wrappedData = JSON.parse(data);
+
+		if (wrappedData.s === undefined) {
+			doh('signature missing');
+			return;
+		}
+
+		const e = Envelope.fromJSON(wrappedData.m);
+		let s = undefined;
+		try {
+			s = e.unwrap();
+		} catch(e) {
+			console.error(e)
+			whohoo(undefined);
+		}
+		// TODO: we probably should expose method for replacing the signature, this is too intrusive
+		s.m = Automerge.change(s.m, 'sign', (doc) => {
+			doc['signature'] = wrappedData.s;
+		});
+		s.setSigner(signer);
+		s.onauthenticate = (v) => {
+			console.log('vvv', v);
+			if (!v) {
+				whohoo(undefined);
+				return;
+			}
+			const opts = {
+				message: pgp.message.fromText(s.toJSON()),
+				publicKeys: keystore.getEncryptKeys(),
+			};
+			pgp.encrypt(opts).then((cipherText) => {
+				const sql = "INSERT INTO store (owner_fingerprint, hash, content) VALUES ('" + signer.fingerprint() + "', '" + digest + "', '" + cipherText.data + "') ON CONFLICT (hash) DO UPDATE SET content = EXCLUDED.content;";
+				db.query(sql, (e, rs) => {
+					if (e !== null && e !== undefined) {
+						doh(e);
+						return;
+					}
+					whohoo(true);
+				});
+			});
+		};
+		s.authenticate(true)
+	});
+}
+
+
+function handleClientMergeGet(db, digest, keystore) {
+	const sql = "SELECT content FROM store WHERE hash = '" + digest + "'";
+	return new Promise<string>((whohoo, doh) => {
+		db.query(sql, (e, rs) => {
+			console.log('rs', e, rs);
+			if (e !== null && e !== undefined) {
+				doh(e);
+				return;
+			} else if (rs.rowCount == 0) { // TODO fix the postgres/sqlite method name issues, this will now break on postgres
+				whohoo(undefined);
+				return;
+			}
+			const cipherText = rs.rows[0]['content'];
+			pgp.message.readArmored(cipherText).then((m) => {
+				const opts = {
+					message: m,
+					privateKeys: [keystore.getPrivateKey()],
+				};
+				pgp.decrypt(opts).then((plainText) => {
+					const o = Syncable.fromJSON(plainText.data);
+					const e = new Envelope(o);
+					whohoo(e.toJSON());
+				}).catch((e) => {
+					console.error('decrypt', e);
+					doh(e);
+				});
+			}).catch((e) => {
+				console.error('mesage', e);
+				doh(e);
+			});
+		});
+	});
+}
+
+// TODO: this still needs to merge with the stored version
+function handleClientMergePut(data, db, digest, keystore, signer) {
+	return new Promise<boolean>((whohoo, doh) => {
+		let s = undefined;
+		try {
+			const e = Envelope.fromJSON(data);
+			s = e.unwrap();
+		} catch(e) {
+			whohoo(false);
+			console.error(e)
+			return;
+		}
+
+		s.setSigner(signer);
+		s.onauthenticate = (v) => {
+			if (!v) {
+				whohoo(false);
+				return;
+			}
+
+			handleClientMergeGet(db, digest, keystore).then((v) => {
+				if (v !== undefined) {
+					const env = Envelope.fromJSON(v);
+					s.merge(env.unwrap());
+				}
+				const opts = {
+					message: pgp.message.fromText(s.toJSON()),
+					publicKeys: keystore.getEncryptKeys(),
+				};
+				pgp.encrypt(opts).then((cipherText) => {
+					const sql = "INSERT INTO store (owner_fingerprint, hash, content) VALUES ('" + signer.fingerprint() + "', '" + digest + "', '" + cipherText.data + "') ON CONFLICT (hash) DO UPDATE SET content = EXCLUDED.content;";
+					db.query(sql, (e, rs) => {
+						if (e !== null && e !== undefined) {
+							doh(e);
+							return;
+						}
+						whohoo(true);
+					});
+				}).catch((e) => {
+					doh(e);	
+				});
+			});
+		};
+		s.authenticate(true)
+	});
+}
+
+export {
+	handleClientMergePut,
+	handleClientMergeGet,
+	handleServerMergePost,
+	handleServerMergePut,
+	handleNoMergeGet,
+};

apps/cic-meta/scripts/server/server.postgres.sql

@@ -0,0 +1,8 @@
+create table if not exists store (
+	id serial primary key not null,
+	owner_fingerprint text not null,
+	hash char(64) not null unique,
+	content text not null
+);
+
+create index if not exists idx_fp on store ((lower(owner_fingerprint)));

apps/cic-meta/scripts/server/server.sql

@@ -0,0 +1,9 @@
+create table if not exists store (
+	/*id serial primary key not null,*/
+	id integer primary key autoincrement,
+	owner_fingerprint text not null,
+	hash char(64) not null unique,
+	content text not null
+);
+
+create index if not exists idx_fp on store ((lower(owner_fingerprint)));

apps/cic-meta/scripts/server/server.sqlite.sql

@@ -0,0 +1,9 @@
+create table if not exists store (
+	/*id serial primary key not null,*/
+	id integer primary key autoincrement,
+	owner_fingerprint text not null,
+	hash char(64) not null unique,
+	content text not null
+);
+
+create index if not exists idx_fp on store ((lower(owner_fingerprint)));

apps/cic-meta/scripts/server/server.ts

@@ -0,0 +1,207 @@
+import * as http from 'http';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as pgp from 'openpgp';
+
+import * as handlers from './handlers';
+import { Envelope, Syncable } from '../../src/sync';
+import { PGPKeyStore, PGPSigner } from '../../src/auth';
+
+import { standardArgs } from './args';
+import { Config } from '../../src/config';
+import { SqliteAdapter, PostgresAdapter } from '../../src/db';
+
+let configPath = '/usr/local/etc/cic-meta';
+
+const argv = standardArgs.argv;
+
+if (argv['config'] !== undefined) {
+	configPath = argv['config'];
+}
+
+const config = new Config(configPath, argv['env-prefix']);
+config.process();
+console.debug(config.toString());
+
+let fp = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PUBLICKEY_ACTIVE_FILE'));
+const pubksa = fs.readFileSync(fp, 'utf-8');
+fp = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATEKEY_FILE'));
+const pksa = fs.readFileSync(fp, 'utf-8');
+
+const dbConfig = {
+	'name': config.get('DATABASE_NAME'),
+	'user': config.get('DATABASE_USER'),
+	'password': config.get('DATABASE_PASSWORD'),
+	'host': config.get('DATABASE_HOST'),
+	'port': config.get('DATABASE_PORT'),
+	'engine': config.get('DATABASE_ENGINE'),
+};
+let db = undefined;
+if (config.get('DATABASE_ENGINE') == 'sqlite') {
+       	db = new SqliteAdapter(dbConfig);
+} else if (config.get('DATABASE_ENGINE') == 'postgres')  {
+       	db = new PostgresAdapter(dbConfig);
+} else {
+	throw 'database engine ' + config.get('DATABASE_ENGINE') + 'not implemented';
+}
+
+
+let signer = undefined;
+const keystore = new PGPKeyStore(config.get('PGP_PASSPHRASE'), pksa, pubksa, pubksa, pubksa, () => {
+	keysLoaded();
+});
+
+function keysLoaded() {
+	signer = new PGPSigner(keystore);
+	prepareServer();
+}
+
+async function migrateDatabase(cb) {
+	try {
+		const sql = "SELECT 1 FROM store;"
+		db.query(sql, (e, rs) => {
+			if (e === null || e === undefined) {
+				cb();
+				return;
+			}
+			console.warn('db check for table "store" fail', e);
+			
+			console.debug('using schema path', config.get('DATABASE_SCHEMA_SQL_PATH'));
+			const sql = fs.readFileSync(config.get('DATABASE_SCHEMA_SQL_PATH'), 'utf-8');
+			db.query(sql, (e, rs) => {
+				if (e !== undefined && e !== null) {
+					console.error('db initialization fail', e);
+					return;
+				} 
+				cb();
+			});
+
+		});
+	} catch(e) {
+		console.warn('table store does not exist', e);
+	}
+}
+
+async function prepareServer() {
+	await migrateDatabase(startServer);
+}
+
+async function startServer() {
+	http.createServer(processRequest).listen(config.get('SERVER_PORT'));
+}
+
+const re_digest = /^\/([a-fA-F0-9]{64})\/?$/;
+function parseDigest(url) {
+	const digest_test = url.match(re_digest);
+	if (digest_test === null) {
+		throw 'invalid digest';	
+	}
+	return digest_test[1].toLowerCase();
+}
+
+async function processRequest(req, res) {
+	let digest = undefined;
+
+	if (!['PUT', 'GET', 'POST'].includes(req.method)) {
+		res.writeHead(405, {"Content-Type": "text/plain"});
+		res.end();
+		return;
+	}
+
+	try {
+		digest = parseDigest(req.url);
+	} catch(e) {
+		res.writeHead(400, {"Content-Type": "text/plain"});
+		res.end();
+		return;
+	}
+
+	const mergeHeader = req.headers['x-cic-automerge'];
+	let mod = req.method.toLowerCase() + ":automerge:";
+	switch (mergeHeader) {
+		case "client":
+			mod += "client"; // client handles merges
+			break;
+		case "server":
+			mod += "server"; // server handles merges
+			break;
+		default:
+			mod += "none"; // merged object only (get only)
+	}
+
+	let data = '';
+	req.on('data', (d) => {
+		data += d;
+	});
+	req.on('end', async () => {
+		console.debug('mode', mod);
+		let content = '';
+		let contentType = 'application/json';
+		let r:any = undefined;
+		try {
+			switch (mod) {
+				case 'put:automerge:client':
+					r = await handlers.handleClientMergePut(data, db, digest, keystore, signer);
+					if (r == false) {
+						res.writeHead(403, {"Content-Type": "text/plain"});
+						res.end();
+						return;
+					}
+					break;
+
+				case 'get:automerge:client':
+					content = await handlers.handleClientMergeGet(db, digest, keystore);	
+					break;
+
+				case 'post:automerge:server':
+					content = await handlers.handleServerMergePost(data, db, digest, keystore, signer);	
+					break;
+
+				case 'put:automerge:server':
+					r = await handlers.handleServerMergePut(data, db, digest, keystore, signer);	
+					if (r == false) {
+						res.writeHead(403, {"Content-Type": "text/plain"});
+						res.end();
+						return;
+					}
+					break;
+				//case 'get:automerge:server':
+				//	content = await handlers.handleServerMergeGet(db, digest, keystore);	
+				//	break;
+
+				case 'get:automerge:none':
+					r = await handlers.handleNoMergeGet(db, digest, keystore);	
+					if (r == false) {
+						res.writeHead(404, {"Content-Type": "text/plain"});
+						res.end();
+						return;
+					}
+					content = r;
+					break;
+
+				default:
+					res.writeHead(400, {"Content-Type": "text/plain"});
+					res.end();
+					return;
+			}
+		} catch(e) {
+			console.error('fail', mod, digest, e);
+			res.writeHead(500, {"Content-Type": "text/plain"});
+			res.end();
+			return;
+		}
+
+		if (content === undefined) {
+			res.writeHead(400, {"Content-Type": "text/plain"});
+			res.end();
+			return;
+		}
+
+		res.writeHead(200, {
+			"Content-Type": contentType,
+			"Content-Length": content.length,
+		});
+		res.write(content);
+		res.end();
+	});
+}