Backports to beta (#2068)

* Fix several RPCs (#1926)

* Fix up pending receipts details.

* Add support for additional params and registry over RPC.

* Fix tests.

* Add test, additional fix.

Fixes #1932.

* Fix up tests.

* Fix test.

* Fix test.

* DB WAL size limit (#1935)

* Limit WAL size

* Check pruning by db modification date (#1924)

* Cache address hash (#1943)

* ECIES without MAC (#1948)

* Use random IV for ECIES AES

* ECIES encrypt/decrypt for a single message

* Derive IV from shared secret

* Apply settings to column families

* fixed #1933 (#1979)

* Fixed neighbours collection (#1996)

* dapps-hosts configuration

* Disabled counter check

* always process trie death row on commit, add more tracing

* fixed transaction addresses mapping, fixes #1971

* simplified iterator

* v1.3.1

* v1.3.1

util/src/crypto.rs

@@ -287,6 +287,7 @@ pub mod ecies {
 	use hash::*;
 	use bytes::*;
 	use crypto::*;
+	use sha3::Hashable;
 
 	/// Encrypt a message with a public key
 	pub fn encrypt(public: &Public, shared_mac: &[u8], plain: &[u8]) -> Result<Bytes, CryptoError> {
@@ -310,9 +311,11 @@ pub mod ecies {
 		{
 			let msgd = &mut msg[1..];
 			r.public().copy_to(&mut msgd[0..64]);
+			let iv = H128::random();
+			iv.copy_to(&mut msgd[64..(64+16)]);
 			{
 				let cipher = &mut msgd[(64 + 16)..(64 + 16 + plain.len())];
-				aes::encrypt(ekey, &H128::new(), plain, cipher);
+				aes::encrypt(ekey, &iv, plain, cipher);
 			}
 			let mut hmac = Hmac::new(Sha256::new(), &mkey);
 			{
@@ -325,6 +328,33 @@ pub mod ecies {
 		Ok(msg)
 	}
 
+	/// Encrypt a message with a public key
+	pub fn encrypt_single_message(public: &Public, plain: &[u8]) -> Result<Bytes, CryptoError> {
+		use ::rcrypto::digest::Digest;
+		use ::rcrypto::sha2::Sha256;
+		let r = try!(KeyPair::create());
+		let z = try!(ecdh::agree(r.secret(), public));
+		let mut key = [0u8; 32];
+		let mut mkey = [0u8; 32];
+		kdf(&z, &[0u8; 0], &mut key);
+		let mut hasher = Sha256::new();
+		let mkey_material = &key[16..32];
+		hasher.input(mkey_material);
+		hasher.result(&mut mkey);
+		let ekey = &key[0..16];
+
+		let mut msgd = vec![0u8; (64 + plain.len())];
+		{
+			r.public().copy_to(&mut msgd[0..64]);
+			let iv = H128::from_slice(&z.sha3()[0..16]);
+			{
+				let cipher = &mut msgd[64..(64 + plain.len())];
+				aes::encrypt(ekey, &iv, plain, cipher);
+			}
+		}
+		Ok(msgd)
+	}
+
 	/// Decrypt a message with a secret key
 	pub fn decrypt(secret: &Secret, shared_mac: &[u8], encrypted: &[u8]) -> Result<Bytes, CryptoError> {
 		use ::rcrypto::digest::Digest;
@@ -370,6 +400,36 @@ pub mod ecies {
 		Ok(msg)
 	}
 
+	/// Decrypt single message with a secret key
+	pub fn decrypt_single_message(secret: &Secret, encrypted: &[u8]) -> Result<Bytes, CryptoError> {
+		use ::rcrypto::digest::Digest;
+		use ::rcrypto::sha2::Sha256;
+
+		let meta_len = 64;
+		if encrypted.len() < meta_len {
+			return Err(CryptoError::InvalidMessage); //invalid message: publickey
+		}
+
+		let e = encrypted;
+		let p = Public::from_slice(&e[0..64]);
+		let z = try!(ecdh::agree(secret, &p));
+		let mut key = [0u8; 32];
+		kdf(&z, &[0u8; 0], &mut key);
+		let ekey = &key[0..16];
+		let mkey_material = &key[16..32];
+		let mut hasher = Sha256::new();
+		let mut mkey = [0u8; 32];
+		hasher.input(mkey_material);
+		hasher.result(&mut mkey);
+
+		let clen = encrypted.len() - meta_len;
+		let cipher = &e[64..(64+clen)];
+		let mut msg = vec![0u8; clen];
+		let iv = H128::from_slice(&z.sha3()[0..16]);
+		aes::decrypt(ekey, &iv, cipher, &mut msg[..]);
+		Ok(msg)
+	}
+
 	fn kdf(secret: &Secret, s1: &[u8], dest: &mut [u8]) {
 		use ::rcrypto::digest::Digest;
 		use ::rcrypto::sha2::Sha256;
@@ -458,4 +518,14 @@ mod tests {
 		let decrypted = ecies::decrypt(kp.secret(), shared, &encrypted).unwrap();
 		assert_eq!(decrypted[..message.len()], message[..]);
 	}
+
+	#[test]
+	fn ecies_shared_single() {
+		let kp = KeyPair::create().unwrap();
+		let message = b"So many books, so little time";
+		let encrypted = ecies::encrypt_single_message(kp.public(), message).unwrap();
+		assert!(encrypted[..] != message[..]);
+		let decrypted = ecies::decrypt_single_message(kp.secret(), &encrypted).unwrap();
+		assert_eq!(decrypted[..message.len()], message[..]);
+	}
 }

util/src/journaldb/archivedb.rs

@@ -163,7 +163,6 @@ impl JournalDB for ArchiveDB {
 		for i in self.overlay.drain().into_iter() {
 			let (key, (value, rc)) = i;
 			if rc > 0 {
-				assert!(rc == 1);
 				batch.put(self.column, &key, &value).expect("Low-level database error. Some issue with your hard disk?");
 				inserts += 1;
 			}
@@ -192,7 +191,6 @@ impl JournalDB for ArchiveDB {
 		for i in self.overlay.drain().into_iter() {
 			let (key, (value, rc)) = i;
 			if rc > 0 {
-				assert!(rc == 1);
 				if try!(self.backing.get(self.column, &key)).is_some() {
 					return Err(BaseDataError::AlreadyExists(key).into());
 				}

util/src/kvdb.rs

@@ -213,23 +213,33 @@ impl Database {
 		if let Some(rate_limit) = config.compaction.write_rate_limit {
 			try!(opts.set_parsed_options(&format!("rate_limiter_bytes_per_sec={}", rate_limit)));
 		}
+		try!(opts.set_parsed_options(&format!("max_total_wal_size={}", 64 * 1024 * 1024)));
 		opts.set_max_open_files(config.max_open_files);
 		opts.create_if_missing(true);
 		opts.set_use_fsync(false);
 
+		opts.set_max_background_flushes(DB_BACKGROUND_FLUSHES);
+		opts.set_max_background_compactions(DB_BACKGROUND_COMPACTIONS);
+
 		// compaction settings
 		opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
 		opts.set_target_file_size_base(config.compaction.initial_file_size);
 		opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier);
 
-		opts.set_max_background_flushes(DB_BACKGROUND_FLUSHES);
-		opts.set_max_background_compactions(DB_BACKGROUND_COMPACTIONS);
+		let mut cf_options = Vec::with_capacity(config.columns.unwrap_or(0) as usize);
 
-		if let Some(cache_size) = config.cache_size {
-			let mut block_opts = BlockBasedOptions::new();
-			// all goes to read cache
-			block_opts.set_cache(Cache::new(cache_size * 1024 * 1024));
-			opts.set_block_based_table_factory(&block_opts);
+		for _ in 0 .. config.columns.unwrap_or(0) {
+			let mut opts = Options::new();
+			opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
+			opts.set_target_file_size_base(config.compaction.initial_file_size);
+			opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier);
+			if let Some(cache_size) = config.cache_size {
+				let mut block_opts = BlockBasedOptions::new();
+				// all goes to read cache
+				block_opts.set_cache(Cache::new(cache_size * 1024 * 1024));
+				opts.set_block_based_table_factory(&block_opts);
+			}
+			cf_options.push(opts);
 		}
 
 		let mut write_opts = WriteOptions::new();
@@ -242,7 +252,7 @@ impl Database {
 			Some(columns) => {
 				let cfnames: Vec<_> = (0..columns).map(|c| format!("col{}", c)).collect();
 				let cfnames: Vec<&str> = cfnames.iter().map(|n| n as &str).collect();
-				match DB::open_cf(&opts, path, &cfnames) {
+				match DB::open_cf(&opts, path, &cfnames, &cf_options) {
 					Ok(db) => {
 						cfs = cfnames.iter().map(|n| db.cf_handle(n).unwrap()).collect();
 						assert!(cfs.len() == columns as usize);
@@ -250,9 +260,9 @@ impl Database {
 					}
 					Err(_) => {
 						// retry and create CFs
-						match DB::open_cf(&opts, path, &[]) {
+						match DB::open_cf(&opts, path, &[], &[]) {
 							Ok(mut db) => {
-								cfs = cfnames.iter().map(|n| db.create_cf(n, &opts).unwrap()).collect();
+								cfs = cfnames.iter().enumerate().map(|(i, n)| db.create_cf(n, &cf_options[i]).unwrap()).collect();
 								Ok(db)
 							},
 							err @ Err(_) => err,

util/src/trie/triedbmut.rs

@@ -56,11 +56,11 @@ impl From<H256> for NodeHandle {
 	}
 }
 
-fn empty_children() -> [Option<NodeHandle>; 16] {
-	[
+fn empty_children() -> Box<[Option<NodeHandle>; 16]> {
+	Box::new([
 		None, None, None, None, None, None, None, None,
 		None, None, None, None, None, None, None, None,
-	]
+	])
 }
 
 /// Node types in the Trie.
@@ -78,7 +78,7 @@ enum Node {
 	/// The child node is always a branch.
 	Extension(Bytes, NodeHandle),
 	/// A branch has up to 16 children and an optional value.
-	Branch([Option<NodeHandle>; 16], Option<Bytes>)
+	Branch(Box<[Option<NodeHandle>; 16]>, Option<Bytes>)
 }
 
 impl Node {
@@ -820,19 +820,19 @@ impl<'a> TrieDBMut<'a> {
 	/// Commit the in-memory changes to disk, freeing their storage and
 	/// updating the state root.
 	pub fn commit(&mut self) {
-		let handle = match self.root_handle() {
-			NodeHandle::Hash(_) => return, // no changes necessary.
-			NodeHandle::InMemory(h) => h,
-		};
-
 		trace!(target: "trie", "Committing trie changes to db.");
 
-		// kill all the nodes on death row.
+		// always kill all the nodes on death row.
 		trace!(target: "trie", "{:?} nodes to remove from db", self.death_row.len());
 		for hash in self.death_row.drain() {
 			self.db.remove(&hash);
 		}
 
+		let handle = match self.root_handle() {
+			NodeHandle::Hash(_) => return, // no changes necessary.
+			NodeHandle::InMemory(h) => h,
+		};
+
 		match self.storage.destroy(handle) {
 			Stored::New(node) => {
 				let root_rlp = node.into_rlp(|child, stream| self.commit_node(child, stream));
@@ -906,21 +906,29 @@ impl<'a> TrieMut for TrieDBMut<'a> {
 			return self.remove(key);
 		}
 
+		trace!(target: "trie", "insert: key={:?}, value={:?}", key.pretty(), value.pretty());
+
 		let root_handle = self.root_handle();
-		let (new_handle, _) = try!(self.insert_at(root_handle, NibbleSlice::new(key), value.to_owned()));
+		let (new_handle, changed) = try!(self.insert_at(root_handle, NibbleSlice::new(key), value.to_owned()));
+
+		trace!(target: "trie", "insert: altered trie={}", changed);
 		self.root_handle = NodeHandle::InMemory(new_handle);
 
 		Ok(())
 	}
 
 	fn remove(&mut self, key: &[u8]) -> super::Result<()> {
+		trace!(target: "trie", "remove: key={:?}", key.pretty());
+
 		let root_handle = self.root_handle();
 		let key = NibbleSlice::new(key);
 		match try!(self.remove_at(root_handle, key)) {
-			Some((handle, _)) => {
+			Some((handle, changed)) => {
+				trace!(target: "trie", "remove: altered trie={}", changed);
 				self.root_handle = NodeHandle::InMemory(handle);
 			}
 			None => {
+				trace!(target: "trie", "remove: obliterated trie");
 				self.root_handle = NodeHandle::Hash(SHA3_NULL_RLP);
 				*self.root = SHA3_NULL_RLP;
 			}