From 4df5f0ca3194f89d3c40cbbdd7a5c7fc71b4eb84 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sun, 17 Jan 2016 11:54:29 +0100 Subject: [PATCH 01/17] Handling all possible overflows --- src/evm/interpreter.rs | 119 ++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 68 deletions(-) diff --git a/src/evm/interpreter.rs b/src/evm/interpreter.rs index e6499b022..01c6564b0 100644 --- a/src/evm/interpreter.rs +++ b/src/evm/interpreter.rs @@ -27,6 +27,14 @@ fn color(instruction: Instruction, name: &'static str) -> String { format!("\x1B[1;{}m{}\x1B[0m", colors[c], name) } +macro_rules! overflowing { + ($x: expr) => {{ + let (v, overflow) = $x; + if overflow { return Err(evm::Error::OutOfGas); } + v + }} +} + type CodePosition = usize; type Gas = U256; type ProgramCounter = usize; @@ -385,14 +393,10 @@ impl Interpreter { InstructionCost::GasMem(default_gas, self.mem_needed(stack.peek(0), stack.peek(1))) }, instructions::SHA3 => { - match add_u256_usize(stack.peek(1), 31) { - (_w, true) => InstructionCost::GasMem(U256::zero(), RequiredMem::OutOfMemory), - (w, false) => { - let words = w >> 5; - let gas = U256::from(schedule.sha3_gas) + (U256::from(schedule.sha3_word_gas) * words); - InstructionCost::GasMem(gas, self.mem_needed(stack.peek(0), stack.peek(1))) - } - } + let w = overflowing!(add_u256_usize(stack.peek(1), 31)); + let words = w >> 5; + let gas = U256::from(schedule.sha3_gas) + (U256::from(schedule.sha3_word_gas) * words); + InstructionCost::GasMem(gas, self.mem_needed(stack.peek(0), stack.peek(1))) }, instructions::CALLDATACOPY => { InstructionCost::GasMemCopy(default_gas, self.mem_needed(stack.peek(0), stack.peek(2)), stack.peek(2).clone()) @@ -409,46 +413,37 @@ impl Interpreter { instructions::LOG0...instructions::LOG4 => { let no_of_topics = instructions::get_log_topics(instruction); let log_gas = schedule.log_gas + schedule.log_topic_gas * no_of_topics; - // TODO [todr] potential overflow of datagass - let data_gas = stack.peek(1).clone() * U256::from(schedule.log_data_gas); - let gas = try!(self.gas_add(data_gas, U256::from(log_gas))); + + let data_gas = overflowing!(stack.peek(1).overflowing_mul(U256::from(schedule.log_data_gas))); + let gas = overflowing!(data_gas.overflowing_add(U256::from(log_gas))); InstructionCost::GasMem(gas, self.mem_needed(stack.peek(0), stack.peek(1))) }, instructions::CALL | instructions::CALLCODE => { - match add_u256_usize(stack.peek(0), schedule.call_gas) { - (_gas, true) => InstructionCost::GasMem(U256::zero(), RequiredMem::OutOfMemory), - (mut gas, false) => { - let mem = self.mem_max( - self.mem_needed(stack.peek(5), stack.peek(6)), - self.mem_needed(stack.peek(3), stack.peek(4)) - ); - - let address = u256_to_address(stack.peek(1)); + let mut gas = overflowing!(add_u256_usize(stack.peek(0), schedule.call_gas)); + let mem = self.mem_max( + self.mem_needed(stack.peek(5), stack.peek(6)), + self.mem_needed(stack.peek(3), stack.peek(4)) + ); + + let address = u256_to_address(stack.peek(1)); - // TODO [todr] Potential overflows - if instruction == instructions::CALL && !ext.exists(&address) { - gas = gas + U256::from(schedule.call_new_account_gas); - }; + if instruction == instructions::CALL && !ext.exists(&address) { + gas = overflowing!(gas.overflowing_add(U256::from(schedule.call_new_account_gas))); + }; - if stack.peek(2).clone() > U256::zero() { - gas = gas + U256::from(schedule.call_value_transfer_gas) - }; + if stack.peek(2).clone() > U256::zero() { + gas = overflowing!(gas.overflowing_add(U256::from(schedule.call_value_transfer_gas))); + }; - InstructionCost::GasMem(gas,mem) - } - } + InstructionCost::GasMem(gas,mem) }, instructions::DELEGATECALL => { - match add_u256_usize(stack.peek(0), schedule.call_gas) { - (_gas, true) => InstructionCost::GasMem(U256::zero(), RequiredMem::OutOfMemory), - (gas, false) => { - let mem = self.mem_max( - self.mem_needed(stack.peek(4), stack.peek(5)), - self.mem_needed(stack.peek(2), stack.peek(3)) - ); - InstructionCost::GasMem(gas, mem) - } - } + let gas = overflowing!(add_u256_usize(stack.peek(0), schedule.call_gas)); + let mem = self.mem_max( + self.mem_needed(stack.peek(4), stack.peek(5)), + self.mem_needed(stack.peek(2), stack.peek(3)) + ); + InstructionCost::GasMem(gas, mem) }, instructions::CREATE => { let gas = U256::from(schedule.create_gas); @@ -471,7 +466,7 @@ impl Interpreter { InstructionCost::GasMem(gas, mem_size) => match mem_size { RequiredMem::Mem(mem_size) => { let (mem_gas, new_mem_size) = self.mem_gas_cost(schedule, mem.size(), &mem_size); - let gas = try!(self.gas_add(gas, mem_gas)); + let gas = overflowing!(gas.overflowing_add(mem_gas)); Ok((gas, new_mem_size)) }, RequiredMem::OutOfMemory => Err(evm::Error::OutOfGas) @@ -479,27 +474,17 @@ impl Interpreter { InstructionCost::GasMemCopy(gas, mem_size, copy) => match mem_size { RequiredMem::Mem(mem_size) => { let (mem_gas, new_mem_size) = self.mem_gas_cost(schedule, mem.size(), &mem_size); - match add_u256_usize(©, 31) { - (_c, true) => Err(evm::Error::OutOfGas), - (copy, false) => { - let copy_gas = U256::from(schedule.copy_gas) * (copy / U256::from(32)); - let gas = try!(self.gas_add(try!(self.gas_add(gas, copy_gas)), mem_gas)); - Ok((gas, new_mem_size)) - } - } + let copy = overflowing!(add_u256_usize(©, 31)); + let copy_gas = U256::from(schedule.copy_gas) * (copy / U256::from(32)); + let gas = overflowing!(gas.overflowing_add(copy_gas)); + let gas = overflowing!(gas.overflowing_add(mem_gas)); + Ok((gas, new_mem_size)) }, RequiredMem::OutOfMemory => Err(evm::Error::OutOfGas) } } } - fn gas_add(&self, a: U256, b: U256) -> Result { - match a.overflowing_add(b) { - (_val, true) => Err(evm::Error::OutOfGas), - (val, false) => Ok(val) - } - } - fn mem_gas_cost(&self, schedule: &evm::Schedule, current_mem_size: usize, mem_size: &U256) -> (U256, usize) { let gas_for_mem = |mem_size: U256| { let s = mem_size >> 5; @@ -924,23 +909,21 @@ impl Interpreter { instructions::DIV => { let a = stack.pop_back(); let b = stack.pop_back(); - stack.push(match !self.is_zero(&b) { - true => { - let (c, _overflow) = a.overflowing_div(b); - c - }, - false => U256::zero() + stack.push(if !self.is_zero(&b) { + let (c, _overflow) = a.overflowing_div(b); + c + } else { + U256::zero() }); }, instructions::MOD => { let a = stack.pop_back(); let b = stack.pop_back(); - stack.push(match !self.is_zero(&b) { - true => { - let (c, _overflow) = a.overflowing_rem(b); - c - }, - false => U256::zero() + stack.push(if !self.is_zero(&b) { + let (c, _overflow) = a.overflowing_rem(b); + c + } else { + U256::zero() }); }, instructions::SDIV => { From eb4a9d8586539ec1577a5e11f51551ec71ff81fc Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sun, 17 Jan 2016 12:06:37 +0100 Subject: [PATCH 02/17] Removing println --- src/evm/interpreter.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/evm/interpreter.rs b/src/evm/interpreter.rs index 01c6564b0..269db9b67 100644 --- a/src/evm/interpreter.rs +++ b/src/evm/interpreter.rs @@ -631,7 +631,6 @@ impl Interpreter { return match call_result { MessageCallResult::Success(gas_left) => { - println!("Unused: {}", gas_left); stack.push(U256::one()); Ok(InstructionResult::UnusedGas(gas_left)) }, From 97d23f551be70ed33efaa792974006a58abaaf7d Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sun, 17 Jan 2016 13:24:57 +0100 Subject: [PATCH 03/17] Cleaning ugly tuple deconstructions --- src/evm/interpreter.rs | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/evm/interpreter.rs b/src/evm/interpreter.rs index 269db9b67..7051b9887 100644 --- a/src/evm/interpreter.rs +++ b/src/evm/interpreter.rs @@ -144,7 +144,7 @@ trait Memory { /// Checks whether offset and size is valid memory range fn is_valid_range(off: usize, size: usize) -> bool { // When size is zero we haven't actually expanded the memory - let (_a, overflow) = off.overflowing_add(size); + let overflow = off.overflowing_add(size).1; size > 0 && !overflow } @@ -890,27 +890,23 @@ impl Interpreter { instructions::ADD => { let a = stack.pop_back(); let b = stack.pop_back(); - let (c, _overflow) = a.overflowing_add(b); - stack.push(c); + stack.push(a.overflowing_add(b).0); }, instructions::MUL => { let a = stack.pop_back(); let b = stack.pop_back(); - let (c, _overflow) = a.overflowing_mul(b); - stack.push(c); + stack.push(a.overflowing_mul(b).0); }, instructions::SUB => { let a = stack.pop_back(); let b = stack.pop_back(); - let (c, _overflow) = a.overflowing_sub(b); - stack.push(c); + stack.push(a.overflowing_sub(b).0); }, instructions::DIV => { let a = stack.pop_back(); let b = stack.pop_back(); stack.push(if !self.is_zero(&b) { - let (c, _overflow) = a.overflowing_div(b); - c + a.overflowing_div(b).0 } else { U256::zero() }); @@ -919,8 +915,7 @@ impl Interpreter { let a = stack.pop_back(); let b = stack.pop_back(); stack.push(if !self.is_zero(&b) { - let (c, _overflow) = a.overflowing_rem(b); - c + a.overflowing_rem(b).0 } else { U256::zero() }); @@ -936,7 +931,7 @@ impl Interpreter { } else if a == min && b == !U256::zero() { min } else { - let (c, _overflow) = a.overflowing_div(b); + let c = a.overflowing_div(b).0; set_sign(c, sign_a ^ sign_b) }); }, @@ -944,10 +939,10 @@ impl Interpreter { let ua = stack.pop_back(); let ub = stack.pop_back(); let (a, sign_a) = get_and_reset_sign(ua); - let (b, _sign_b) = get_and_reset_sign(ub); + let b = get_and_reset_sign(ub).0; stack.push(if !self.is_zero(&b) { - let (c, _overflow) = a.overflowing_rem(b); + let c = a.overflowing_rem(b).0; set_sign(c, sign_a) } else { U256::zero() @@ -956,7 +951,7 @@ impl Interpreter { instructions::EXP => { let base = stack.pop_back(); let expon = stack.pop_back(); - let (res, _overflow) = base.overflowing_pow(expon); + let res = base.overflowing_pow(expon).0; stack.push(res); }, instructions::NOT => { @@ -1034,8 +1029,8 @@ impl Interpreter { stack.push(if !self.is_zero(&c) { // upcast to 512 let a5 = U512::from(a); - let (res, _overflow) = a5.overflowing_add(U512::from(b)); - let (x, _overflow) = res.overflowing_rem(U512::from(c)); + let res = a5.overflowing_add(U512::from(b)).0; + let x = res.overflowing_rem(U512::from(c)).0; U256::from(x) } else { U256::zero() @@ -1048,8 +1043,8 @@ impl Interpreter { stack.push(if !self.is_zero(&c) { let a5 = U512::from(a); - let (res, _overflow) = a5.overflowing_mul(U512::from(b)); - let (x, _overflow) = res.overflowing_rem(U512::from(c)); + let res = a5.overflowing_mul(U512::from(b)).0; + let x = res.overflowing_rem(U512::from(c)).0; U256::from(x) } else { U256::zero() @@ -1105,8 +1100,7 @@ fn get_and_reset_sign(value: U256) -> (U256, bool) { fn set_sign(value: U256, sign: bool) -> U256 { if sign { - let (val, _overflow) = (!U256::zero() ^ value).overflowing_add(U256::one()); - val + (!U256::zero() ^ value).overflowing_add(U256::one()).0; } else { value } From 83ac4bce06057a7820655a72b4f8074b7625fe9d Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sun, 17 Jan 2016 13:47:16 +0100 Subject: [PATCH 04/17] Handling overflows in gas_for_mem calculation --- src/evm/interpreter.rs | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/evm/interpreter.rs b/src/evm/interpreter.rs index 7051b9887..e20df0f62 100644 --- a/src/evm/interpreter.rs +++ b/src/evm/interpreter.rs @@ -465,7 +465,7 @@ impl Interpreter { }, InstructionCost::GasMem(gas, mem_size) => match mem_size { RequiredMem::Mem(mem_size) => { - let (mem_gas, new_mem_size) = self.mem_gas_cost(schedule, mem.size(), &mem_size); + let (mem_gas, new_mem_size) = try!(self.mem_gas_cost(schedule, mem.size(), &mem_size)); let gas = overflowing!(gas.overflowing_add(mem_gas)); Ok((gas, new_mem_size)) }, @@ -473,7 +473,7 @@ impl Interpreter { }, InstructionCost::GasMemCopy(gas, mem_size, copy) => match mem_size { RequiredMem::Mem(mem_size) => { - let (mem_gas, new_mem_size) = self.mem_gas_cost(schedule, mem.size(), &mem_size); + let (mem_gas, new_mem_size) = try!(self.mem_gas_cost(schedule, mem.size(), &mem_size)); let copy = overflowing!(add_u256_usize(©, 31)); let copy_gas = U256::from(schedule.copy_gas) * (copy / U256::from(32)); let gas = overflowing!(gas.overflowing_add(copy_gas)); @@ -485,21 +485,24 @@ impl Interpreter { } } - fn mem_gas_cost(&self, schedule: &evm::Schedule, current_mem_size: usize, mem_size: &U256) -> (U256, usize) { + fn mem_gas_cost(&self, schedule: &evm::Schedule, current_mem_size: usize, mem_size: &U256) -> Result<(U256, usize), evm::Error> { let gas_for_mem = |mem_size: U256| { let s = mem_size >> 5; - s * U256::from(schedule.memory_gas) + s * s / U256::from(schedule.quad_coeff_div) + // s * memory_gas + s * s / quad_coeff_div + let a = overflowing!(s.overflowing_mul(U256::from(schedule.memory_gas))); + let b = overflowing!(s.overflowing_mul(s / U256::from(schedule.quad_coeff_div))); + Ok(a + b) }; let current_mem_size = U256::from(current_mem_size); - let req_mem_size_rounded = ((mem_size.clone() + U256::from(31)) >> 5) << 5; - let new_mem_gas = gas_for_mem(U256::from(req_mem_size_rounded)); - let current_mem_gas = gas_for_mem(current_mem_size); + let req_mem_size_rounded = (overflowing!(mem_size.overflowing_add(U256::from(31))) >> 5) << 5; + let new_mem_gas = try!(gas_for_mem(U256::from(req_mem_size_rounded))); + let current_mem_gas = try!(gas_for_mem(current_mem_size)); - (if req_mem_size_rounded > current_mem_size { + Ok((if req_mem_size_rounded > current_mem_size { new_mem_gas - current_mem_gas } else { U256::zero() - }, req_mem_size_rounded.low_u64() as usize) + }, req_mem_size_rounded.low_u64() as usize)) } fn mem_max(&self, m_a: RequiredMem, m_b: RequiredMem) -> RequiredMem { @@ -1100,7 +1103,7 @@ fn get_and_reset_sign(value: U256) -> (U256, bool) { fn set_sign(value: U256, sign: bool) -> U256 { if sign { - (!U256::zero() ^ value).overflowing_add(U256::one()).0; + (!U256::zero() ^ value).overflowing_add(U256::one()).0 } else { value } @@ -1121,6 +1124,23 @@ fn address_to_u256(value: Address) -> U256 { U256::from(H256::from(value).as_slice()) } +#[test] +fn test_mem_gas_cost() { + // given + let interpreter = Interpreter; + let schedule = evm::Schedule::default(); + let current_mem_size = 5; + let mem_size = !U256::zero(); + + // when + let result = interpreter.mem_gas_cost(&schedule, current_mem_size, &mem_size); + + // then + if let Ok(_) = result { + assert!(false, "Should fail with OutOfGas"); + } +} + #[cfg(test)] mod tests { use common::*; @@ -1149,7 +1169,7 @@ mod tests { let mem_size = U256::from(5); // when - let (mem_cost, mem_size) = interpreter.mem_gas_cost(&schedule, current_mem_size, &mem_size); + let (mem_cost, mem_size) = interpreter.mem_gas_cost(&schedule, current_mem_size, &mem_size).unwrap(); // then assert_eq!(mem_cost, U256::from(3)); From aca68dcfe71a67aba8aea13bdc1891e674045f47 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sun, 17 Jan 2016 13:59:30 +0100 Subject: [PATCH 05/17] Fixing mem_gas_cost calcualtion by jumping to U512 --- src/evm/interpreter.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/evm/interpreter.rs b/src/evm/interpreter.rs index e20df0f62..f3c5920e5 100644 --- a/src/evm/interpreter.rs +++ b/src/evm/interpreter.rs @@ -490,8 +490,9 @@ impl Interpreter { let s = mem_size >> 5; // s * memory_gas + s * s / quad_coeff_div let a = overflowing!(s.overflowing_mul(U256::from(schedule.memory_gas))); - let b = overflowing!(s.overflowing_mul(s / U256::from(schedule.quad_coeff_div))); - Ok(a + b) + // We need to go to U512 to calculate s*s/quad_coeff_div + let b = U256::from(U512::from(s) * U512::from(s) / U512::from(schedule.quad_coeff_div)); + Ok(overflowing!(a.overflowing_add(b))) }; let current_mem_size = U256::from(current_mem_size); let req_mem_size_rounded = (overflowing!(mem_size.overflowing_add(U256::from(31))) >> 5) << 5; From c8d94c981b1468a89562ccff4c554582a5f3e067 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sun, 17 Jan 2016 14:08:31 +0100 Subject: [PATCH 06/17] Additional check before conversion --- src/evm/interpreter.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/evm/interpreter.rs b/src/evm/interpreter.rs index f3c5920e5..47b66bd2c 100644 --- a/src/evm/interpreter.rs +++ b/src/evm/interpreter.rs @@ -491,8 +491,12 @@ impl Interpreter { // s * memory_gas + s * s / quad_coeff_div let a = overflowing!(s.overflowing_mul(U256::from(schedule.memory_gas))); // We need to go to U512 to calculate s*s/quad_coeff_div - let b = U256::from(U512::from(s) * U512::from(s) / U512::from(schedule.quad_coeff_div)); - Ok(overflowing!(a.overflowing_add(b))) + let b = U512::from(s) * U512::from(s) / U512::from(schedule.quad_coeff_div); + if b > U512::from(!U256::zero()) { + Err(evm::Error::OutOfGas) + } else { + Ok(overflowing!(a.overflowing_add(U256::from(b)))) + } }; let current_mem_size = U256::from(current_mem_size); let req_mem_size_rounded = (overflowing!(mem_size.overflowing_add(U256::from(31))) >> 5) << 5; From 342c4e736de2ccf2068b4ce557c123393a6575a5 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sun, 17 Jan 2016 14:17:05 +0100 Subject: [PATCH 07/17] Getting rid of RequiredMem enum --- src/evm/interpreter.rs | 98 +++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 64 deletions(-) diff --git a/src/evm/interpreter.rs b/src/evm/interpreter.rs index 47b66bd2c..bd3a153b9 100644 --- a/src/evm/interpreter.rs +++ b/src/evm/interpreter.rs @@ -243,15 +243,10 @@ impl<'a> CodeReader<'a> { } } -enum RequiredMem { - Mem(U256), - OutOfMemory -} - enum InstructionCost { Gas(U256), - GasMem(U256, RequiredMem), - GasMemCopy(U256, RequiredMem, U256) + GasMem(U256, U256), + GasMemCopy(U256, U256, U256) } enum InstructionResult { @@ -381,31 +376,31 @@ impl Interpreter { InstructionCost::Gas(U256::from(schedule.sload_gas)) }, instructions::MSTORE => { - InstructionCost::GasMem(default_gas, self.mem_needed_const(stack.peek(0), 32)) + InstructionCost::GasMem(default_gas, try!(self.mem_needed_const(stack.peek(0), 32))) }, instructions::MLOAD => { - InstructionCost::GasMem(default_gas, self.mem_needed_const(stack.peek(0), 32)) + InstructionCost::GasMem(default_gas, try!(self.mem_needed_const(stack.peek(0), 32))) }, instructions::MSTORE8 => { - InstructionCost::GasMem(default_gas, self.mem_needed_const(stack.peek(0), 1)) + InstructionCost::GasMem(default_gas, try!(self.mem_needed_const(stack.peek(0), 1))) }, instructions::RETURN => { - InstructionCost::GasMem(default_gas, self.mem_needed(stack.peek(0), stack.peek(1))) + InstructionCost::GasMem(default_gas, try!(self.mem_needed(stack.peek(0), stack.peek(1)))) }, instructions::SHA3 => { let w = overflowing!(add_u256_usize(stack.peek(1), 31)); let words = w >> 5; let gas = U256::from(schedule.sha3_gas) + (U256::from(schedule.sha3_word_gas) * words); - InstructionCost::GasMem(gas, self.mem_needed(stack.peek(0), stack.peek(1))) + InstructionCost::GasMem(gas, try!(self.mem_needed(stack.peek(0), stack.peek(1)))) }, instructions::CALLDATACOPY => { - InstructionCost::GasMemCopy(default_gas, self.mem_needed(stack.peek(0), stack.peek(2)), stack.peek(2).clone()) + InstructionCost::GasMemCopy(default_gas, try!(self.mem_needed(stack.peek(0), stack.peek(2))), stack.peek(2).clone()) }, instructions::CODECOPY => { - InstructionCost::GasMemCopy(default_gas, self.mem_needed(stack.peek(0), stack.peek(2)), stack.peek(2).clone()) + InstructionCost::GasMemCopy(default_gas, try!(self.mem_needed(stack.peek(0), stack.peek(2))), stack.peek(2).clone()) }, instructions::EXTCODECOPY => { - InstructionCost::GasMemCopy(default_gas, self.mem_needed(stack.peek(1), stack.peek(3)), stack.peek(3).clone()) + InstructionCost::GasMemCopy(default_gas, try!(self.mem_needed(stack.peek(1), stack.peek(3))), stack.peek(3).clone()) }, instructions::JUMPDEST => { InstructionCost::Gas(U256::one()) @@ -416,13 +411,13 @@ impl Interpreter { let data_gas = overflowing!(stack.peek(1).overflowing_mul(U256::from(schedule.log_data_gas))); let gas = overflowing!(data_gas.overflowing_add(U256::from(log_gas))); - InstructionCost::GasMem(gas, self.mem_needed(stack.peek(0), stack.peek(1))) + InstructionCost::GasMem(gas, try!(self.mem_needed(stack.peek(0), stack.peek(1)))) }, instructions::CALL | instructions::CALLCODE => { let mut gas = overflowing!(add_u256_usize(stack.peek(0), schedule.call_gas)); - let mem = self.mem_max( - self.mem_needed(stack.peek(5), stack.peek(6)), - self.mem_needed(stack.peek(3), stack.peek(4)) + let mem = cmp::max( + try!(self.mem_needed(stack.peek(5), stack.peek(6))), + try!(self.mem_needed(stack.peek(3), stack.peek(4))) ); let address = u256_to_address(stack.peek(1)); @@ -439,15 +434,15 @@ impl Interpreter { }, instructions::DELEGATECALL => { let gas = overflowing!(add_u256_usize(stack.peek(0), schedule.call_gas)); - let mem = self.mem_max( - self.mem_needed(stack.peek(4), stack.peek(5)), - self.mem_needed(stack.peek(2), stack.peek(3)) + let mem = cmp::max( + try!(self.mem_needed(stack.peek(4), stack.peek(5))), + try!(self.mem_needed(stack.peek(2), stack.peek(3))) ); InstructionCost::GasMem(gas, mem) }, instructions::CREATE => { let gas = U256::from(schedule.create_gas); - let mem = self.mem_needed(stack.peek(1), stack.peek(2)); + let mem = try!(self.mem_needed(stack.peek(1), stack.peek(2))); InstructionCost::GasMem(gas, mem) }, instructions::EXP => { @@ -463,24 +458,18 @@ impl Interpreter { InstructionCost::Gas(gas) => { Ok((gas, 0)) }, - InstructionCost::GasMem(gas, mem_size) => match mem_size { - RequiredMem::Mem(mem_size) => { - let (mem_gas, new_mem_size) = try!(self.mem_gas_cost(schedule, mem.size(), &mem_size)); - let gas = overflowing!(gas.overflowing_add(mem_gas)); - Ok((gas, new_mem_size)) - }, - RequiredMem::OutOfMemory => Err(evm::Error::OutOfGas) + InstructionCost::GasMem(gas, mem_size) => { + let (mem_gas, new_mem_size) = try!(self.mem_gas_cost(schedule, mem.size(), &mem_size)); + let gas = overflowing!(gas.overflowing_add(mem_gas)); + Ok((gas, new_mem_size)) }, - InstructionCost::GasMemCopy(gas, mem_size, copy) => match mem_size { - RequiredMem::Mem(mem_size) => { - let (mem_gas, new_mem_size) = try!(self.mem_gas_cost(schedule, mem.size(), &mem_size)); - let copy = overflowing!(add_u256_usize(©, 31)); - let copy_gas = U256::from(schedule.copy_gas) * (copy / U256::from(32)); - let gas = overflowing!(gas.overflowing_add(copy_gas)); - let gas = overflowing!(gas.overflowing_add(mem_gas)); - Ok((gas, new_mem_size)) - }, - RequiredMem::OutOfMemory => Err(evm::Error::OutOfGas) + InstructionCost::GasMemCopy(gas, mem_size, copy) => { + let (mem_gas, new_mem_size) = try!(self.mem_gas_cost(schedule, mem.size(), &mem_size)); + let copy = overflowing!(add_u256_usize(©, 31)); + let copy_gas = U256::from(schedule.copy_gas) * (copy / U256::from(32)); + let gas = overflowing!(gas.overflowing_add(copy_gas)); + let gas = overflowing!(gas.overflowing_add(mem_gas)); + Ok((gas, new_mem_size)) } } } @@ -510,35 +499,16 @@ impl Interpreter { }, req_mem_size_rounded.low_u64() as usize)) } - fn mem_max(&self, m_a: RequiredMem, m_b: RequiredMem) -> RequiredMem { - match (m_a, m_b) { - (RequiredMem::Mem(a), RequiredMem::Mem(b)) => { - RequiredMem::Mem(cmp::max(a, b)) - }, - (RequiredMem::OutOfMemory, _) | (_, RequiredMem::OutOfMemory) => { - RequiredMem::OutOfMemory - } - } + fn mem_needed_const(&self, mem: &U256, add: usize) -> Result { + Ok(overflowing!(mem.overflowing_add(U256::from(add)))) } - fn mem_needed_const(&self, mem: &U256, add: usize) -> RequiredMem { - match mem.overflowing_add(U256::from(add)) { - (_, true) => RequiredMem::OutOfMemory, - (mem, false) => RequiredMem::Mem(mem) - } - } - - fn mem_needed(&self, offset: &U256, size: &U256) -> RequiredMem { + fn mem_needed(&self, offset: &U256, size: &U256) -> Result { if self.is_zero(size) { - return RequiredMem::Mem(U256::zero()); + return Ok(U256::zero()); } - match offset.clone().overflowing_add(size.clone()) { - (_result, true) => RequiredMem::OutOfMemory, - (result, false) => { - RequiredMem::Mem(result) - } - } + Ok(overflowing!(offset.overflowing_add(size.clone()))) } fn exec_instruction(&self, From 62a0737e59cc904d5c98299b6a7ba26ddf6bbcbd Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 17 Jan 2016 22:18:47 +0100 Subject: [PATCH 08/17] Iterator for NibbleSlice and TrieDB. --- util/src/nibbleslice.rs | 30 +++++++- util/src/trie/node.rs | 2 +- util/src/trie/triedb.rs | 138 +++++++++++++++++++++++++++++++++++++ util/src/trie/triedbmut.rs | 5 +- 4 files changed, 171 insertions(+), 4 deletions(-) diff --git a/util/src/nibbleslice.rs b/util/src/nibbleslice.rs index 6f4232945..b9028dff3 100644 --- a/util/src/nibbleslice.rs +++ b/util/src/nibbleslice.rs @@ -34,6 +34,22 @@ pub struct NibbleSlice<'a> { offset_encode_suffix: usize, } +pub struct NibbleSliceIterator<'a> { + p: &'a NibbleSlice<'a>, + i: usize, +} + +impl<'a> Iterator for NibbleSliceIterator<'a> { + type Item = u8; + fn next(&mut self) -> Option { + self.i += 1; + match self.i <= self.p.len() { + true => Some(self.p.at(self.i - 1)), + false => None, + } + } +} + impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { /// Create a new nibble slice with the given byte-slice. pub fn new(data: &[u8]) -> NibbleSlice { NibbleSlice::new_offset(data, 0) } @@ -41,7 +57,7 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { /// Create a new nibble slice with the given byte-slice with a nibble offset. pub fn new_offset(data: &'a [u8], offset: usize) -> NibbleSlice { NibbleSlice{data: data, offset: offset, data_encode_suffix: &b""[..], offset_encode_suffix: 0} } - /// + /// Create a composed nibble slice; one followed by the other. pub fn new_composed(a: &'a NibbleSlice, b: &'a NibbleSlice) -> NibbleSlice<'a> { NibbleSlice{data: a.data, offset: a.offset, data_encode_suffix: b.data, offset_encode_suffix: b.offset} } /*pub fn new_composed_bytes_offset(a: &NibbleSlice, b: &NibbleSlice) -> (Bytes, usize) { @@ -60,6 +76,10 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { (r, a.len() + b.len()) }*/ + pub fn iter(&'a self) -> NibbleSliceIterator<'a> { + NibbleSliceIterator { p: self, i: 0 } + } + /// Create a new nibble slice from the given HPE encoded data (e.g. output of `encoded()`). pub fn from_encoded(data: &'a [u8]) -> (NibbleSlice, bool) { (Self::new_offset(data, if data[0] & 16 == 16 {1} else {2}), data[0] & 32 == 32) @@ -189,6 +209,14 @@ mod tests { } } + #[test] + fn iterator() { + let n = NibbleSlice::new(D); + let mut nibbles: Vec = vec![]; + nibbles.extend(n.iter()); + assert_eq!(nibbles, (0u8..6).collect::>()) + } + #[test] fn mid() { let n = NibbleSlice::new(D); diff --git a/util/src/trie/node.rs b/util/src/trie/node.rs index b5745b66f..b10b0e05e 100644 --- a/util/src/trie/node.rs +++ b/util/src/trie/node.rs @@ -5,7 +5,7 @@ use rlp::*; use super::journal::*; /// Type of node in the trie and essential information thereof. -#[derive(Eq, PartialEq, Debug)] +#[derive(Clone, Eq, PartialEq, Debug)] pub enum Node<'a> { Empty, Leaf(NibbleSlice<'a>, &'a[u8]), diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index bd34e710d..0ee95ea4e 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -198,6 +198,126 @@ impl<'db> TrieDB<'db> { } } +#[derive(Clone, Eq, PartialEq)] +enum Status { + Entering, + At, + AtChild(usize), + Exiting, +} + +#[derive(Clone, Eq, PartialEq)] +struct Crumb<'a> { + node: Node<'a>, +// key: &'a[u8], + status: Status, +} + +impl<'a> Crumb<'a> { + /// Move on to next status in the node's sequence. + fn increment(&mut self) { + self.status = match (&self.status, &self.node) { + (_, &Node::Empty) => Status::Exiting, + (&Status::Entering, _) => Status::At, + (&Status::At, &Node::Branch(_, _)) => Status::AtChild(0), + (&Status::AtChild(x), &Node::Branch(_, _)) if x < 15 => Status::AtChild(x + 1), + _ => Status::Exiting, + } + } +} + +/// Iterator for going through all values in the trie. +#[derive(Clone)] +pub struct TrieDBIterator<'a> { + db: &'a TrieDB<'a>, + trail: Vec>, + key_nibbles: Bytes, +} + +impl<'a> TrieDBIterator<'a> { + /// Create a new iterator. + fn new(db: &'a TrieDB) -> TrieDBIterator<'a> { + let mut r = TrieDBIterator { + db: db, + trail: vec![], + key_nibbles: Vec::new(), + }; + r.descend(db.db.lookup(&db.root).expect("Trie root not found!")); + r + } + + /// Descend into a payload. + fn descend(&mut self, d: &'a [u8]) { + self.trail.push(Crumb { + status: Status::Entering, + node: self.db.get_node(d) + }); + match self.trail.last().unwrap().node { + Node::Leaf(n, _) | Node::Extension(n, _) => { + println!("Entering. Extend key {:?}, {:?}", self.key_nibbles, n.iter().collect::>()); + self.key_nibbles.extend(n.iter()); + }, + _ => {} + } + } + + /// Descend into a payload and get the next item. + fn descend_next(&mut self, d: &'a [u8]) -> Option<(Bytes, &'a [u8])> { self.descend(d); self.next() } + + /// The present key. + fn key(&self) -> Bytes { + // collapse the key_nibbles down to bytes. + self.key_nibbles.iter().step(2).zip(self.key_nibbles.iter().skip(1).step(2)).map(|(h, l)| h * 16 + l).collect() + } +} + +impl<'a> Iterator for TrieDBIterator<'a> { + type Item = (Bytes, &'a [u8]); + + fn next(&mut self) -> Option { + let b = match self.trail.last_mut() { + Some(ref mut b) => { b.increment(); b.clone() }, + None => return None + }; + match (b.status, b.node) { + (Status::Exiting, n) => { + match n { + Node::Leaf(n, _) | Node::Extension(n, _) => { + println!("Exiting. Truncate key {:?}, {:?}", self.key_nibbles, n.iter().collect::>()); + let l = self.key_nibbles.len(); + self.key_nibbles.truncate(l - n.len()); + }, + Node::Branch(_, _) => { println!("Exit branch. Pop {:?}.", self.key_nibbles); self.key_nibbles.pop(); }, + _ => {} + } + self.trail.pop(); + self.next() + }, + (Status::At, Node::Leaf(_, v)) => Some((self.key(), v)), + (Status::At, Node::Extension(_, d)) => self.descend_next(d), + (Status::At, Node::Branch(_, Some(v))) => Some((self.key(), v)), + (Status::At, Node::Branch(_, _)) => self.next(), + (Status::AtChild(i), Node::Branch(children, _)) if children[i].len() > 0 => { + match i { + 0 => { println!("Enter first child of branch. Push {:?}.", self.key_nibbles); self.key_nibbles.push(0); }, + i => *self.key_nibbles.last_mut().unwrap() = i as u8, + } + self.descend_next(children[i]) + }, + (Status::AtChild(i), Node::Branch(_, _)) => { + if i == 0 { println!("Enter first child of branch. Push {:?}.", self.key_nibbles); self.key_nibbles.push(0); } + self.next() + }, + _ => panic!() // Should never see Entering or AtChild without a Branch here. + } + } +} + +impl<'db> TrieDB<'db> { + /// Get all keys/values stored in the trie. + pub fn iter(&self) -> TrieDBIterator { TrieDBIterator::new(self) } +} + impl<'db> Trie for TrieDB<'db> { fn root(&self) -> &H256 { &self.root } @@ -218,3 +338,21 @@ impl<'db> fmt::Debug for TrieDB<'db> { writeln!(f, "]") } } + +#[test] +fn iterator() { + use memorydb::*; + use super::triedbmut::*; + + let d = vec![ &b"A"[..], &b"AA"[..], &b"AB"[..], &b"B"[..] ]; + + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + { + let mut t = TrieDBMut::new(&mut memdb, &mut root); + for x in &d { + t.insert(&x, &x); + } + } + assert_eq!(d.iter().map(|i|i.to_vec()).collect::>(), TrieDB::new(&memdb, &root).iter().map(|x|x.0).collect::>()); +} diff --git a/util/src/trie/triedbmut.rs b/util/src/trie/triedbmut.rs index 832b532f8..0f3dde4fb 100644 --- a/util/src/trie/triedbmut.rs +++ b/util/src/trie/triedbmut.rs @@ -65,8 +65,9 @@ impl<'db> TrieDBMut<'db> { r } - /// Create a new trie with the backing database `db` and `root` - /// Panics, if `root` does not exist + /// Create a new trie with the backing database `db` and `root`. + /// Panics, if `root` does not exist. + // TODO: return Result pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { if !db.exists(root) { flush(format!("Trie root not found {}", root)); From 9203dbdca21dfc7d36b877d5cd4068a09a4bf4f5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 17 Jan 2016 22:24:42 +0100 Subject: [PATCH 09/17] Minor tidy up. --- util/src/trie/triedb.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index 0ee95ea4e..9cc6103c4 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -109,7 +109,12 @@ impl<'db> TrieDB<'db> { /// Get the root node's RLP. fn root_node(&self) -> Node { - Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) + Node::decoded(self.root_data()) + } + + /// Get the data of the root node. + fn root_data(&self) -> &[u8] { + self.db.lookup(&self.root).expect("Trie root not found!") } /// Get the root node as a `Node`. @@ -242,7 +247,7 @@ impl<'a> TrieDBIterator<'a> { trail: vec![], key_nibbles: Vec::new(), }; - r.descend(db.db.lookup(&db.root).expect("Trie root not found!")); + r.descend(db.root_data()); r } From 589ecf10af02655b48bcde011b10af5df7007c0f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 17 Jan 2016 22:30:07 +0100 Subject: [PATCH 10/17] Additonal test for Trie iterator. --- util/src/trie/triedb.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index 9cc6103c4..ebb97f74c 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -360,4 +360,5 @@ fn iterator() { } } assert_eq!(d.iter().map(|i|i.to_vec()).collect::>(), TrieDB::new(&memdb, &root).iter().map(|x|x.0).collect::>()); + assert_eq!(d, TrieDB::new(&memdb, &root).iter().map(|x|x.1).collect::>()); } From a131c33bb228cf3bf3d4786546c2193098ee3d9b Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 17 Jan 2016 23:07:58 +0100 Subject: [PATCH 11/17] Multithreaded block queue --- Cargo.toml | 1 + src/block.rs | 34 ++++--- src/client.rs | 103 +++++++++++--------- src/ethereum/ethash.rs | 4 + src/lib.rs | 1 + src/queue.rs | 214 +++++++++++++++++++++++++++++++++++++---- src/service.rs | 4 +- src/sync/chain.rs | 4 +- src/sync/mod.rs | 2 +- src/sync/tests.rs | 6 +- src/verification.rs | 37 +++++-- util/src/io/service.rs | 2 +- 12 files changed, 317 insertions(+), 95 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cea775ef3..0583aa78f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ time = "0.1" #interpolate_idents = { git = "https://github.com/SkylerLipthay/interpolate_idents" } evmjit = { path = "rust-evmjit", optional = true } ethash = { path = "ethash" } +num_cpus = "0.2" [features] jit = ["evmjit"] diff --git a/src/block.rs b/src/block.rs index d149d6132..e5ca18e8c 100644 --- a/src/block.rs +++ b/src/block.rs @@ -1,6 +1,7 @@ use common::*; use engine::*; use state::*; +use verification::PreVerifiedBlock; /// A transaction/receipt execution entry. pub struct Entry { @@ -263,30 +264,39 @@ impl IsBlock for SealedBlock { fn block(&self) -> &Block { &self.block } } -/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact<'x, 'y>(block_bytes: &[u8], engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +/// Enact the block given by block header, transactions and uncles +pub fn enact<'x, 'y>(header: &Header, transactions: &[Transaction], uncles: &[Header], engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { { - let header = BlockView::new(block_bytes).header_view(); let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce()); trace!("enact(): root={}, author={}, author_balance={}\n", s.root(), header.author(), s.balance(&header.author())); } - let block = BlockView::new(block_bytes); - let header = block.header_view(); - let mut b = OpenBlock::new(engine, db, parent, last_hashes, header.author(), header.extra_data()); - b.set_difficulty(header.difficulty()); - b.set_gas_limit(header.gas_limit()); + let mut b = OpenBlock::new(engine, db, parent, last_hashes, header.author().clone(), header.extra_data().clone()); + b.set_difficulty(*header.difficulty()); + b.set_gas_limit(*header.gas_limit()); b.set_timestamp(header.timestamp()); -// info!("enact: Enacting #{}. env_info={:?}", header.number(), b.env_info()); - for t in block.transactions().into_iter() { try!(b.push_transaction(t, None)); } - for u in block.uncles().into_iter() { try!(b.push_uncle(u)); } + for t in transactions { try!(b.push_transaction(t.clone(), None)); } + for u in uncles { try!(b.push_uncle(u.clone())); } Ok(b.close()) } +/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header +pub fn enact_bytes<'x, 'y>(block_bytes: &[u8], engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { + let block = BlockView::new(block_bytes); + let header = block.header(); + enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) +} + +/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header +pub fn enact_verified<'x, 'y>(block: &PreVerifiedBlock, engine: &'x Engine, db: OverlayDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { + let view = BlockView::new(&block.bytes); + enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) +} + /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: OverlayDB, parent: &Header, last_hashes: &LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); - Ok(try!(try!(enact(block_bytes, engine, db, parent, last_hashes)).seal(header.seal()))) + Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(header.seal()))) } #[test] diff --git a/src/client.rs b/src/client.rs index 3ee84ccd7..e02ab37d8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -88,7 +88,7 @@ pub trait BlockChainClient : Sync + Send { fn block_receipts(&self, hash: &H256) -> Option; /// Import a block into the blockchain. - fn import_block(&mut self, byte: &[u8]) -> ImportResult; + fn import_block(&mut self, bytes: Bytes) -> ImportResult; /// Get block queue information. fn queue_status(&self) -> BlockQueueStatus; @@ -152,58 +152,75 @@ impl Client { } /// This is triggered by a message coming from a block queue when the block is ready for insertion - pub fn import_verified_block(&mut self, bytes: Bytes) { - let block = BlockView::new(&bytes); - let header = block.header(); - if let Err(e) = verify_block_family(&header, &bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) { - warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.queue.mark_as_bad(&header.hash()); + pub fn import_verified_blocks(&mut self) { + + let mut bad = HashSet::new(); + let blocks = self.queue.drain(128); + if blocks.is_empty() { return; - }; - let parent = match self.chain.read().unwrap().block_header(&header.parent_hash) { - Some(p) => p, - None => { - warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); + } + + for block in blocks { + if bad.contains(&block.header.parent_hash) { + self.queue.mark_as_bad(&block.header.hash()); + bad.insert(block.header.hash()); + continue; + } + + let header = &block.header; + if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) { + warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); self.queue.mark_as_bad(&header.hash()); + bad.insert(block.header.hash()); return; - }, - }; - // build last hashes - let mut last_hashes = LastHashes::new(); - last_hashes.resize(256, H256::new()); - last_hashes[0] = header.parent_hash.clone(); - for i in 0..255 { - match self.chain.read().unwrap().block_details(&last_hashes[i]) { - Some(details) => { - last_hashes[i + 1] = details.parent.clone(); + }; + let parent = match self.chain.read().unwrap().block_header(&header.parent_hash) { + Some(p) => p, + None => { + warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); + self.queue.mark_as_bad(&header.hash()); + bad.insert(block.header.hash()); + return; }, - None => break, + }; + // build last hashes + let mut last_hashes = LastHashes::new(); + last_hashes.resize(256, H256::new()); + last_hashes[0] = header.parent_hash.clone(); + for i in 0..255 { + match self.chain.read().unwrap().block_details(&last_hashes[i]) { + Some(details) => { + last_hashes[i + 1] = details.parent.clone(); + }, + None => break, + } } - } - let result = match enact(&bytes, self.engine.deref().deref(), self.state_db.clone(), &parent, &last_hashes) { - Ok(b) => b, - Err(e) => { - warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + let result = match enact_verified(&block, self.engine.deref().deref(), self.state_db.clone(), &parent, &last_hashes) { + Ok(b) => b, + Err(e) => { + warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + bad.insert(block.header.hash()); + self.queue.mark_as_bad(&header.hash()); + return; + } + }; + if let Err(e) = verify_block_final(&header, result.block().header()) { + warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); self.queue.mark_as_bad(&header.hash()); return; } - }; - if let Err(e) = verify_block_final(&header, result.block().header()) { - warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.queue.mark_as_bad(&header.hash()); - return; - } - self.chain.write().unwrap().insert_block(&bytes); //TODO: err here? - match result.drain().commit() { - Ok(_) => (), - Err(e) => { - warn!(target: "client", "State DB commit failed: {:?}", e); - return; + self.chain.write().unwrap().insert_block(&block.bytes); //TODO: err here? + match result.drain().commit() { + Ok(_) => (), + Err(e) => { + warn!(target: "client", "State DB commit failed: {:?}", e); + return; + } } + info!(target: "client", "Imported #{} ({})", header.number(), header.hash()); } - info!(target: "client", "Imported #{} ({})", header.number(), header.hash()); } } @@ -261,8 +278,8 @@ impl BlockChainClient for Client { unimplemented!(); } - fn import_block(&mut self, bytes: &[u8]) -> ImportResult { - let header = BlockView::new(bytes).header(); + fn import_block(&mut self, bytes: Bytes) -> ImportResult { + let header = BlockView::new(&bytes).header(); if self.chain.read().unwrap().is_known(&header.hash()) { return Err(ImportError::AlreadyInChain); } diff --git a/src/ethereum/ethash.rs b/src/ethereum/ethash.rs index 99ffc3186..8a86cf4e5 100644 --- a/src/ethereum/ethash.rs +++ b/src/ethereum/ethash.rs @@ -146,6 +146,10 @@ impl Engine for Ethash { } Ok(()) } + + fn verify_transaction(&self, t: &Transaction, _header: &Header) -> Result<(), Error> { + t.sender().map(|_|()) // Perform EC recovery and cache sender + } } impl Ethash { diff --git a/src/lib.rs b/src/lib.rs index 7e4fdf33e..8180736a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,7 @@ extern crate heapsize; extern crate crypto; extern crate time; extern crate env_logger; +extern crate num_cpus; #[cfg(feature = "jit" )] extern crate evmjit; #[macro_use] diff --git a/src/queue.rs b/src/queue.rs index 5ca361834..a51d3014c 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -1,59 +1,231 @@ +use std::thread::{JoinHandle, self}; +use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; use util::*; use verification::*; use error::*; use engine::Engine; use sync::*; use views::*; +use header::*; /// A queue of blocks. Sits between network or other I/O and the BlockChain. /// Sorts them ready for blockchain insertion. pub struct BlockQueue { engine: Arc>, + more_to_verify: Arc, + verification: Arc>, + verifiers: Vec>, + deleting: Arc, + ready_signal: Arc, + processing: HashSet +} + +struct UnVerifiedBlock { + header: Header, + bytes: Bytes, +} + +struct VerifyingBlock { + hash: H256, + block: Option, +} + +struct QueueSignal { + signalled: AtomicBool, message_channel: IoChannel, +} + +impl QueueSignal { + fn set(&self) { + if self.signalled.compare_and_swap(false, true, AtomicOrdering::Relaxed) == false { + self.message_channel.send(UserMessage(SyncMessage::BlockVerified)).expect("Error seding BlockVerified message"); + } + } + fn reset(&self) { + self.signalled.store(false, AtomicOrdering::Relaxed); + } +} + +#[derive(Default)] +struct Verification { + unverified: VecDeque, + verified: VecDeque, + verifying: VecDeque, bad: HashSet, } impl BlockQueue { /// Creates a new queue instance. pub fn new(engine: Arc>, message_channel: IoChannel) -> BlockQueue { + let verification = Arc::new(Mutex::new(Verification::default())); + let more_to_verify = Arc::new(Condvar::new()); + let ready_signal = Arc::new(QueueSignal { signalled: AtomicBool::new(false), message_channel: message_channel }); + let deleting = Arc::new(AtomicBool::new(false)); + + let mut verifiers: Vec> = Vec::new(); + let thread_count = max(::num_cpus::get(), 2) - 1; + for _ in 0..thread_count { + let verification = verification.clone(); + let engine = engine.clone(); + let more_to_verify = more_to_verify.clone(); + let ready_signal = ready_signal.clone(); + let deleting = deleting.clone(); + verifiers.push(thread::spawn(move || BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting))); + } BlockQueue { engine: engine, - message_channel: message_channel, - bad: HashSet::new(), + ready_signal: ready_signal.clone(), + more_to_verify: more_to_verify.clone(), + verification: verification.clone(), + verifiers: verifiers, + deleting: deleting.clone(), + processing: HashSet::new(), + } + } + + fn verify(verification: Arc>, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc) { + while !deleting.load(AtomicOrdering::Relaxed) { + { + let mut lock = verification.lock().unwrap(); + while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Relaxed) { + lock = wait.wait(lock).unwrap(); + } + + if deleting.load(AtomicOrdering::Relaxed) { + return; + } + } + + let block = { + let mut v = verification.lock().unwrap(); + if v.unverified.is_empty() { + continue; + } + let block = v.unverified.pop_front().unwrap(); + v.verifying.push_back(VerifyingBlock{ hash: block.header.hash(), block: None }); + block + }; + + let block_hash = block.header.hash(); + match verify_block_unordered(block.header, block.bytes, engine.deref().deref()) { + Ok(verified) => { + let mut v = verification.lock().unwrap(); + for e in &mut v.verifying { + if e.hash == block_hash { + e.block = Some(verified); + break; + } + } + if !v.verifying.is_empty() && v.verifying.front().unwrap().hash == block_hash { + // we're next! + let mut vref = v.deref_mut(); + BlockQueue::drain_verifying(&mut vref.verifying, &mut vref.verified, &mut vref.bad); + ready.set(); + } + }, + Err(err) => { + let mut v = verification.lock().unwrap(); + warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); + v.bad.insert(block_hash.clone()); + v.verifying.retain(|e| e.hash != block_hash); + let mut vref = v.deref_mut(); + BlockQueue::drain_verifying(&mut vref.verifying, &mut vref.verified, &mut vref.bad); + ready.set(); + } + } + } + } + + fn drain_verifying(verifying: &mut VecDeque, verified: &mut VecDeque, bad: &mut HashSet) { + while !verifying.is_empty() && verifying.front().unwrap().block.is_some() { + let block = verifying.pop_front().unwrap().block.unwrap(); + if bad.contains(&block.header.parent_hash) { + bad.insert(block.header.hash()); + } + else { + verified.push_back(block); + } } } /// Clear the queue and stop verification activity. pub fn clear(&mut self) { + let mut verification = self.verification.lock().unwrap(); + verification.unverified.clear(); + verification.verifying.clear(); } /// Add a block to the queue. - pub fn import_block(&mut self, bytes: &[u8]) -> ImportResult { - let header = BlockView::new(bytes).header(); - if self.bad.contains(&header.hash()) { - return Err(ImportError::Bad(None)); + pub fn import_block(&mut self, bytes: Bytes) -> ImportResult { + let header = BlockView::new(&bytes).header(); + if self.processing.contains(&header.hash()) { + return Err(ImportError::AlreadyQueued); + } + { + let mut verification = self.verification.lock().unwrap(); + if verification.bad.contains(&header.hash()) { + return Err(ImportError::Bad(None)); + } + + if verification.bad.contains(&header.parent_hash) { + verification.bad.insert(header.hash()); + return Err(ImportError::Bad(None)); + } } - if self.bad.contains(&header.parent_hash) { - self.bad.insert(header.hash()); - return Err(ImportError::Bad(None)); + match verify_block_basic(&header, &bytes, self.engine.deref().deref()) { + Ok(()) => { + self.processing.insert(header.hash()); + self.verification.lock().unwrap().unverified.push_back(UnVerifiedBlock { header: header, bytes: bytes }); + self.more_to_verify.notify_all(); + }, + Err(err) => { + warn!(target: "client", "Stage 1 block verification failed for {}\nError: {:?}", BlockView::new(&bytes).header_view().sha3(), err); + self.verification.lock().unwrap().bad.insert(header.hash()); + } } - - try!(verify_block_basic(&header, bytes, self.engine.deref().deref()).map_err(|e| { - warn!(target: "client", "Stage 1 block verification failed for {}\nError: {:?}", BlockView::new(&bytes).header_view().sha3(), e); - e - })); - try!(verify_block_unordered(&header, bytes, self.engine.deref().deref()).map_err(|e| { - warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", BlockView::new(&bytes).header_view().sha3(), e); - e - })); - try!(self.message_channel.send(UserMessage(SyncMessage::BlockVerified(bytes.to_vec()))).map_err(|e| Error::from(e))); Ok(()) } + /// Mark given block and all its children as bad. Stops verification. pub fn mark_as_bad(&mut self, hash: &H256) { - self.bad.insert(hash.clone()); - //TODO: walk the queue + let mut verification_lock = self.verification.lock().unwrap(); + let mut verification = verification_lock.deref_mut(); + verification.bad.insert(hash.clone()); + let mut new_verified = VecDeque::new(); + for block in verification.verified.drain(..) { + if verification.bad.contains(&block.header.parent_hash) { + verification.bad.insert(block.header.hash()); + } + else { + new_verified.push_back(block); + } + } + verification.verified = new_verified; + } + + pub fn drain(&mut self, max: usize) -> Vec { + let mut verification = self.verification.lock().unwrap(); + let count = min(max, verification.verified.len()); + let mut result = Vec::with_capacity(count); + for _ in 0..count { + let block = verification.verified.pop_front().unwrap(); + self.processing.remove(&block.header.hash()); + result.push(block); + } + self.ready_signal.reset(); + result + } +} + +impl Drop for BlockQueue { + fn drop(&mut self) { + self.clear(); + self.deleting.store(true, AtomicOrdering::Relaxed); + self.more_to_verify.notify_all(); + for t in self.verifiers.drain(..) { + t.join().unwrap(); + } } } diff --git a/src/service.rs b/src/service.rs index 3bc137c9c..036c99bc4 100644 --- a/src/service.rs +++ b/src/service.rs @@ -55,8 +55,8 @@ impl IoHandler for ClientIoHandler { match net_message { &mut UserMessage(ref mut message) => { match message { - &mut SyncMessage::BlockVerified(ref mut bytes) => { - self.client.write().unwrap().import_verified_block(mem::replace(bytes, Bytes::new())); + &mut SyncMessage::BlockVerified => { + self.client.write().unwrap().import_verified_blocks(); }, _ => {}, // ignore other messages } diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 43f5968f4..15fe6d1f9 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -401,7 +401,7 @@ impl ChainSync { let header_view = HeaderView::new(header_rlp.as_raw()); // TODO: Decompose block and add to self.headers and self.bodies instead if header_view.number() == From::from(self.last_imported_block + 1) { - match io.chain().import_block(block_rlp.as_raw()) { + match io.chain().import_block(block_rlp.as_raw().to_vec()) { Err(ImportError::AlreadyInChain) => { trace!(target: "sync", "New block already in chain {:?}", h); }, @@ -655,7 +655,7 @@ impl ChainSync { block_rlp.append_raw(body.at(0).as_raw(), 1); block_rlp.append_raw(body.at(1).as_raw(), 1); let h = &headers.1[i].hash; - match io.chain().import_block(&block_rlp.out()) { + match io.chain().import_block(block_rlp.out()) { Err(ImportError::AlreadyInChain) => { trace!(target: "sync", "Block already in chain {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 491fa8e40..da91a6889 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -43,7 +43,7 @@ pub enum SyncMessage { /// New block has been imported into the blockchain NewChainBlock(Bytes), /// A block is ready - BlockVerified(Bytes), + BlockVerified, } pub type NetSyncMessage = NetworkIoMessage; diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 84a8bf21f..05d7ac317 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -43,7 +43,7 @@ impl TestBlockChainClient { rlp.append(&header); rlp.append_raw(&rlp::NULL_RLP, 1); rlp.append_raw(uncles.as_raw(), 1); - self.import_block(rlp.as_raw()).unwrap(); + self.import_block(rlp.as_raw().to_vec()).unwrap(); } } } @@ -110,7 +110,7 @@ impl BlockChainClient for TestBlockChainClient { None } - fn import_block(&mut self, b: &[u8]) -> ImportResult { + fn import_block(&mut self, b: Bytes) -> ImportResult { let header = Rlp::new(&b).val_at::(0); let number: usize = header.number as usize; if number > self.blocks.len() { @@ -132,7 +132,7 @@ impl BlockChainClient for TestBlockChainClient { if number == self.numbers.len() { self.difficulty = self.difficulty + header.difficulty; self.last_hash = header.hash(); - self.blocks.insert(header.hash(), b.to_vec()); + self.blocks.insert(header.hash(), b); self.numbers.insert(number, header.hash()); let mut parent_hash = header.parent_hash; if number > 0 { diff --git a/src/verification.rs b/src/verification.rs index 6df49ac31..3d852dc3e 100644 --- a/src/verification.rs +++ b/src/verification.rs @@ -9,6 +9,16 @@ use common::*; use engine::Engine; use blockchain::*; +/// Preprocessed block data gathered in `verify_block_unordered` call +pub struct PreVerifiedBlock { + /// Populated block header + pub header: Header, + /// Populated block transactions + pub transactions: Vec, + /// Block bytes + pub bytes: Bytes, +} + /// Phase 1 quick block verification. Only does checks that are cheap. Operates on a single block pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error> { try!(verify_header(&header, engine)); @@ -29,19 +39,26 @@ pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Res /// Phase 2 verification. Perform costly checks such as transaction signatures and block nonce for ethash. /// Still operates on a individual block -/// TODO: return cached transactions, header hash. -pub fn verify_block_unordered(header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error> { - try!(engine.verify_block_unordered(&header, Some(bytes))); - for u in Rlp::new(bytes).at(2).iter().map(|rlp| rlp.as_val::
()) { +/// Returns a PreVerifiedBlock structure populated with transactions +pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> Result { + try!(engine.verify_block_unordered(&header, Some(&bytes))); + for u in Rlp::new(&bytes).at(2).iter().map(|rlp| rlp.as_val::
()) { try!(engine.verify_block_unordered(&u, None)); } - // Verify transactions. - // TODO: pass in pre-recovered transactions - maybe verify_transaction wants to call `sender()`. - let v = BlockView::new(bytes); - for t in v.transactions() { - try!(engine.verify_transaction(&t, &header)); + // Verify transactions. + let mut transactions = Vec::new(); + { + let v = BlockView::new(&bytes); + for t in v.transactions() { + try!(engine.verify_transaction(&t, &header)); + transactions.push(t); + } } - Ok(()) + Ok(PreVerifiedBlock { + header: header, + transactions: transactions, + bytes: bytes, + }) } /// Phase 3 verification. Check block information against parent and uncles. diff --git a/util/src/io/service.rs b/util/src/io/service.rs index 4ccfb2407..dee7603d7 100644 --- a/util/src/io/service.rs +++ b/util/src/io/service.rs @@ -155,7 +155,7 @@ pub struct IoChannel where Message: Send { } impl IoChannel where Message: Send { - pub fn send(&mut self, message: Message) -> Result<(), IoError> { + pub fn send(&self, message: Message) -> Result<(), IoError> { try!(self.channel.send(IoMessage::UserMessage(message))); Ok(()) } From d05e7e031ba436a7f15c1d1684a3a9e3e8b61107 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 18 Jan 2016 00:24:20 +0100 Subject: [PATCH 12/17] Basic queue test --- src/queue.rs | 14 ++++++++++++++ util/src/io/service.rs | 14 +++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/queue.rs b/src/queue.rs index a51d3014c..9af6294a7 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -229,3 +229,17 @@ impl Drop for BlockQueue { } } +#[cfg(test)] +mod tests { + use util::*; + use spec::*; + use queue::*; + + #[test] + fn test_block_queue() { + // TODO better test + let spec = Spec::new_test(); + let engine = spec.to_engine().unwrap(); + let _ = BlockQueue::new(Arc::new(engine), IoChannel::disconnected()); + } +} diff --git a/util/src/io/service.rs b/util/src/io/service.rs index dee7603d7..4a96d19a7 100644 --- a/util/src/io/service.rs +++ b/util/src/io/service.rs @@ -151,14 +151,22 @@ impl Handler for IoManager where Message: Send + 'static { /// Allows sending messages into the event loop. All the IO handlers will get the message /// in the `message` callback. pub struct IoChannel where Message: Send { - channel: Sender> + channel: Option>> } impl IoChannel where Message: Send { + /// Send a msessage through the channel pub fn send(&self, message: Message) -> Result<(), IoError> { - try!(self.channel.send(IoMessage::UserMessage(message))); + if let Some(ref channel) = self.channel { + try!(channel.send(IoMessage::UserMessage(message))); + } Ok(()) } + + /// Create a new channel to connected to event loop. + pub fn disconnected() -> IoChannel { + IoChannel { channel: None } + } } /// General IO Service. Starts an event loop and dispatches IO requests. @@ -198,7 +206,7 @@ impl IoService where Message: Send + 'static { /// Create a new message channel pub fn channel(&mut self) -> IoChannel { - IoChannel { channel: self.host_channel.clone() } + IoChannel { channel: Some(self.host_channel.clone()) } } } From fab99e853854a0ff9fb82f7b9046dc4d37ef17dc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 18 Jan 2016 00:51:55 +0100 Subject: [PATCH 13/17] Disable trieDB ref counting for now. --- .gitignore | 1 + src/views.rs | 51 +++++++++++++++++++++++++++++++++++++---- util/src/hash.rs | 10 +++++--- util/src/overlaydb.rs | 4 +++- util/src/trie/triedb.rs | 14 ++++------- 5 files changed, 63 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 801bb1c76..4a2c83c0d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ Cargo.lock # Generated by Cargo /target/ +/util/target # vim stuff *.swp diff --git a/src/views.rs b/src/views.rs index 6c616774d..a9c723ef2 100644 --- a/src/views.rs +++ b/src/views.rs @@ -61,6 +61,44 @@ impl<'a> Hashable for TransactionView<'a> { } } +/// View onto transaction rlp. +pub struct AccountView<'a> { + rlp: Rlp<'a> +} + +impl<'a> AccountView<'a> { + /// Creates new view onto block from raw bytes. + pub fn new(bytes: &'a [u8]) -> AccountView<'a> { + AccountView { + rlp: Rlp::new(bytes) + } + } + + /// Creates new view onto block from rlp. + pub fn new_from_rlp(rlp: Rlp<'a>) -> AccountView<'a> { + AccountView { + rlp: rlp + } + } + + /// Return reference to underlaying rlp. + pub fn rlp(&self) -> &Rlp<'a> { + &self.rlp + } + + /// Get the nonce field of the transaction. + pub fn nonce(&self) -> U256 { self.rlp.val_at(0) } + + /// Get the gas_price field of the transaction. + pub fn balance(&self) -> U256 { self.rlp.val_at(1) } + + /// Get the gas field of the transaction. + pub fn storage_root(&self) -> H256 { self.rlp.val_at(2) } + + /// Get the value field of the transaction. + pub fn code_hash(&self) -> H256 { self.rlp.val_at(3) } +} + /// View onto block rlp. pub struct BlockView<'a> { rlp: Rlp<'a> @@ -97,13 +135,13 @@ impl<'a> BlockView<'a> { } /// Return List of transactions in given block. - pub fn transaction_views(&self) -> Vec { - self.rlp.at(1).iter().map(|rlp| TransactionView::new_from_rlp(rlp)).collect() + pub fn transactions(&self) -> Vec { + self.rlp.val_at(1) } /// Return List of transactions in given block. - pub fn transactions(&self) -> Vec { - self.rlp.val_at(1) + pub fn transaction_views(&self) -> Vec { + self.rlp.at(1).iter().map(|rlp| TransactionView::new_from_rlp(rlp)).collect() } /// Return transaction hashes. @@ -116,6 +154,11 @@ impl<'a> BlockView<'a> { self.rlp.val_at(2) } + /// Return List of transactions in given block. + pub fn uncle_views(&self) -> Vec { + self.rlp.at(2).iter().map(|rlp| HeaderView::new_from_rlp(rlp)).collect() + } + /// Return list of uncle hashes of given block. pub fn uncle_hashes(&self) -> Vec { self.rlp.at(2).iter().map(|rlp| rlp.as_raw().sha3()).collect() diff --git a/util/src/hash.rs b/util/src/hash.rs index 3c90b841d..17057ef07 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -215,10 +215,14 @@ macro_rules! impl_hash { } impl fmt::Display for $from { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - for i in self.0[0..3].iter() { + for i in self.0[0..2].iter() { try!(write!(f, "{:02x}", i)); } - write!(f, "…{:02x}", self.0.last().unwrap()) + try!(write!(f, "…")); + for i in self.0[$size - 4..$size].iter() { + try!(write!(f, "{:02x}", i)); + } + Ok(()) } } @@ -544,7 +548,7 @@ mod tests { fn hash() { let h = H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); - assert_eq!(format!("{}", h), "012345…ef"); + assert_eq!(format!("{}", h), "0123…89abcdef"); assert_eq!(format!("{:?}", h), "0123456789abcdef"); assert_eq!(h.hex(), "0123456789abcdef"); assert!(h == h); diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index 929a492ef..1006cd28c 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -70,7 +70,9 @@ impl OverlayDB { let mut ret = 0u32; for i in self.overlay.drain().into_iter() { let (key, (value, rc)) = i; - if rc != 0 { + // until we figure out state trie pruning, only commit stuff when it has a strictly positive delkta of RCs - + // this prevents RCs being reduced to 0 where the DB would pretent that the node had been removed. + if rc > 0 { match self.payload(&key) { Some(x) => { let (back_value, back_rc) = x; diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index ebb97f74c..9e4cf36e2 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -42,7 +42,7 @@ impl<'db> TrieDB<'db> { /// Panics, if `root` does not exist pub fn new(db: &'db HashDB, root: &'db H256) -> Self { if !db.exists(root) { - flush(format!("Trie root not found {}", root)); + flushln!("TrieDB::new({}): Trie root not found!", root); panic!("Trie root not found!"); } TrieDB { @@ -258,10 +258,7 @@ impl<'a> TrieDBIterator<'a> { node: self.db.get_node(d) }); match self.trail.last().unwrap().node { - Node::Leaf(n, _) | Node::Extension(n, _) => { - println!("Entering. Extend key {:?}, {:?}", self.key_nibbles, n.iter().collect::>()); - self.key_nibbles.extend(n.iter()); - }, + Node::Leaf(n, _) | Node::Extension(n, _) => { self.key_nibbles.extend(n.iter()); }, _ => {} } } @@ -288,11 +285,10 @@ impl<'a> Iterator for TrieDBIterator<'a> { (Status::Exiting, n) => { match n { Node::Leaf(n, _) | Node::Extension(n, _) => { - println!("Exiting. Truncate key {:?}, {:?}", self.key_nibbles, n.iter().collect::>()); let l = self.key_nibbles.len(); self.key_nibbles.truncate(l - n.len()); }, - Node::Branch(_, _) => { println!("Exit branch. Pop {:?}.", self.key_nibbles); self.key_nibbles.pop(); }, + Node::Branch(_, _) => { self.key_nibbles.pop(); }, _ => {} } self.trail.pop(); @@ -304,13 +300,13 @@ impl<'a> Iterator for TrieDBIterator<'a> { (Status::At, Node::Branch(_, _)) => self.next(), (Status::AtChild(i), Node::Branch(children, _)) if children[i].len() > 0 => { match i { - 0 => { println!("Enter first child of branch. Push {:?}.", self.key_nibbles); self.key_nibbles.push(0); }, + 0 => self.key_nibbles.push(0), i => *self.key_nibbles.last_mut().unwrap() = i as u8, } self.descend_next(children[i]) }, (Status::AtChild(i), Node::Branch(_, _)) => { - if i == 0 { println!("Enter first child of branch. Push {:?}.", self.key_nibbles); self.key_nibbles.push(0); } + if i == 0 { self.key_nibbles.push(0); } self.next() }, _ => panic!() // Should never see Entering or AtChild without a Branch here. From 4ccd2ad8fe60ec91dc184a3d0faa36381787ee48 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 18 Jan 2016 00:54:06 +0100 Subject: [PATCH 14/17] Add local node. --- util/src/network/host.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 90cbfdaad..37b58f1f0 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -561,13 +561,13 @@ impl IoHandler> for Host where Messa let port = self.info.config.listen_address.port(); self.info.listen_port = port; -// self.add_node("enode://a9a921de2ff09a9a4d38b623c67b2d6b477a8e654ae95d874750cbbcb31b33296496a7b4421934e2629269e180823e52c15c2b19fc59592ec51ffe4f2de76ed7@127.0.0.1:30303"); - // GO bootnodes + self.add_node("enode://a9a921de2ff09a9a4d38b623c67b2d6b477a8e654ae95d874750cbbcb31b33296496a7b4421934e2629269e180823e52c15c2b19fc59592ec51ffe4f2de76ed7@127.0.0.1:30303"); +/* // GO bootnodes self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); // IE self.add_node("enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303"); // BR self.add_node("enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303"); // SG // ETH/DEV cpp-ethereum (poc-9.ethdev.com) - self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); + self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303");*/ } fn stream_hup<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { From d7b692562072a2c8704ab74390d7ebc6208e06dd Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 18 Jan 2016 00:58:06 +0100 Subject: [PATCH 15/17] Reduce spam. --- src/executive.rs | 2 +- src/state.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/executive.rs b/src/executive.rs index 2a26dd3f0..6c2b29e3f 100644 --- a/src/executive.rs +++ b/src/executive.rs @@ -171,7 +171,7 @@ impl<'a> Executive<'a> { // at first, transfer value to destination self.state.transfer_balance(¶ms.sender, ¶ms.address, ¶ms.value); - debug!("Executive::call(params={:?}) self.env_info={:?}", params, self.info); + trace!("Executive::call(params={:?}) self.env_info={:?}", params, self.info); if self.engine.is_builtin(¶ms.code_address) { // if destination is builtin, try to execute it diff --git a/src/state.rs b/src/state.rs index fc4d6f817..a186d6cd6 100644 --- a/src/state.rs +++ b/src/state.rs @@ -150,10 +150,10 @@ impl State { let e = try!(Executive::new(self, env_info, engine).transact(t)); //println!("Executed: {:?}", e); - debug!("Applied transaction. Diff:\n{}\n", StateDiff::diff_pod(&old, &self.to_pod())); + trace!("Applied transaction. Diff:\n{}\n", StateDiff::diff_pod(&old, &self.to_pod())); self.commit(); let receipt = Receipt::new(self.root().clone(), e.cumulative_gas_used, e.logs); - debug!("Transaction receipt: {:?}", receipt); + trace!("Transaction receipt: {:?}", receipt); Ok(receipt) } From e862fc62577bec590aae47b708e8016ecc91fb85 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 18 Jan 2016 01:39:19 +0100 Subject: [PATCH 16/17] Typo --- src/queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queue.rs b/src/queue.rs index 9af6294a7..5803b3e5f 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -38,7 +38,7 @@ struct QueueSignal { impl QueueSignal { fn set(&self) { if self.signalled.compare_and_swap(false, true, AtomicOrdering::Relaxed) == false { - self.message_channel.send(UserMessage(SyncMessage::BlockVerified)).expect("Error seding BlockVerified message"); + self.message_channel.send(UserMessage(SyncMessage::BlockVerified)).expect("Error sending BlockVerified message"); } } fn reset(&self) { From 77d2303b55f601f603f6c68894fcbe82c627669a Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 18 Jan 2016 12:49:39 +0100 Subject: [PATCH 17/17] Use sha3 crate in ethash --- ethash/Cargo.toml | 2 +- ethash/src/compute.rs | 54 ++++++++++---------------------- ethash/src/lib.rs | 2 +- util/Cargo.toml | 5 +-- util/sha3/Cargo.toml | 11 +++++++ util/{ => sha3}/build.rs | 0 util/sha3/src/lib.rs | 4 +++ util/{ => sha3}/src/tinykeccak.c | 0 util/src/sha3.rs | 5 ++- 9 files changed, 37 insertions(+), 46 deletions(-) create mode 100644 util/sha3/Cargo.toml rename util/{ => sha3}/build.rs (100%) create mode 100644 util/sha3/src/lib.rs rename util/{ => sha3}/src/tinykeccak.c (100%) diff --git a/ethash/Cargo.toml b/ethash/Cargo.toml index 2633a85eb..16af6525c 100644 --- a/ethash/Cargo.toml +++ b/ethash/Cargo.toml @@ -6,4 +6,4 @@ authors = ["arkpar u32 { #[inline] fn sha3_512(input: &[u8], output: &mut [u8]) { - let mut sha3 = Keccak::new_keccak512(); - sha3.update(input); - sha3.finalize(output); + unsafe { sha3::sha3_512(output.as_mut_ptr(), output.len(), input.as_ptr(), input.len()) }; } #[inline] @@ -107,9 +105,7 @@ fn get_seedhash(block_number: u64) -> H256 { let epochs = block_number / ETHASH_EPOCH_LENGTH; let mut ret: H256 = [0u8; 32]; for _ in 0..epochs { - let mut sha3 = Keccak::new_keccak256(); - sha3.update(&ret); - sha3.finalize(&mut ret); + unsafe { sha3::sha3_256(ret[..].as_mut_ptr(), 32, ret[..].as_ptr(), 32) }; } ret } @@ -125,15 +121,12 @@ pub fn quick_get_difficulty(header_hash: &H256, nonce: u64, mix_hash: &H256) -> unsafe { ptr::copy_nonoverlapping(header_hash.as_ptr(), buf.as_mut_ptr(), 32) }; unsafe { ptr::copy_nonoverlapping(mem::transmute(&nonce), buf[32..].as_mut_ptr(), 8) }; - let mut sha3 = Keccak::new_keccak512(); - sha3.update(&buf[0..40]); - sha3.finalize(&mut buf); + unsafe { sha3::sha3_512(buf.as_mut_ptr(), 64, buf.as_ptr(), 40) }; unsafe { ptr::copy_nonoverlapping(mix_hash.as_ptr(), buf[64..].as_mut_ptr(), 32) }; let mut hash = [0u8; 32]; - let mut sha3 = Keccak::new_keccak256(); - sha3.update(&buf); - sha3.finalize(&mut hash); + unsafe { sha3::sha3_256(hash.as_mut_ptr(), hash.len(), buf.as_ptr(), buf.len()) }; + hash.as_mut_ptr(); hash } @@ -157,10 +150,7 @@ fn hash_compute(light: &Light, full_size: usize, header_hash: &H256, nonce: u64 // compute sha3-512 hash and replicate across mix unsafe { - let mut sha3 = Keccak::new_keccak512(); - sha3.update(&s_mix.get_unchecked(0).bytes[0..40]); - sha3.finalize(&mut s_mix.get_unchecked_mut(0).bytes); - + sha3::sha3_512(s_mix.get_unchecked_mut(0).bytes.as_mut_ptr(), NODE_BYTES, s_mix.get_unchecked(0).bytes.as_ptr(), 40); let (f_mix, mut mix) = s_mix.split_at_mut(1); for w in 0..MIX_WORDS { *mix.get_unchecked_mut(0).as_words_mut().get_unchecked_mut(w) = *f_mix.get_unchecked(0).as_words().get_unchecked(w % NODE_WORDS); @@ -189,15 +179,13 @@ fn hash_compute(light: &Light, full_size: usize, header_hash: &H256, nonce: u64 *mix.get_unchecked_mut(0).as_words_mut().get_unchecked_mut(i) = reduction; } - let mut mix_hash: H256 = [0u8; 32]; + let mut mix_hash = [0u8; 32]; + let mut buf = [0u8; 32 + 64]; + ptr::copy_nonoverlapping(f_mix.get_unchecked_mut(0).bytes.as_ptr(), buf.as_mut_ptr(), 64); + ptr::copy_nonoverlapping(mix.get_unchecked_mut(0).bytes.as_ptr(), buf[64..].as_mut_ptr(), 32); ptr::copy_nonoverlapping(mix.get_unchecked_mut(0).bytes.as_ptr(), mix_hash.as_mut_ptr(), 32); let mut value: H256 = [0u8; 32]; - - let mut sha3 = Keccak::new_keccak256(); - sha3.update(&f_mix.get_unchecked(0).bytes); - sha3.update(&mix_hash); - sha3.finalize(&mut value); - + sha3::sha3_256(value.as_mut_ptr(), value.len(), buf.as_ptr(), buf.len()); ProofOfWork { mix_hash: mix_hash, value: value, @@ -212,10 +200,7 @@ fn calculate_dag_item(node_index: u32, light: &Light) -> Node { let init = cache_nodes.get_unchecked(node_index as usize % num_parent_nodes); let mut ret = init.clone(); *ret.as_words_mut().get_unchecked_mut(0) ^= node_index; - - let mut sha3 = Keccak::new_keccak512(); - sha3.update(&ret.bytes); - sha3.finalize(&mut ret.bytes); + sha3::sha3_512(ret.bytes.as_mut_ptr(), ret.bytes.len(), ret.bytes.as_ptr(), ret.bytes.len()); for i in 0..ETHASH_DATASET_PARENTS { let parent_index = fnv_hash(node_index ^ i, *ret.as_words().get_unchecked(i as usize % NODE_WORDS)) % num_parent_nodes as u32; @@ -224,10 +209,7 @@ fn calculate_dag_item(node_index: u32, light: &Light) -> Node { *ret.as_words_mut().get_unchecked_mut(w) = fnv_hash(*ret.as_words().get_unchecked(w), *parent.as_words().get_unchecked(w)); } } - - let mut sha3 = Keccak::new_keccak512(); - sha3.update(&ret.bytes); - sha3.finalize(&mut ret.bytes); + sha3::sha3_512(ret.bytes.as_mut_ptr(), ret.bytes.len(), ret.bytes.as_ptr(), ret.bytes.len()); ret } } @@ -246,9 +228,7 @@ fn light_new(block_number: u64) -> Light { unsafe { sha3_512(&seedhash[0..32], &mut nodes.get_unchecked_mut(0).bytes); for i in 1..num_nodes { - let mut sha3 = Keccak::new_keccak512(); - sha3.update(&nodes.get_unchecked_mut(i - 1).bytes); - sha3.finalize(&mut nodes.get_unchecked_mut(i).bytes); + sha3::sha3_512(nodes.get_unchecked_mut(i).bytes.as_mut_ptr(), NODE_BYTES, nodes.get_unchecked(i - 1).bytes.as_ptr(), NODE_BYTES); } for _ in 0..ETHASH_CACHE_ROUNDS { @@ -275,9 +255,9 @@ fn test_difficulty_test() { let mix_hash = [0x1f, 0xff, 0x04, 0xce, 0xc9, 0x41, 0x73, 0xfd, 0x59, 0x1e, 0x3d, 0x89, 0x60, 0xce, 0x6b, 0xdf, 0x8b, 0x19, 0x71, 0x04, 0x8c, 0x71, 0xff, 0x93, 0x7b, 0xb2, 0xd3, 0x2a, 0x64, 0x31, 0xab, 0x6d ]; let nonce = 0xd7b3ac70a301a249; let boundary_good = [0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3e, 0x9b, 0x6c, 0x69, 0xbc, 0x2c, 0xe2, 0xa2, 0x4a, 0x8e, 0x95, 0x69, 0xef, 0xc7, 0xd7, 0x1b, 0x33, 0x35, 0xdf, 0x36, 0x8c, 0x9a, 0xe9, 0x7e, 0x53, 0x84]; - assert!(quick_check_difficulty(&hash, nonce, &mix_hash, &boundary_good)); + assert_eq!(quick_get_difficulty(&hash, nonce, &mix_hash)[..], boundary_good[..]); let boundary_bad = [0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3a, 0x9b, 0x6c, 0x69, 0xbc, 0x2c, 0xe2, 0xa2, 0x4a, 0x8e, 0x95, 0x69, 0xef, 0xc7, 0xd7, 0x1b, 0x33, 0x35, 0xdf, 0x36, 0x8c, 0x9a, 0xe9, 0x7e, 0x53, 0x84]; - assert!(!quick_check_difficulty(&hash, nonce, &mix_hash, &boundary_bad)); + assert!(quick_get_difficulty(&hash, nonce, &mix_hash)[..] != boundary_bad[..]); } #[test] diff --git a/ethash/src/lib.rs b/ethash/src/lib.rs index 166574bef..f7b6d2308 100644 --- a/ethash/src/lib.rs +++ b/ethash/src/lib.rs @@ -1,6 +1,6 @@ //! Ethash implementation //! See https://github.com/ethereum/wiki/wiki/Ethash -extern crate tiny_keccak; +extern crate sha3; mod sizes; mod compute; diff --git a/util/Cargo.toml b/util/Cargo.toml index 6b60fddf1..02fdad17f 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -5,10 +5,6 @@ license = "GPL-3.0" name = "ethcore-util" version = "0.1.0" authors = ["Ethcore "] -build = "build.rs" - -[build-dependencies] -gcc = "0.3" [dependencies] log = "0.3" @@ -27,6 +23,7 @@ elastic-array = "0.4" heapsize = "0.2" itertools = "0.4" slab = { git = "https://github.com/arkpar/slab.git" } +sha3 = { path = "sha3" } [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/util/sha3/Cargo.toml b/util/sha3/Cargo.toml new file mode 100644 index 000000000..ac423b22f --- /dev/null +++ b/util/sha3/Cargo.toml @@ -0,0 +1,11 @@ +[package] +description = "Rust bindings for tinykeccak C library" +homepage = "http://ethcore.io" +license = "GPL-3.0" +name = "sha3" +version = "0.1.0" +authors = ["Ethcore "] +build = "build.rs" + +[build-dependencies] +gcc = "0.3" diff --git a/util/build.rs b/util/sha3/build.rs similarity index 100% rename from util/build.rs rename to util/sha3/build.rs diff --git a/util/sha3/src/lib.rs b/util/sha3/src/lib.rs new file mode 100644 index 000000000..de2bf6e3e --- /dev/null +++ b/util/sha3/src/lib.rs @@ -0,0 +1,4 @@ +extern { + pub fn sha3_256(out: *mut u8, outlen: usize, input: *const u8, inputlen: usize) -> i32; + pub fn sha3_512(out: *mut u8, outlen: usize, input: *const u8, inputlen: usize) -> i32; +} diff --git a/util/src/tinykeccak.c b/util/sha3/src/tinykeccak.c similarity index 100% rename from util/src/tinykeccak.c rename to util/sha3/src/tinykeccak.c diff --git a/util/src/sha3.rs b/util/src/sha3.rs index c251edcfb..a33ac61f7 100644 --- a/util/src/sha3.rs +++ b/util/src/sha3.rs @@ -1,14 +1,13 @@ //! Wrapper around tiny-keccak crate. +extern crate sha3 as sha3_ext; use std::mem::uninitialized; use bytes::{BytesConvertable, Populatable}; use hash::{H256, FixedHash}; +use self::sha3_ext::*; pub const SHA3_EMPTY: H256 = H256( [0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70] ); -extern { - fn sha3_256(out: *mut u8, outlen: usize, input: *const u8, inputlen: usize) -> i32; -} /// Types implementing this trait are sha3able. ///