grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of github.com:ethcore/parity into move_hash

Browse Source
This commit is contained in:
debris
2016-08-09 09:58:18 +02:00
parent 8269887949 ab8b763ea5
commit 974d537849
43 changed files with 922 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
ethcore/src/blockchain/blockchain.rs
View File

@@ -775,12 +775,20 @@ impl BlockChain {
/// Applt pending insertion updates
pub fn commit(&self) {
let mut best_block = self.best_block.write();
let mut write_hashes = self.block_hashes.write();
let mut write_txs = self.transaction_addresses.write();
let mut pending_best_block = self.pending_best_block.write();
let mut pending_write_hashes = self.pending_block_hashes.write();
let mut pending_write_txs = self.pending_transaction_addresses.write();
for n in pending_write_hashes.keys() {
self.note_used(CacheID::BlockHashes(*n));
}
for hash in pending_write_txs.keys() {
self.note_used(CacheID::TransactionAddresses(hash.clone()));
}
let mut best_block = self.best_block.write();
let mut write_hashes = self.block_hashes.write();
let mut write_txs = self.transaction_addresses.write();
// update best block
if let Some(block) = pending_best_block.take() {
*best_block = block;

40
ethcore/src/client/client.rs
View File

@@ -594,19 +594,35 @@ impl Client {
}
}
/// Take a snapshot.
pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(&self, writer: W) -> Result<(), ::error::Error> {
/// Take a snapshot at the given block.
/// If the ID given is "latest", this will default to 1000 blocks behind.
pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), ::error::Error> {
let db = self.state_db.lock().boxed_clone();
let best_block_number = self.chain_info().best_block_number;
let start_block_number = if best_block_number > 1000 {
best_block_number - 1000
} else {
0
};
let start_hash = self.block_hash(BlockID::Number(start_block_number))
.expect("blocks within HISTORY are always stored.");
let block_number = try!(self.block_number(at).ok_or(snapshot::Error::InvalidStartingBlock(at)));
try!(snapshot::take_snapshot(&self.chain, start_hash, db.as_hashdb(), writer));
if best_block_number > HISTORY + block_number && db.is_pruned() {
return Err(snapshot::Error::OldBlockPrunedDB.into());
}
let start_hash = match at {
BlockID::Latest => {
let start_num = if best_block_number > 1000 {
best_block_number - 1000
} else {
0
};
self.block_hash(BlockID::Number(start_num))
.expect("blocks within HISTORY are always stored.")
}
_ => match self.block_hash(at) {
Some(hash) => hash,
None => return Err(snapshot::Error::InvalidStartingBlock(at).into()),
},
};
try!(snapshot::take_snapshot(&self.chain, start_hash, db.as_hashdb(), writer, p));
Ok(())
}
@@ -784,8 +800,8 @@ impl BlockChainClient for Client {
Self::block_hash(&self.chain, id)
}
fn code(&self, address: &Address) -> Option<Bytes> {
self.state().code(address)
fn code(&self, address: &Address, id: BlockID) -> Option<Option<Bytes>> {
self.state_at(id).map(|s| s.code(address))
}
fn balance(&self, address: &Address, id: BlockID) -> Option<U256> {

7
ethcore/src/client/test_client.rs
View File

@@ -319,8 +319,11 @@ impl BlockChainClient for TestBlockChainClient {
self.nonce(address, BlockID::Latest).unwrap()
}
fn code(&self, address: &Address) -> Option<Bytes> {
self.code.read().get(address).cloned()
fn code(&self, address: &Address, id: BlockID) -> Option<Option<Bytes>> {
match id {
BlockID::Latest => Some(self.code.read().get(address).cloned()),
_ => None,
}
}
fn balance(&self, address: &Address, id: BlockID) -> Option<U256> {

10
ethcore/src/client/traits.rs
View File

@@ -78,8 +78,14 @@ pub trait BlockChainClient : Sync + Send {
/// Get block hash.
fn block_hash(&self, id: BlockID) -> Option<H256>;
/// Get address code.
fn code(&self, address: &Address) -> Option<Bytes>;
/// Get address code at given block's state.
fn code(&self, address: &Address, id: BlockID) -> Option<Option<Bytes>>;
/// Get address code at the latest block's state.
fn latest_code(&self, address: &Address) -> Option<Bytes> {
self.code(address, BlockID::Latest)
.expect("code will return Some if given BlockID::Latest; qed")
}
/// Get address balance at the given block's state.
///

22
ethcore/src/evm/evm.rs
View File

@@ -151,10 +151,14 @@ impl CostType for usize {
}
fn from_u256(val: U256) -> Result<Self> {
if U256::from(val.low_u64()) != val {
let res = val.low_u64() as usize;
// validate if value fits into usize
if U256::from(res) != val {
return Err(Error::OutOfGas);
}
Ok(val.low_u64() as usize)
Ok(res)
}
fn as_usize(&self) -> usize {
@@ -191,6 +195,7 @@ pub trait Evm {
#[test]
#[cfg(test)]
fn should_calculate_overflow_mul_shr_without_overflow() {
// given
let num = 1048576;
@@ -207,6 +212,7 @@ fn should_calculate_overflow_mul_shr_without_overflow() {
}
#[test]
#[cfg(test)]
fn should_calculate_overflow_mul_shr_with_overflow() {
// given
let max = ::std::u64::MAX;
@@ -225,3 +231,15 @@ fn should_calculate_overflow_mul_shr_with_overflow() {
assert!(o1);
}
#[test]
#[cfg(test)]
fn should_validate_u256_to_usize_conversion() {
// given
let v = U256::from(::std::usize::MAX) + U256::from(1);
// when
let res = usize::from_u256(v);
// then
assert!(res.is_err());
}

8
ethcore/src/evm/interpreter/mod.rs
View File

@@ -515,11 +515,11 @@ impl<Cost: CostType> Interpreter<Cost> {
Ok(InstructionResult::Ok)
}
fn copy_data_to_memory(&mut self, stack: &mut Stack<U256>, data: &[u8]) {
fn copy_data_to_memory(&mut self, stack: &mut Stack<U256>, source: &[u8]) {
let dest_offset = stack.pop_back();
let source_offset = stack.pop_back();
let size = stack.pop_back();
let source_size = U256::from(data.len());
let source_size = U256::from(source.len());
let output_end = match source_offset > source_size || size > source_size || source_offset + size > source_size {
true => {
@@ -531,14 +531,14 @@ impl<Cost: CostType> Interpreter<Cost> {
for i in zero_slice.iter_mut() {
*i = 0;
}
data.len()
source.len()
},
false => (size.low_u64() + source_offset.low_u64()) as usize
};
if source_offset < source_size {
let output_begin = source_offset.low_u64() as usize;
self.mem.write_slice(dest_offset, &data[output_begin..output_end]);
self.mem.write_slice(dest_offset, &source[output_begin..output_end]);
}
}

2
ethcore/src/miner/miner.rs
View File

@@ -528,7 +528,7 @@ impl MinerService for Miner {
fn code(&self, chain: &MiningBlockChainClient, address: &Address) -> Option<Bytes> {
let sealing_work = self.sealing_work.lock();
sealing_work.queue.peek_last_ref().map_or_else(|| chain.code(address), |b| b.block().fields().state.code(address))
sealing_work.queue.peek_last_ref().map_or_else(|| chain.latest_code(address), |b| b.block().fields().state.code(address))
}
fn set_author(&self, author: Address) {

13
ethcore/src/snapshot/error.rs
View File

@@ -18,6 +18,8 @@
use std::fmt;
use ids::BlockID;
use util::H256;
use util::trie::TrieError;
use util::rlp::DecoderError;
@@ -26,9 +28,13 @@ use util::rlp::DecoderError;
#[derive(Debug)]
pub enum Error {
/// Invalid starting block for snapshot.
InvalidStartingBlock(H256),
InvalidStartingBlock(BlockID),
/// Block not found.
BlockNotFound(H256),
/// Incomplete chain.
IncompleteChain,
/// Old starting block in a pruned database.
OldBlockPrunedDB,
/// Trie error.
Trie(TrieError),
/// Decoder error.
@@ -40,8 +46,11 @@ pub enum Error {
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Error::InvalidStartingBlock(ref hash) => write!(f, "Invalid starting block hash: {}", hash),
Error::InvalidStartingBlock(ref id) => write!(f, "Invalid starting block: {:?}", id),
Error::BlockNotFound(ref hash) => write!(f, "Block not found in chain: {}", hash),
Error::IncompleteChain => write!(f, "Cannot create snapshot due to incomplete chain."),
Error::OldBlockPrunedDB => write!(f, "Attempted to create a snapshot at an old block while using \
a pruned database. Please re-run with the --pruning archive flag."),
Error::Io(ref err) => err.fmt(f),
Error::Decoder(ref err) => err.fmt(f),
Error::Trie(ref err) => err.fmt(f),

80
ethcore/src/snapshot/mod.rs
View File

@@ -18,10 +18,12 @@
use std::collections::VecDeque;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use account_db::{AccountDB, AccountDBMut};
use blockchain::{BlockChain, BlockProvider};
use engines::Engine;
use ids::BlockID;
use views::BlockView;
use util::{Bytes, Hashable, HashDB, snappy, TrieDB, TrieDBMut, TrieMut};
@@ -58,9 +60,49 @@ const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
// How many blocks to include in a snapshot, starting from the head of the chain.
const SNAPSHOT_BLOCKS: u64 = 30000;
/// A progress indicator for snapshots.
#[derive(Debug)]
pub struct Progress {
accounts: AtomicUsize,
blocks: AtomicUsize,
size: AtomicUsize, // Todo [rob] use Atomicu64 when it stabilizes.
done: AtomicBool,
}
impl Progress {
/// Create a new progress indicator.
pub fn new() -> Self {
Progress {
accounts: AtomicUsize::new(0),
blocks: AtomicUsize::new(0),
size: AtomicUsize::new(0),
done: AtomicBool::new(false),
}
}
/// Get the number of accounts snapshotted thus far.
pub fn accounts(&self) -> usize { self.accounts.load(Ordering::Relaxed) }
/// Get the number of blocks snapshotted thus far.
pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Relaxed) }
/// Get the written size of the snapshot in bytes.
pub fn size(&self) -> usize { self.size.load(Ordering::Relaxed) }
/// Whether the snapshot is complete.
pub fn done(&self) -> bool { self.done.load(Ordering::SeqCst) }
}
/// Take a snapshot using the given blockchain, starting block hash, and database, writing into the given writer.
pub fn take_snapshot<W: SnapshotWriter + Send>(chain: &BlockChain, start_block_hash: H256, state_db: &HashDB, writer: W) -> Result<(), Error> {
let start_header = try!(chain.block_header(&start_block_hash).ok_or(Error::InvalidStartingBlock(start_block_hash)));
pub fn take_snapshot<W: SnapshotWriter + Send>(
chain: &BlockChain,
block_at: H256,
state_db: &HashDB,
writer: W,
p: &Progress
) -> Result<(), Error> {
let start_header = try!(chain.block_header(&block_at)
.ok_or(Error::InvalidStartingBlock(BlockID::Hash(block_at))));
let state_root = start_header.state_root();
let number = start_header.number();
@@ -68,8 +110,8 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(chain: &BlockChain, start_block_h
let writer = Mutex::new(writer);
let (state_hashes, block_hashes) = try!(scope(|scope| {
let block_guard = scope.spawn(|| chunk_blocks(chain, (number, start_block_hash), &writer));
let state_res = chunk_state(state_db, state_root, &writer);
let block_guard = scope.spawn(|| chunk_blocks(chain, (number, block_at), &writer, p));
let state_res = chunk_state(state_db, state_root, &writer, p);
state_res.and_then(|state_hashes| {
block_guard.join().map(|block_hashes| (state_hashes, block_hashes))
@@ -83,11 +125,13 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(chain: &BlockChain, start_block_h
block_hashes: block_hashes,
state_root: *state_root,
block_number: number,
block_hash: start_block_hash,
block_hash: block_at,
};
try!(writer.into_inner().finish(manifest_data));
p.done.store(true, Ordering::SeqCst);
Ok(())
}
@@ -100,6 +144,7 @@ struct BlockChunker<'a> {
hashes: Vec<H256>,
snappy_buffer: Vec<u8>,
writer: &'a Mutex<SnapshotWriter + 'a>,
progress: &'a Progress,
}
impl<'a> BlockChunker<'a> {
@@ -162,7 +207,8 @@ impl<'a> BlockChunker<'a> {
let parent_total_difficulty = parent_details.total_difficulty;
let mut rlp_stream = RlpStream::new_list(3 + self.rlps.len());
let num_entries = self.rlps.len();
let mut rlp_stream = RlpStream::new_list(3 + num_entries);
rlp_stream.append(&parent_number).append(&parent_hash).append(&parent_total_difficulty);
for pair in self.rlps.drain(..) {
@@ -178,6 +224,9 @@ impl<'a> BlockChunker<'a> {
try!(self.writer.lock().write_block_chunk(hash, compressed));
trace!(target: "snapshot", "wrote block chunk. hash: {}, size: {}, uncompressed size: {}", hash.hex(), size, raw_data.len());
self.progress.size.fetch_add(size, Ordering::SeqCst);
self.progress.blocks.fetch_add(num_entries, Ordering::SeqCst);
self.hashes.push(hash);
Ok(())
}
@@ -189,7 +238,7 @@ impl<'a> BlockChunker<'a> {
/// The path parameter is the directory to store the block chunks in.
/// This function assumes the directory exists already.
/// Returns a list of chunk hashes, with the first having the blocks furthest from the genesis.
pub fn chunk_blocks<'a>(chain: &'a BlockChain, start_block_info: (u64, H256), writer: &Mutex<SnapshotWriter + 'a>) -> Result<Vec<H256>, Error> {
pub fn chunk_blocks<'a>(chain: &'a BlockChain, start_block_info: (u64, H256), writer: &Mutex<SnapshotWriter + 'a>, progress: &'a Progress) -> Result<Vec<H256>, Error> {
let (start_number, start_hash) = start_block_info;
let first_hash = if start_number < SNAPSHOT_BLOCKS {
@@ -197,8 +246,7 @@ pub fn chunk_blocks<'a>(chain: &'a BlockChain, start_block_info: (u64, H256), wr
chain.genesis_hash()
} else {
let first_num = start_number - SNAPSHOT_BLOCKS;
chain.block_hash(first_num)
.expect("number before best block number; whole chain is stored; qed")
try!(chain.block_hash(first_num).ok_or(Error::IncompleteChain))
};
let mut chunker = BlockChunker {
@@ -208,6 +256,7 @@ pub fn chunk_blocks<'a>(chain: &'a BlockChain, start_block_info: (u64, H256), wr
hashes: Vec::new(),
snappy_buffer: vec![0; snappy::max_compressed_len(PREFERRED_CHUNK_SIZE)],
writer: writer,
progress: progress,
};
try!(chunker.chunk_all(first_hash));
@@ -222,6 +271,7 @@ struct StateChunker<'a> {
cur_size: usize,
snappy_buffer: Vec<u8>,
writer: &'a Mutex<SnapshotWriter + 'a>,
progress: &'a Progress,
}
impl<'a> StateChunker<'a> {
@@ -249,7 +299,8 @@ impl<'a> StateChunker<'a> {
// Write out the buffer to disk, pushing the created chunk's hash to
// the list.
fn write_chunk(&mut self) -> Result<(), Error> {
let mut stream = RlpStream::new_list(self.rlps.len());
let num_entries = self.rlps.len();
let mut stream = RlpStream::new_list(num_entries);
for rlp in self.rlps.drain(..) {
stream.append_raw(&rlp, 1);
}
@@ -263,6 +314,9 @@ impl<'a> StateChunker<'a> {
try!(self.writer.lock().write_state_chunk(hash, compressed));
trace!(target: "snapshot", "wrote state chunk. size: {}, uncompressed size: {}", compressed_size, raw_data.len());
self.progress.accounts.fetch_add(num_entries, Ordering::SeqCst);
self.progress.size.fetch_add(compressed_size, Ordering::SeqCst);
self.hashes.push(hash);
self.cur_size = 0;
@@ -275,7 +329,7 @@ impl<'a> StateChunker<'a> {
///
/// Returns a list of hashes of chunks created, or any error it may
/// have encountered.
pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter + 'a>) -> Result<Vec<H256>, Error> {
pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter + 'a>, progress: &'a Progress) -> Result<Vec<H256>, Error> {
let account_trie = try!(TrieDB::new(db, &root));
let mut chunker = StateChunker {
@@ -284,10 +338,9 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
cur_size: 0,
snappy_buffer: vec![0; snappy::max_compressed_len(PREFERRED_CHUNK_SIZE)],
writer: writer,
progress: progress,
};
trace!(target: "snapshot", "beginning state chunking");
// account_key here is the address' hash.
for (account_key, account_data) in account_trie.iter() {
let account = Account::from_thin_rlp(account_data);
@@ -383,6 +436,7 @@ impl StateRebuilder {
let chunk_size = account_fat_rlps.len() / ::num_cpus::get() + 1;
// build account tries in parallel.
// Todo [rob] keep a thread pool around so we don't do this per-chunk.
try!(scope(|scope| {
let mut handles = Vec::new();
for (account_chunk, out_pairs_chunk) in account_fat_rlps.chunks(chunk_size).zip(pairs.chunks_mut(chunk_size)) {

4
ethcore/src/snapshot/tests/blocks.rs
View File

@@ -20,7 +20,7 @@ use devtools::RandomTempPath;
use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer};
use blockchain::BlockChain;
use snapshot::{chunk_blocks, BlockRebuilder};
use snapshot::{chunk_blocks, BlockRebuilder, Progress};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
use util::{Mutex, snappy};
@@ -55,7 +55,7 @@ fn chunk_and_restore(amount: u64) {
// snapshot it.
let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap());
let block_hashes = chunk_blocks(&bc, (amount, best_hash), &writer).unwrap();
let block_hashes = chunk_blocks(&bc, (amount, best_hash), &writer, &Progress::new()).unwrap();
writer.into_inner().finish(::snapshot::ManifestData {
state_hashes: Vec::new(),
block_hashes: block_hashes,

4
ethcore/src/snapshot/tests/state.rs
View File

@@ -16,7 +16,7 @@
//! State snapshotting tests.
use snapshot::{chunk_state, StateRebuilder};
use snapshot::{chunk_state, Progress, StateRebuilder};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
use super::helpers::{compare_dbs, StateProducer};
@@ -48,7 +48,7 @@ fn snap_and_restore() {
let state_root = producer.state_root();
let writer = Mutex::new(PackedWriter::new(&snap_file).unwrap());
let state_hashes = chunk_state(&old_db, &state_root, &writer).unwrap();
let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::new()).unwrap();
writer.into_inner().finish(::snapshot::ManifestData {
state_hashes: state_hashes,

18
ethcore/src/verification/verification.rs
View File

@@ -40,7 +40,8 @@ pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Res
try!(verify_header(&header, engine));
try!(verify_block_integrity(bytes, &header.transactions_root, &header.uncles_hash));
try!(engine.verify_block_basic(&header, Some(bytes)));
for u in Rlp::new(bytes).at(2).iter().map(|rlp| rlp.as_val::<Header>()) {
for u in try!(UntrustedRlp::new(bytes).at(2)).iter().map(|rlp| rlp.as_val::<Header>()) {
let u = try!(u);
try!(verify_header(&u, engine));
try!(engine.verify_block_basic(&u, None));
}
@@ -58,8 +59,8 @@ pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Res
/// Returns a `PreverifiedBlock` structure populated with transactions
pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> Result<PreverifiedBlock, Error> {
try!(engine.verify_block_unordered(&header, Some(&bytes)));
for u in Rlp::new(&bytes).at(2).iter().map(|rlp| rlp.as_val::<Header>()) {
try!(engine.verify_block_unordered(&u, None));
for u in try!(UntrustedRlp::new(&bytes).at(2)).iter().map(|rlp| rlp.as_val::<Header>()) {
try!(engine.verify_block_unordered(&try!(u), None));
}
// Verify transactions.
let mut transactions = Vec::new();
@@ -84,7 +85,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &
try!(verify_parent(&header, &parent));
try!(engine.verify_block_family(&header, &parent, Some(bytes)));
let num_uncles = Rlp::new(bytes).at(2).item_count();
let num_uncles = try!(UntrustedRlp::new(bytes).at(2)).item_count();
if num_uncles != 0 {
if num_uncles > engine.maximum_uncle_count() {
return Err(From::from(BlockError::TooManyUncles(OutOfBounds { min: None, max: Some(engine.maximum_uncle_count()), found: num_uncles })));
@@ -106,7 +107,8 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &
}
}
for uncle in Rlp::new(bytes).at(2).iter().map(|rlp| rlp.as_val::<Header>()) {
for uncle in try!(UntrustedRlp::new(bytes).at(2)).iter().map(|rlp| rlp.as_val::<Header>()) {
let uncle = try!(uncle);
if excluded.contains(&uncle.hash()) {
return Err(From::from(BlockError::UncleInChain(uncle.hash())))
}
@@ -210,13 +212,13 @@ fn verify_parent(header: &Header, parent: &Header) -> Result<(), Error> {
/// Verify block data against header: transactions root and uncles hash.
fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), Error> {
let block = Rlp::new(block);
let tx = block.at(1);
let block = UntrustedRlp::new(block);
let tx = try!(block.at(1));
let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here
if expected_root != transactions_root {
return Err(From::from(BlockError::InvalidTransactionsRoot(Mismatch { expected: expected_root.clone(), found: transactions_root.clone() })))
}
let expected_uncles = &block.at(2).as_raw().sha3();
let expected_uncles = &try!(block.at(2)).as_raw().sha3();
if expected_uncles != uncles_hash {
return Err(From::from(BlockError::InvalidUnclesHash(Mismatch { expected: expected_uncles.clone(), found: uncles_hash.clone() })))
}