diff --git a/README.md b/README.md index 4df7cad34..f8b24f088 100644 --- a/README.md +++ b/README.md @@ -15,77 +15,28 @@ ### Building from source -##### Ubuntu 14.04, 15.04, 15.10 +First (if you don't already have it) get multirust: +- Linux: ```bash -# install rocksdb -add-apt-repository ppa:ethcore/ethcore -apt-get update -apt-get install -y --force-yes librocksdb-dev - -# install multirust -curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes - -# install beta -multirust update beta - -# download and build parity -git clone https://github.com/ethcore/parity -cd parity - -# parity should be build with rust beta -multirust override beta - -# build in release -cargo build --release -``` - -##### Other Linux - -```bash -# install rocksdb -git clone --tag v4.1 --depth=1 https://github.com/facebook/rocksdb.git -cd rocksdb -make shared_lib -sudo cp -a librocksdb.so* /usr/lib -sudo ldconfig -cd .. - -# install rust beta curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes - -# install beta -multirust update beta - -# download and build parity -git clone https://github.com/ethcore/parity -cd parity - -# parity should be build with rust beta -multirust override beta - -# build in release -cargo build --release ``` -##### OSX with Homebrew +- OSX with Homebrew: +```bash +brew update && brew install multirust +``` + +Then, download and build Parity: ```bash -# install rocksdb && multirust -brew update -brew install rocksdb -brew install multirust - -# install beta -multirust update beta - -# download and build parity +# download Parity code git clone https://github.com/ethcore/parity cd parity -# use rust beta for building parity +# parity should be built with rust beta multirust override beta +# build in release mode cargo build --release ``` - diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index a1194c665..70c6c2fb2 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -144,20 +144,20 @@ impl IsBlock for ExecutedBlock { /// Block that is ready for transactions to be added. /// -/// It's a bit like a Vec, eccept that whenever a transaction is pushed, we execute it and +/// It's a bit like a Vec, except that whenever a transaction is pushed, we execute it and /// maintain the system `state()`. We also archive execution receipts in preparation for later block creation. -pub struct OpenBlock<'x, 'y> { +pub struct OpenBlock<'x> { block: ExecutedBlock, engine: &'x Engine, - last_hashes: &'y LastHashes, + last_hashes: LastHashes, } /// Just like OpenBlock, except that we've applied `Engine::on_close_block`, finished up the non-seal header fields, /// and collected the uncles. /// /// There is no function available to push a transaction. If you want that you'll need to `reopen()` it. -pub struct ClosedBlock<'x, 'y> { - open_block: OpenBlock<'x, 'y>, +pub struct ClosedBlock<'x> { + open_block: OpenBlock<'x>, uncle_bytes: Bytes, } @@ -169,9 +169,9 @@ pub struct SealedBlock { uncle_bytes: Bytes, } -impl<'x, 'y> OpenBlock<'x, 'y> { +impl<'x> OpenBlock<'x> { /// Create a new OpenBlock ready for transaction pushing. - pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes, author: Address, extra_data: Bytes) -> Self { + pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { let mut r = OpenBlock { block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())), engine: engine, @@ -259,7 +259,7 @@ impl<'x, 'y> OpenBlock<'x, 'y> { } /// Turn this into a `ClosedBlock`. A BlockChain must be provided in order to figure out the uncles. - pub fn close(self) -> ClosedBlock<'x, 'y> { + pub fn close(self) -> ClosedBlock<'x> { let mut s = self; s.engine.on_close_block(&mut s.block); s.block.base.header.transactions_root = ordered_trie_root(s.block.base.transactions.iter().map(|ref e| e.rlp_bytes().to_vec()).collect()); @@ -275,16 +275,16 @@ impl<'x, 'y> OpenBlock<'x, 'y> { } } -impl<'x, 'y> IsBlock for OpenBlock<'x, 'y> { +impl<'x> IsBlock for OpenBlock<'x> { fn block(&self) -> &ExecutedBlock { &self.block } } -impl<'x, 'y> IsBlock for ClosedBlock<'x, 'y> { +impl<'x> IsBlock for ClosedBlock<'x> { fn block(&self) -> &ExecutedBlock { &self.open_block.block } } -impl<'x, 'y> ClosedBlock<'x, 'y> { - fn new(open_block: OpenBlock<'x, 'y>, uncle_bytes: Bytes) -> Self { +impl<'x> ClosedBlock<'x> { + fn new(open_block: OpenBlock<'x>, uncle_bytes: Bytes) -> Self { ClosedBlock { open_block: open_block, uncle_bytes: uncle_bytes, @@ -307,7 +307,7 @@ impl<'x, 'y> ClosedBlock<'x, 'y> { } /// Turn this back into an `OpenBlock`. - pub fn reopen(self) -> OpenBlock<'x, 'y> { self.open_block } + pub fn reopen(self) -> OpenBlock<'x> { self.open_block } /// Drop this object and return the underlieing database. pub fn drain(self) -> JournalDB { self.open_block.block.state.drop().1 } @@ -332,7 +332,7 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact<'x, 'y>(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact<'x>(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { { if ::log::max_log_level() >= ::log::LogLevel::Trace { let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce()); @@ -350,20 +350,20 @@ pub fn enact<'x, 'y>(header: &Header, transactions: &[SignedTransaction], uncles } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes<'x, 'y>(block_bytes: &[u8], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact_bytes<'x>(block_bytes: &[u8], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified<'x, 'y>(block: &PreVerifiedBlock, engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact_verified<'x>(block: &PreVerifiedBlock, engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards -pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: &LastHashes) -> Result { +pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(header.seal()))) } @@ -384,7 +384,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); let _ = b.seal(vec![]); } @@ -398,14 +398,14 @@ mod tests { let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(vec![]).unwrap(); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(vec![]).unwrap(); let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, &vec![genesis_header.hash()]).unwrap(); + let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, vec![genesis_header.hash()]).unwrap(); assert_eq!(e.rlp_bytes(), orig_bytes); diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index c39f158f0..62763386f 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -285,18 +285,24 @@ impl BlockQueue { } /// Mark given block and all its children as bad. Stops verification. - pub fn mark_as_bad(&mut self, hash: &H256) { + pub fn mark_as_bad(&mut self, block_hashes: &[H256]) { let mut verification_lock = self.verification.lock().unwrap(); + let mut processing = self.processing.write().unwrap(); + let mut verification = verification_lock.deref_mut(); - verification.bad.insert(hash.clone()); - self.processing.write().unwrap().remove(&hash); + + verification.bad.reserve(block_hashes.len()); + for hash in block_hashes { + verification.bad.insert(hash.clone()); + processing.remove(&hash); + } + let mut new_verified = VecDeque::new(); for block in verification.verified.drain(..) { if verification.bad.contains(&block.header.parent_hash) { verification.bad.insert(block.header.hash()); - self.processing.write().unwrap().remove(&block.header.hash()); - } - else { + processing.remove(&block.header.hash()); + } else { new_verified.push_back(block); } } @@ -304,10 +310,10 @@ impl BlockQueue { } /// Mark given block as processed - pub fn mark_as_good(&mut self, hashes: &[H256]) { + pub fn mark_as_good(&mut self, block_hashes: &[H256]) { let mut processing = self.processing.write().unwrap(); - for h in hashes { - processing.remove(&h); + for hash in block_hashes { + processing.remove(&hash); } } diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index cb38cc66c..ef6917b3a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -21,7 +21,7 @@ use util::panics::*; use blockchain::{BlockChain, BlockProvider, CacheSize}; use views::BlockView; use error::*; -use header::BlockNumber; +use header::{BlockNumber, Header}; use state::State; use spec::Spec; use engine::Engine; @@ -230,85 +230,127 @@ impl Client { self.block_queue.write().unwrap().flush(); } + fn build_last_hashes(&self, header: &Header) -> LastHashes { + let mut last_hashes = LastHashes::new(); + last_hashes.resize(256, H256::new()); + last_hashes[0] = header.parent_hash.clone(); + let chain = self.chain.read().unwrap(); + for i in 0..255 { + match chain.block_details(&last_hashes[i]) { + Some(details) => { + last_hashes[i + 1] = details.parent.clone(); + }, + None => break, + } + } + last_hashes + } + + fn check_and_close_block(&self, block: &PreVerifiedBlock) -> Result { + let engine = self.engine.deref().deref(); + let header = &block.header; + + // Verify Block Family + let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); + if let Err(e) = verify_family_result { + warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + }; + + // Check if Parent is in chain + let chain_has_parent = self.chain.read().unwrap().block_header(&header.parent_hash); + if let None = chain_has_parent { + warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); + return Err(()); + }; + + // Enact Verified Block + let parent = chain_has_parent.unwrap(); + let last_hashes = self.build_last_hashes(header); + let db = self.state_db.lock().unwrap().clone(); + + let enact_result = enact_verified(&block, engine, db, &parent, last_hashes); + if let Err(e) = enact_result { + warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + }; + + // Final Verification + let closed_block = enact_result.unwrap(); + if let Err(e) = verify_block_final(&header, closed_block.block().header()) { + warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + } + + Ok(closed_block) + } + /// This is triggered by a message coming from a block queue when the block is ready for insertion pub fn import_verified_blocks(&self, io: &IoChannel) -> usize { - let mut ret = 0; - let mut bad = HashSet::new(); + let max_blocks_to_import = 128; + + let mut good_blocks = Vec::with_capacity(max_blocks_to_import); + let mut bad_blocks = HashSet::new(); + let _import_lock = self.import_lock.lock(); - let blocks = self.block_queue.write().unwrap().drain(128); - let mut good_blocks = Vec::with_capacity(128); + let blocks = self.block_queue.write().unwrap().drain(max_blocks_to_import); + for block in blocks { - if bad.contains(&block.header.parent_hash) { - self.block_queue.write().unwrap().mark_as_bad(&block.header.hash()); - bad.insert(block.header.hash()); + let header = &block.header; + + if bad_blocks.contains(&header.parent_hash) { + bad_blocks.insert(header.hash()); continue; } - let header = &block.header; - if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) { - warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); - bad.insert(block.header.hash()); - break; - }; - let parent = match self.chain.read().unwrap().block_header(&header.parent_hash) { - Some(p) => p, - None => { - warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); - bad.insert(block.header.hash()); - break; - }, - }; - // build last hashes - let mut last_hashes = LastHashes::new(); - last_hashes.resize(256, H256::new()); - last_hashes[0] = header.parent_hash.clone(); - for i in 0..255 { - match self.chain.read().unwrap().block_details(&last_hashes[i]) { - Some(details) => { - last_hashes[i + 1] = details.parent.clone(); - }, - None => break, - } - } - - let db = self.state_db.lock().unwrap().clone(); - let result = match enact_verified(&block, self.engine.deref().deref(), db, &parent, &last_hashes) { - Ok(b) => b, - Err(e) => { - warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - bad.insert(block.header.hash()); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); - break; - } - }; - if let Err(e) = verify_block_final(&header, result.block().header()) { - warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); + let closed_block = self.check_and_close_block(&block); + if let Err(_) = closed_block { + bad_blocks.insert(header.hash()); break; } - good_blocks.push(header.hash().clone()); + // Insert block + let closed_block = closed_block.unwrap(); + self.chain.write().unwrap().insert_block(&block.bytes, closed_block.block().receipts().clone()); + good_blocks.push(header.hash()); + + let ancient = if header.number() >= HISTORY { + let n = header.number() - HISTORY; + let chain = self.chain.read().unwrap(); + Some((n, chain.block_hash(n).unwrap())) + } else { + None + }; + + // Commit results + closed_block.drain() + .commit(header.number(), &header.hash(), ancient) + .expect("State DB commit failed."); - self.chain.write().unwrap().insert_block(&block.bytes, result.block().receipts().clone()); //TODO: err here? - let ancient = if header.number() >= HISTORY { Some(header.number() - HISTORY) } else { None }; - match result.drain().commit(header.number(), &header.hash(), ancient.map(|n|(n, self.chain.read().unwrap().block_hash(n).unwrap()))) { - Ok(_) => (), - Err(e) => { - warn!(target: "client", "State DB commit failed: {:?}", e); - break; - } - } self.report.write().unwrap().accrue_block(&block); trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); - ret += 1; } - self.block_queue.write().unwrap().mark_as_good(&good_blocks); - if !good_blocks.is_empty() && self.block_queue.read().unwrap().queue_info().is_empty() { - io.send(NetworkIoMessage::User(SyncMessage::BlockVerified)).unwrap(); + + let imported = good_blocks.len(); + let bad_blocks = bad_blocks.into_iter().collect::>(); + + { + let mut block_queue = self.block_queue.write().unwrap(); + block_queue.mark_as_bad(&bad_blocks); + block_queue.mark_as_good(&good_blocks); } - ret + + { + let block_queue = self.block_queue.read().unwrap(); + if !good_blocks.is_empty() && block_queue.queue_info().is_empty() { + io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { + good: good_blocks, + bad: bad_blocks, + })).unwrap(); + } + } + + imported } /// Get a copy of the best block's state. @@ -389,7 +431,7 @@ impl BlockChainClient for Client { None => BlockStatus::Unknown } } - + fn block_total_difficulty(&self, id: BlockId) -> Option { let chain = self.chain.read().unwrap(); Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) @@ -499,7 +541,7 @@ impl BlockChainClient for Client { .collect::>() }) .collect::>() - + }) .collect() } diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 8c411e7f0..54f02524d 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -282,7 +282,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); assert_eq!(b.state().balance(&Address::zero()), U256::from_str("4563918244f40000").unwrap()); } @@ -295,7 +295,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let mut uncle = Header::new(); let uncle_author = address_from_hex("ef2d6d194084c2de36e0dabfce45d046b37d1106"); uncle.author = uncle_author.clone(); diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index d4786bc21..5eb3beeb2 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -40,23 +40,17 @@ //! - Ubuntu 14.04 and later: //! //! ```bash -//! # install rocksdb -//! add-apt-repository "deb http://ppa.launchpad.net/giskou/librocksdb/ubuntu trusty main" -//! apt-get update -//! apt-get install -y --force-yes librocksdb //! //! # install multirust //! curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes //! -//! # install nightly and make it default -//! multirust update nightly && multirust default nightly -//! //! # export rust LIBRARY_PATH //! export LIBRARY_PATH=/usr/local/lib //! //! # download and build parity //! git clone https://github.com/ethcore/parity //! cd parity +//! multirust override beta //! cargo build --release //! ``` //! @@ -65,18 +59,15 @@ //! ```bash //! # install rocksdb && multirust //! brew update -//! brew install rocksdb //! brew install multirust //! -//! # install nightly and make it default -//! multirust update nightly && multirust default nightly -//! //! # export rust LIBRARY_PATH //! export LIBRARY_PATH=/usr/local/lib //! //! # download and build parity //! git clone https://github.com/ethcore/parity //! cd parity +//! multirust override beta //! cargo build --release //! ``` diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 534aab49d..cdf3425e8 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -26,7 +26,12 @@ use client::Client; #[derive(Clone)] pub enum SyncMessage { /// New block has been imported into the blockchain - NewChainBlock(Bytes), //TODO: use Cow + NewChainBlocks { + /// Hashes of blocks imported to blockchain + good: Vec, + /// Hashes of blocks not imported to blockchain + bad: Vec, + }, /// A block is ready BlockVerified, } diff --git a/install-deps.sh b/install-deps.sh index f5e1e44cf..6cc7de002 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -342,8 +342,6 @@ function run_installer() exe brew update echo - info "Installing rocksdb" - exe brew install rocksdb info "Installing multirust" exe brew install multirust sudo multirust default beta @@ -391,7 +389,6 @@ function run_installer() linux_version find_multirust - find_rocksdb find_curl find_git @@ -402,21 +399,6 @@ function run_installer() find_sudo } - function find_rocksdb() - { - depCount=$((depCount+1)) - if [[ $(ldconfig -v 2>/dev/null | grep rocksdb | wc -l) == 1 ]]; then - depFound=$((depFound+1)) - check "apt-get" - isRocksDB=true - INSTALL_FILES+="${blue}${dim}==> librocksdb:${reset}$n" - else - uncheck "librocksdb is missing" - isRocksDB=false - INSTALL_FILES+="${blue}${dim}==> librocksdb:${reset}$n" - fi - } - function find_multirust() { depCount=$((depCount+2)) @@ -562,34 +544,6 @@ function run_installer() fi } - function ubuntu_rocksdb_installer() - { - sudo apt-get update -qq - sudo apt-get install -qq -y software-properties-common - sudo apt-add-repository -y ppa:ethcore/ethcore - sudo apt-get -f -y install - sudo apt-get update -qq - sudo apt-get install -qq -y librocksdb-dev librocksdb - } - - function linux_rocksdb_installer() - { - if [[ $isUbuntu == true ]]; then - ubuntu_rocksdb_installer - else - oldpwd=`pwd` - cd /tmp - exe git clone --branch v4.2 --depth=1 https://github.com/facebook/rocksdb.git - cd rocksdb - exe make shared_lib - sudo cp -a librocksdb.so* /usr/lib - sudo ldconfig - cd /tmp - rm -rf /tmp/rocksdb - cd $oldpwd - fi - } - function linux_installer() { if [[ $isGCC == false || $isGit == false || $isMake == false || $isCurl == false ]]; then @@ -610,12 +564,6 @@ function run_installer() echo fi - if [[ $isRocksDB == false ]]; then - info "Installing rocksdb..." - linux_rocksdb_installer - echo - fi - if [[ $isMultirust == false ]]; then info "Installing multirust..." if [[ $isSudo == false ]]; then @@ -655,10 +603,9 @@ function run_installer() find_git find_make find_gcc - find_rocksdb find_multirust - if [[ $isCurl == false || $isGit == false || $isMake == false || $isGCC == false || $isRocksDB == false || $isMultirustBeta == false ]]; then + if [[ $isCurl == false || $isGit == false || $isMake == false || $isGCC == false || $isMultirustBeta == false ]]; then abort_install fi fi diff --git a/install-parity.sh b/install-parity.sh index 60d3471d5..439700306 100755 --- a/install-parity.sh +++ b/install-parity.sh @@ -236,14 +236,29 @@ function run_installer() { linux_version - find_rocksdb - find_curl find_apt find_sudo } + function find_git() + { + depCount=$((depCount+1)) + GIT_PATH=`which git 2>/dev/null` + + if [[ -f $GIT_PATH ]] + then + depFound=$((depFound+1)) + check "git" + isGit=true + else + uncheck "git is missing" + isGit=false + INSTALL_FILES+="${blue}${dim}==> git:${reset}${n}" + fi + } + function find_brew() { BREW_PATH=`which brew 2>/dev/null` @@ -333,20 +348,6 @@ function run_installer() fi } - function find_rocksdb() - { - depCount=$((depCount+1)) - if [[ $(ldconfig -v 2>/dev/null | grep rocksdb | wc -l) == 1 ]]; then - depFound=$((depFound+1)) - check "librocksdb" - isRocksDB=true - else - uncheck "librocksdb is missing" - isRocksDB=false - INSTALL_FILES+="${blue}${dim}==>${reset}\tlibrocksdb${n}" - fi - } - function find_apt() { depCount=$((depCount+1)) @@ -386,10 +387,9 @@ function run_installer() info "Verifying installation" if [[ $OS_TYPE == "linux" ]]; then - find_rocksdb find_apt - if [[ $isRocksDB == false || $isApt == false ]]; then + if [[ $isApt == false ]]; then abortInstall fi fi @@ -397,23 +397,11 @@ function run_installer() function linux_deps_installer() { - if [[ $isRocksDB == false || $isCurl == false ]]; then + if [[ $isCurl == false ]]; then info "Preparing apt..." sudo apt-get update -qq echo fi - - if [[ $isRocksDB == false ]]; then - info "Installing rocksdb..." - - sudo apt-get install -qq -y software-properties-common - sudo apt-add-repository -y ppa:ethcore/ethcore - sudo apt-get -f -y install - sudo apt-get update -qq - sudo apt-get install -qq -y librocksdb - - echo - fi if [[ $isCurl == false ]]; then info "Installing curl..." diff --git a/parity/main.rs b/parity/main.rs index 964529303..c0c260371 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -71,9 +71,9 @@ Options: --listen-address URL Specify the IP/port on which to listen for peers [default: 0.0.0.0:30304]. --public-address URL Specify the IP/port on which peers may connect. --address URL Equivalent to --listen-address URL --public-address URL. - --peers NUM Try to manintain that many peers [default: 25]. + --peers NUM Try to maintain that many peers [default: 25]. --no-discovery Disable new peer discovery. - --upnp Use UPnP to try to figure out the correct network settings. + --no-upnp Disable trying to figure out the correct public adderss over UPnP. --node-key KEY Specify node secret key, either as 64-character hex string or input to SHA3 operation. --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. @@ -102,7 +102,7 @@ struct Args { flag_address: Option, flag_peers: u32, flag_no_discovery: bool, - flag_upnp: bool, + flag_no_upnp: bool, flag_node_key: Option, flag_cache_pref_size: usize, flag_cache_max_size: usize, @@ -246,7 +246,7 @@ impl Configuration { fn net_settings(&self, spec: &Spec) -> NetworkConfiguration { let mut ret = NetworkConfiguration::new(); - ret.nat_enabled = self.args.flag_upnp; + ret.nat_enabled = !self.args.flag_no_upnp; ret.boot_nodes = self.init_nodes(spec); let (listen, public) = self.net_addresses(); ret.listen_address = listen; diff --git a/util/Cargo.toml b/util/Cargo.toml index 96ad8a5c2..3c742715e 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -39,6 +39,7 @@ target_info = "0.1" [features] default = [] dev = ["clippy"] +x64asm = [] [build-dependencies] vergen = "*" diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs new file mode 100644 index 000000000..da82084b8 --- /dev/null +++ b/util/benches/bigint.rs @@ -0,0 +1,84 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! benchmarking for rlp +//! should be started with: +//! ```bash +//! multirust run nightly cargo bench +//! ``` + +#![feature(test)] +#![feature(asm)] + +extern crate test; +extern crate ethcore_util; +extern crate rand; + +use test::{Bencher, black_box}; +use ethcore_util::uint::*; + +#[bench] +fn u256_add(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_add(U256::from(new)).0 }) + }); +} + +#[bench] +fn u256_sub(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_sub(U256::from(new)).0 }) + }); +} + +#[bench] +fn u512_sub(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U512([rand::random::(), rand::random::(), rand::random::(), rand::random::(), + rand::random::(), rand::random::(), rand::random::(), rand::random::()]), + |old, new| { old.overflowing_sub(U512([0, 0, 0, 0, 0, 0, 0, new])).0 }) + }); +} + +#[bench] +fn u512_add(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U512([0, 0, 0, 0, 0, 0, 0, 0]), + |old, new| { old.overflowing_add(U512([new, new, new, new, new, new, new, new])).0 }) + }); +} + +#[bench] +fn u256_mul(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_mul(U256::from(new)).0 }) + }); +} + + +#[bench] +fn u128_mul(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U128([12345u64, 0u64]), |old, new| { old.overflowing_mul(U128::from(new)).0 }) + }); +} + diff --git a/util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 b/util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 new file mode 100644 index 000000000..afc376774 --- /dev/null +++ b/util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 @@ -0,0 +1,21 @@ +{ + "address": "3f49624084b67849c7b4e805c5988c21a430f9d9", + "Crypto": { + "cipher": "aes-128-ctr", + "ciphertext": "9f27e3dd4fc73e7103ed61e5493662189a3eb52223ae49e3d1deacc04c889eae", + "cipherparams": { + "iv": "457494bf05f2618c397dc74dbb5181c0" + }, + "kdf": "scrypt", + "kdfparams": { + "dklen": 32, + "n": 262144, + "p": 1, + "r": 8, + "salt": "db14edb18c41ee7f5ec4397df89c3a2ae4d0af60884c52bb54ce490574f8df33" + }, + "mac": "572d24532438d31fdf513c744a3ff26c933ffda5744ee42bc71661cbe3f2112e" + }, + "id": "62a0ad73-556d-496a-8e1c-0783d30d3ace", + "version": 3 +} diff --git a/util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf b/util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf new file mode 100644 index 000000000..b14922037 --- /dev/null +++ b/util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf @@ -0,0 +1,21 @@ +{ + "address": "5ba4dcf897e97c2bdf8315b9ef26c13c085988cf", + "Crypto": { + "cipher": "aes-128-ctr", + "ciphertext": "d4a08ec930163778273920f6ad1d49b71836337be6fd9863993ac700a612fddd", + "cipherparams": { + "iv": "89ce5ec129fc27cd5bcbeb8c92bdad50" + }, + "kdf": "scrypt", + "kdfparams": { + "dklen": 32, + "n": 262144, + "p": 1, + "r": 8, + "salt": "612ab108dc37e69ee8af37a7b24bf7f2234086d7bbf945bacdeccce331f7f84a" + }, + "mac": "4152caa7444e06784223d735cea80cd2690b4c587ad8db3d5529442227b25695" + }, + "id": "35086353-fb12-4029-b56b-033cd61ce35b", + "version": 3 +} diff --git a/util/src/hash.rs b/util/src/hash.rs index 71c690ef6..a6e8f7950 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -235,7 +235,7 @@ macro_rules! impl_hash { } impl serde::Serialize for $from { - fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> + fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> where S: serde::Serializer { let mut hex = "0x".to_owned(); hex.push_str(self.to_hex().as_ref()); @@ -250,7 +250,7 @@ macro_rules! impl_hash { impl serde::de::Visitor for HashVisitor { type Value = $from; - + fn visit_str(&mut self, value: &str) -> Result where E: serde::Error { // 0x + len if value.len() != 2 + $size * 2 { @@ -719,4 +719,3 @@ mod tests { assert_eq!(r, u); } } - diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index d7db8ac11..7178508a2 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -333,7 +333,9 @@ pub struct KeyFileContent { /// Holds cypher and decrypt function settings. pub crypto: KeyFileCrypto, /// The identifier. - pub id: Uuid + pub id: Uuid, + /// Account (if present) + pub account: Option
, } #[derive(Debug)] @@ -374,7 +376,19 @@ impl KeyFileContent { KeyFileContent { id: new_uuid(), version: KeyFileVersion::V3(3), - crypto: crypto + crypto: crypto, + account: None + } + } + + /// Loads key from valid json, returns error and records warning if key is mallformed + pub fn load(json: &Json) -> Result { + match Self::from_json(json) { + Ok(key_file) => Ok(key_file), + Err(e) => { + warn!(target: "sstore", "Error parsing json for key: {:?}", e); + Err(()) + } } } @@ -407,6 +421,9 @@ impl KeyFileContent { Ok(id) => id }; + let account = as_object.get("address").and_then(|json| json.as_string()).and_then( + |account_text| match Address::from_str(account_text) { Ok(account) => Some(account), Err(_) => None }); + let crypto = match as_object.get("crypto") { None => { return Err(KeyFileParseError::NoCryptoSection); } Some(crypto_json) => match KeyFileCrypto::from_json(crypto_json) { @@ -418,7 +435,8 @@ impl KeyFileContent { Ok(KeyFileContent { version: version, id: id.clone(), - crypto: crypto + crypto: crypto, + account: account }) } @@ -427,6 +445,7 @@ impl KeyFileContent { map.insert("id".to_owned(), Json::String(uuid_to_string(&self.id))); map.insert("version".to_owned(), Json::U64(CURRENT_DECLARED_VERSION)); map.insert("crypto".to_owned(), self.crypto.to_json()); + if let Some(ref address) = self.account { map.insert("address".to_owned(), Json::String(format!("{:?}", address))); } Json::Object(map) } } @@ -599,6 +618,8 @@ impl KeyDirectory { Err(_) => Err(KeyFileLoadError::ParseError(KeyFileParseError::InvalidJson)) } } + + } @@ -653,7 +674,7 @@ mod file_tests { } #[test] - fn can_read_scrypt_krf() { + fn can_read_scrypt_kdf() { let json = Json::from_str( r#" { @@ -689,6 +710,47 @@ mod file_tests { } } + #[test] + fn can_read_scrypt_kdf_params() { + let json = Json::from_str( + r#" + { + "crypto" : { + "cipher" : "aes-128-ctr", + "cipherparams" : { + "iv" : "83dbcc02d8ccb40e466191a123791e0e" + }, + "ciphertext" : "d172bf743a674da9cdad04534d56926ef8358534d458fffccd4e6ad2fbde479c", + "kdf" : "scrypt", + "kdfparams" : { + "dklen" : 32, + "n" : 262144, + "r" : 1, + "p" : 8, + "salt" : "ab0c7876052600dd703518d6fc3fe8984592145b591fc8fb5c6d43190334ba19" + }, + "mac" : "2103ac29920d71da29f15d75b4a16dbe95cfd7ff8faea1056c33131d846e3097" + }, + "id" : "3198bc9c-6672-5ab3-d995-4942343ae5b6", + "version" : 3 + } + "#).unwrap(); + + match KeyFileContent::from_json(&json) { + Ok(key_file) => { + match key_file.crypto.kdf { + KeyFileKdf::Scrypt(scrypt_params) => { + assert_eq!(262144, scrypt_params.n); + assert_eq!(1, scrypt_params.r); + assert_eq!(8, scrypt_params.p); + }, + _ => { panic!("expected kdf params of crypto to be of scrypt type" ); } + } + }, + Err(e) => panic!("Error parsing valid file: {:?}", e) + } + } + #[test] fn can_return_error_no_id() { let json = Json::from_str( @@ -844,7 +906,7 @@ mod file_tests { panic!("Should be error of no identifier, got ok"); }, Err(KeyFileParseError::Crypto(CryptoParseError::Scrypt(_))) => { }, - Err(other_error) => { panic!("should be error of no identifier, got {:?}", other_error); } + Err(other_error) => { panic!("should be scrypt parse error, got {:?}", other_error); } } } diff --git a/util/src/keys/geth_import.rs b/util/src/keys/geth_import.rs new file mode 100644 index 000000000..2ee3c987c --- /dev/null +++ b/util/src/keys/geth_import.rs @@ -0,0 +1,165 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Geth keys import/export tool + +use common::*; +use keys::store::SecretStore; +use keys::directory::KeyFileContent; + +/// Enumerates all geth keys in the directory and returns collection of tuples `(accountId, filename)` +pub fn enumerate_geth_keys(path: &Path) -> Result, io::Error> { + let mut entries = Vec::new(); + for entry in try!(fs::read_dir(path)) { + let entry = try!(entry); + if !try!(fs::metadata(entry.path())).is_dir() { + match entry.file_name().to_str() { + Some(name) => { + let parts: Vec<&str> = name.split("--").collect(); + if parts.len() != 3 { continue; } + match Address::from_str(parts[2]) { + Ok(account_id) => { entries.push((account_id, name.to_owned())); } + Err(e) => { panic!("error: {:?}", e); } + } + }, + None => { continue; } + }; + } + } + Ok(entries) +} + +/// Geth import error +#[derive(Debug)] +pub enum ImportError { + /// Io error reading geth file + IoError(io::Error), + /// format error + FormatError, +} + +impl From for ImportError { + fn from (err: io::Error) -> ImportError { + ImportError::IoError(err) + } +} + +/// Imports one geth key to the store +pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) -> Result<(), ImportError> { + let mut file = try!(fs::File::open(geth_keyfile_path)); + let mut buf = String::new(); + try!(file.read_to_string(&mut buf)); + + let mut json_result = Json::from_str(&buf); + let mut json = match json_result { + Ok(ref mut parsed_json) => try!(parsed_json.as_object_mut().ok_or(ImportError::FormatError)), + Err(_) => { return Err(ImportError::FormatError); } + }; + let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::FormatError)).clone(); + json.insert("crypto".to_owned(), Json::Object(crypto_object)); + json.remove("Crypto"); + match KeyFileContent::load(&Json::Object(json.clone())) { + Ok(key_file) => try!(secret_store.import_key(key_file)), + Err(_) => { return Err(ImportError::FormatError); } + }; + Ok(()) +} + +/// Imports all geth keys in the directory +pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> { + use std::path::PathBuf; + let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory)); + for &(ref address, ref file_path) in geth_files.iter() { + let mut path = PathBuf::new(); + path.push(geth_keyfiles_directory); + path.push(file_path); + if let Err(e) = import_geth_key(secret_store, Path::new(&path)) { + warn!("Skipped geth address {}, error importing: {:?}", address, e) + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use common::*; + use keys::store::SecretStore; + + + #[test] + fn can_enumerate() { + let keys = enumerate_geth_keys(Path::new("res/geth_keystore")).unwrap(); + assert_eq!(2, keys.len()); + } + + #[test] + fn can_import() { + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_key(&mut secret_store, Path::new("res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9")).unwrap(); + let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + assert!(key.is_some()); + } + + #[test] + fn can_import_directory() { + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + + let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + assert!(key.is_some()); + + let key = secret_store.account(&Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap()); + assert!(key.is_some()); + } + + #[test] + fn imports_as_scrypt_keys() { + use keys::directory::{KeyDirectory, KeyFileKdf}; + let temp = ::devtools::RandomTempPath::create_dir(); + { + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + } + + let key_directory = KeyDirectory::new(&temp.as_path()); + let key_file = key_directory.get(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap()).unwrap(); + + match key_file.crypto.kdf { + KeyFileKdf::Scrypt(scrypt_params) => { + assert_eq!(262144, scrypt_params.n); + assert_eq!(8, scrypt_params.r); + assert_eq!(1, scrypt_params.p); + }, + _ => { panic!("expected kdf params of crypto to be of scrypt type" ); } + } + } + + #[test] + fn can_decrypt_with_imported() { + use keys::store::EncryptedHashMap; + + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + + let val = secret_store.get::(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap(), "123"); + assert!(val.is_ok()); + assert_eq!(32, val.unwrap().len()); + } +} diff --git a/util/src/keys/mod.rs b/util/src/keys/mod.rs index fd52136d7..b9c9dcb08 100644 --- a/util/src/keys/mod.rs +++ b/util/src/keys/mod.rs @@ -18,3 +18,4 @@ pub mod directory; pub mod store; +mod geth_import; diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index ae44d567a..c4fa377f9 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -19,11 +19,12 @@ use keys::directory::*; use common::*; use rcrypto::pbkdf2::*; +use rcrypto::scrypt::*; use rcrypto::hmac::*; use crypto; const KEY_LENGTH: u32 = 32; -const KEY_ITERATIONS: u32 = 4096; +const KEY_ITERATIONS: u32 = 10240; const KEY_LENGTH_AES: u32 = KEY_LENGTH/2; const KEY_LENGTH_USIZE: usize = KEY_LENGTH as usize; @@ -60,15 +61,62 @@ pub struct SecretStore { } impl SecretStore { - /// new instance of Secret Store + /// new instance of Secret Store in default home directory pub fn new() -> SecretStore { let mut path = ::std::env::home_dir().expect("Failed to get home dir"); - path.push(".keys"); + path.push(".parity"); + path.push("keys"); + Self::new_in(&path) + } + + /// new instance of Secret Store in specific directory + pub fn new_in(path: &Path) -> SecretStore { SecretStore { - directory: KeyDirectory::new(&path) + directory: KeyDirectory::new(path) } } + /// trys to import keys in the known locations + pub fn try_import_existing(&mut self) { + use std::path::PathBuf; + use keys::geth_import; + + let mut import_path = PathBuf::new(); + import_path.push(::std::env::home_dir().expect("Failed to get home dir")); + import_path.push(".ethereum"); + import_path.push("keystore"); + if let Err(e) = geth_import::import_geth_keys(self, &import_path) { + warn!(target: "sstore", "Error retrieving geth keys: {:?}", e) + } + } + + /// Lists all accounts and corresponding key ids + pub fn accounts(&self) -> Result, ::std::io::Error> { + let accounts = try!(self.directory.list()).iter().map(|key_id| self.directory.get(key_id)) + .filter(|key| key.is_some()) + .map(|key| { let some_key = key.unwrap(); (some_key.account, some_key.id) }) + .filter(|&(ref account, _)| account.is_some()) + .map(|(account, id)| (account.unwrap(), id)) + .collect::>(); + Ok(accounts) + } + + /// Resolves key_id by account address + pub fn account(&self, account: &Address) -> Option { + let mut accounts = match self.accounts() { + Ok(accounts) => accounts, + Err(e) => { warn!(target: "sstore", "Failed to load accounts: {}", e); return None; } + }; + accounts.retain(|&(ref store_account, _)| account == store_account); + accounts.first().and_then(|&(_, ref key_id)| Some(key_id.clone())) + } + + /// Imports pregenerated key, returns error if not saved correctly + pub fn import_key(&mut self, key_file: KeyFileContent) -> Result<(), ::std::io::Error> { + try!(self.directory.save(key_file)); + Ok(()) + } + #[cfg(test)] fn new_test(path: &::devtools::RandomTempPath) -> SecretStore { SecretStore { @@ -90,6 +138,15 @@ fn derive_key(password: &str, salt: &H256) -> (Bytes, Bytes) { derive_key_iterations(password, salt, KEY_ITERATIONS) } +fn derive_key_scrypt(password: &str, salt: &H256, n: u32, p: u32, r: u32) -> (Bytes, Bytes) { + let mut derived_key = vec![0u8; KEY_LENGTH_USIZE]; + let scrypt_params = ScryptParams::new(n.trailing_zeros() as u8, r, p); + scrypt(password.as_bytes(), &salt.as_slice(), &scrypt_params, &mut derived_key); + let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE]; + let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE]; + (derived_right_bits.to_vec(), derived_left_bits.to_vec()) +} + fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes { let mut mac = vec![0u8; KEY_LENGTH_AES_USIZE + cipher_text.len()]; mac[0..KEY_LENGTH_AES_USIZE].clone_from_slice(derived_left_bits); @@ -101,24 +158,22 @@ impl EncryptedHashMap for SecretStore { fn get(&self, key: &H128, password: &str) -> Result { match self.directory.get(key) { Some(key_file) => { - let decrypted_bytes = match key_file.crypto.kdf { - KeyFileKdf::Pbkdf2(ref params) => { - let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, ¶ms.salt, params.c); - if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) - .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } - - let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; - match key_file.crypto.cipher_type { - CryptoCipherType::Aes128Ctr(ref iv) => { - crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val); - } - } - val - } - _ => { unimplemented!(); } + let (derived_left_bits, derived_right_bits) = match key_file.crypto.kdf { + KeyFileKdf::Pbkdf2(ref params) => derive_key_iterations(password, ¶ms.salt, params.c), + KeyFileKdf::Scrypt(ref params) => derive_key_scrypt(password, ¶ms.salt, params.n, params.p, params.r) }; - match Value::from_bytes(&decrypted_bytes) { + if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) + .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } + + let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; + match key_file.crypto.cipher_type { + CryptoCipherType::Aes128Ctr(ref iv) => { + crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val); + } + }; + + match Value::from_bytes(&val) { Ok(value) => Ok(value), Err(bytes_error) => Err(EncryptedHashMapError::InvalidValueFormat(bytes_error)) } @@ -259,6 +314,27 @@ mod tests { result } + fn pregenerate_accounts(temp: &RandomTempPath, count: usize) -> Vec { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let mut write_sstore = SecretStore::new_test(&temp); + let mut result = Vec::new(); + for i in 0..count { + let mut key_file = + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32)); + key_file.account = Some(x!(i as u64)); + result.push(key_file.id.clone()); + write_sstore.import_key(key_file).unwrap(); + } + result + } + #[test] fn can_get() { let temp = RandomTempPath::create_dir(); @@ -293,5 +369,35 @@ mod tests { assert_eq!(4, sstore.directory.list().unwrap().len()) } + #[test] + fn can_import_account() { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let temp = RandomTempPath::create_dir(); + let mut key_file = + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32)); + key_file.account = Some(Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + let mut sstore = SecretStore::new_test(&temp); + + sstore.import_key(key_file).unwrap(); + + assert_eq!(1, sstore.accounts().unwrap().len()); + assert!(sstore.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()).is_some()); + } + + #[test] + fn can_list_accounts() { + let temp = RandomTempPath::create_dir(); + pregenerate_accounts(&temp, 30); + let sstore = SecretStore::new_test(&temp); + let accounts = sstore.accounts().unwrap(); + assert_eq!(30, accounts.len()); + } } diff --git a/util/src/lib.rs b/util/src/lib.rs index 0d074df8c..be1e788c9 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -16,6 +16,7 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] +#![cfg_attr(feature="x64asm", feature(asm))] #![cfg_attr(feature="dev", plugin(clippy))] // Clippy settings diff --git a/util/src/network/host.rs b/util/src/network/host.rs index d50f2d86d..f54c85855 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -106,6 +106,7 @@ const IDLE: usize = LAST_HANDSHAKE + 2; const DISCOVERY: usize = LAST_HANDSHAKE + 3; const DISCOVERY_REFRESH: usize = LAST_HANDSHAKE + 4; const DISCOVERY_ROUND: usize = LAST_HANDSHAKE + 5; +const INIT_PUBLIC: usize = LAST_HANDSHAKE + 6; const FIRST_SESSION: usize = 0; const LAST_SESSION: usize = FIRST_SESSION + MAX_SESSIONS - 1; const FIRST_HANDSHAKE: usize = LAST_SESSION + 1; @@ -261,7 +262,9 @@ pub struct HostInfo { /// TCP connection port. pub listen_port: u16, /// Registered capabilities (handlers) - pub capabilities: Vec + pub capabilities: Vec, + /// Public address + discovery port + public_endpoint: NodeEndpoint, } impl HostInfo { @@ -294,16 +297,15 @@ struct ProtocolTimer { /// Root IO handler. Manages protocol handlers, IO timers and network connections. pub struct Host where Message: Send + Sync + Clone { pub info: RwLock, - tcp_listener: Mutex, + tcp_listener: Mutex>, handshakes: Arc>>, sessions: Arc>>, - discovery: Option>, + discovery: Mutex>, nodes: RwLock, handlers: RwLock>>>, timers: RwLock>, timer_counter: RwLock, stats: Arc, - public_endpoint: NodeEndpoint, pinned_nodes: Vec, } @@ -316,27 +318,6 @@ impl Host where Message: Send + Sync + Clone { }; let udp_port = config.udp_port.unwrap_or(listen_address.port()); - let public_endpoint = match config.public_address { - None => { - let public_address = select_public_address(listen_address.port()); - let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; - if config.nat_enabled { - match map_external_address(&local_endpoint) { - Some(endpoint) => { - info!("NAT Mappped to external address {}", endpoint.address); - endpoint - }, - None => local_endpoint - } - } else { - local_endpoint - } - } - Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } - }; - - // Setup the server socket - let tcp_listener = TcpListener::bind(&listen_address).unwrap(); let keys = if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { @@ -350,10 +331,8 @@ impl Host where Message: Send + Sync + Clone { }, |s| KeyPair::from_secret(s).expect("Error creating node secret key")) }; - let discovery = if config.discovery_enabled && !config.pin { - Some(Discovery::new(&keys, listen_address.clone(), public_endpoint.clone(), DISCOVERY)) - } else { None }; let path = config.config_path.clone(); + let local_endpoint = NodeEndpoint { address: listen_address, udp_port: udp_port }; let mut host = Host:: { info: RwLock::new(HostInfo { keys: keys, @@ -363,9 +342,10 @@ impl Host where Message: Send + Sync + Clone { client_version: version(), listen_port: 0, capabilities: Vec::new(), + public_endpoint: local_endpoint, // will be replaced by public once it is resolved }), - discovery: discovery.map(Mutex::new), - tcp_listener: Mutex::new(tcp_listener), + discovery: Mutex::new(None), + tcp_listener: Mutex::new(None), handshakes: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_HANDSHAKE, MAX_HANDSHAKES))), sessions: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_SESSION, MAX_SESSIONS))), nodes: RwLock::new(NodeTable::new(path)), @@ -373,16 +353,12 @@ impl Host where Message: Send + Sync + Clone { timers: RwLock::new(HashMap::new()), timer_counter: RwLock::new(USER_TIMER), stats: Arc::new(NetworkStats::default()), - public_endpoint: public_endpoint, pinned_nodes: Vec::new(), }; let port = listen_address.port(); host.info.write().unwrap().deref_mut().listen_port = port; let boot_nodes = host.info.read().unwrap().config.boot_nodes.clone(); - if let Some(ref mut discovery) = host.discovery { - discovery.lock().unwrap().init_node_list(host.nodes.read().unwrap().unordered_entries()); - } for n in boot_nodes { host.add_node(&n); } @@ -400,8 +376,8 @@ impl Host where Message: Send + Sync + Clone { let entry = NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }; self.pinned_nodes.push(n.id.clone()); self.nodes.write().unwrap().add_node(n); - if let Some(ref mut discovery) = self.discovery { - discovery.lock().unwrap().add_node(entry); + if let &mut Some(ref mut discovery) = self.discovery.lock().unwrap().deref_mut() { + discovery.add_node(entry); } } } @@ -412,7 +388,61 @@ impl Host where Message: Send + Sync + Clone { } pub fn client_url(&self) -> String { - format!("{}", Node::new(self.info.read().unwrap().id().clone(), self.public_endpoint.clone())) + format!("{}", Node::new(self.info.read().unwrap().id().clone(), self.info.read().unwrap().public_endpoint.clone())) + } + + fn init_public_interface(&self, io: &IoContext>) { + io.clear_timer(INIT_PUBLIC).unwrap(); + let mut tcp_listener = self.tcp_listener.lock().unwrap(); + if tcp_listener.is_some() { + return; + } + // public_endpoint in host info contains local adderss at this point + let listen_address = self.info.read().unwrap().public_endpoint.address.clone(); + let udp_port = self.info.read().unwrap().config.udp_port.unwrap_or(listen_address.port()); + let public_endpoint = match self.info.read().unwrap().config.public_address { + None => { + let public_address = select_public_address(listen_address.port()); + let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; + if self.info.read().unwrap().config.nat_enabled { + match map_external_address(&local_endpoint) { + Some(endpoint) => { + info!("NAT mappped to external address {}", endpoint.address); + endpoint + }, + None => local_endpoint + } + } else { + local_endpoint + } + } + Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } + }; + + // Setup the server socket + *tcp_listener = Some(TcpListener::bind(&listen_address).unwrap()); + self.info.write().unwrap().public_endpoint = public_endpoint.clone(); + io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); + info!("Public node URL: {}", self.client_url()); + + // Initialize discovery. + let discovery = { + let info = self.info.read().unwrap(); + if info.config.discovery_enabled && !info.config.pin { + Some(Discovery::new(&info.keys, listen_address.clone(), public_endpoint, DISCOVERY)) + } else { None } + }; + + if let Some(mut discovery) = discovery { + discovery.init_node_list(self.nodes.read().unwrap().unordered_entries()); + for n in self.nodes.read().unwrap().unordered_entries() { + discovery.add_node(n.clone()); + } + io.register_stream(DISCOVERY).expect("Error registering UDP listener"); + io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); + io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); + *self.discovery.lock().unwrap().deref_mut() = Some(discovery); + } } fn maintain_network(&self, io: &IoContext>) { @@ -526,7 +556,7 @@ impl Host where Message: Send + Sync + Clone { fn accept(&self, io: &IoContext>) { trace!(target: "network", "Accepting incoming connection"); loop { - let socket = match self.tcp_listener.lock().unwrap().accept() { + let socket = match self.tcp_listener.lock().unwrap().as_ref().unwrap().accept() { Ok(None) => break, Ok(Some((sock, _addr))) => sock, Err(e) => { @@ -666,8 +696,9 @@ impl Host where Message: Send + Sync + Clone { if let Ok(address) = session.remote_addr() { let entry = NodeEntry { id: session.id().clone(), endpoint: NodeEndpoint { address: address, udp_port: address.port() } }; self.nodes.write().unwrap().add_node(Node::new(entry.id.clone(), entry.endpoint.clone())); - if let Some(ref discovery) = self.discovery { - discovery.lock().unwrap().add_node(entry); + let mut discovery = self.discovery.lock().unwrap(); + if let &mut Some(ref mut discovery) = discovery.deref_mut() { + discovery.add_node(entry); } } } @@ -764,13 +795,8 @@ impl Host where Message: Send + Sync + Clone { impl IoHandler> for Host where Message: Send + Sync + Clone + 'static { /// Initialize networking fn initialize(&self, io: &IoContext>) { - io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); io.register_timer(IDLE, MAINTENANCE_TIMEOUT).expect("Error registering Network idle timer"); - if self.discovery.is_some() { - io.register_stream(DISCOVERY).expect("Error registering UDP listener"); - io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); - io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); - } + io.register_timer(INIT_PUBLIC, 0).expect("Error registering initialization timer"); self.maintain_network(io) } @@ -788,7 +814,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_readable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(stream, io), DISCOVERY => { - let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().readable() }; + let node_changes = { self.discovery.lock().unwrap().as_mut().unwrap().readable() }; if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } @@ -804,7 +830,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_writable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(stream, io), DISCOVERY => { - self.discovery.as_ref().unwrap().lock().unwrap().writable(); + self.discovery.lock().unwrap().as_mut().unwrap().writable(); io.update_registration(DISCOVERY).expect("Error updating discovery registration"); } _ => panic!("Received unknown writable token"), @@ -814,14 +840,15 @@ impl IoHandler> for Host where Messa fn timeout(&self, io: &IoContext>, token: TimerToken) { match token { IDLE => self.maintain_network(io), + INIT_PUBLIC => self.init_public_interface(io), FIRST_SESSION ... LAST_SESSION => self.connection_timeout(token, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_timeout(token, io), DISCOVERY_REFRESH => { - self.discovery.as_ref().unwrap().lock().unwrap().refresh(); + self.discovery.lock().unwrap().as_mut().unwrap().refresh(); io.update_registration(DISCOVERY).expect("Error updating discovery registration"); }, DISCOVERY_ROUND => { - let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().round() }; + let node_changes = { self.discovery.lock().unwrap().as_mut().unwrap().round() }; if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } @@ -896,8 +923,8 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().register_socket(reg, event_loop).expect("Error registering socket"); } } - DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), - TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), + DISCOVERY => self.discovery.lock().unwrap().as_ref().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), + TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().as_ref().unwrap(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), _ => warn!("Unexpected stream registration") } } @@ -919,7 +946,6 @@ impl IoHandler> for Host where Messa } } DISCOVERY => (), - TCP_ACCEPT => event_loop.deregister(self.tcp_listener.lock().unwrap().deref()).unwrap(), _ => warn!("Unexpected stream deregistration") } } @@ -938,8 +964,8 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().update_socket(reg, event_loop).expect("Error updating socket"); } } - DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), - TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), + DISCOVERY => self.discovery.lock().unwrap().as_ref().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), + TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().as_ref().unwrap(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), _ => warn!("Unexpected stream update") } } diff --git a/util/src/network/service.rs b/util/src/network/service.rs index 1cd48abe1..7b9388e85 100644 --- a/util/src/network/service.rs +++ b/util/src/network/service.rs @@ -42,7 +42,6 @@ impl NetworkService where Message: Send + Sync + Clone + 'stat let host = Arc::new(Host::new(config)); let stats = host.stats().clone(); let host_info = host.client_version(); - info!("Node URL: {}", host.client_url()); try!(io_service.register_handler(host)); Ok(NetworkService { io_service: io_service, diff --git a/util/src/uint.rs b/util/src/uint.rs index 6490cbd9b..5a2730126 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -51,6 +51,339 @@ macro_rules! impl_map_from { } } +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +macro_rules! uint_overflowing_add { + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ + uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) + }) +} + +macro_rules! uint_overflowing_add_reg { + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ + let $name(ref me) = $self_expr; + let $name(ref you) = $other; + let mut ret = [0u64; $n_words]; + let mut carry = [0u64; $n_words]; + let mut b_carry = false; + let mut overflow = false; + + for i in 0..$n_words { + ret[i] = me[i].wrapping_add(you[i]); + + if ret[i] < me[i] { + if i < $n_words - 1 { + carry[i + 1] = 1; + b_carry = true; + } else { + overflow = true; + } + } + } + if b_carry { + let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow); + (ret, overflow) + } else { + ($name(ret), overflow) + } + }) +} + + +#[cfg(all(feature="x64asm", target_arch="x86_64"))] +macro_rules! uint_overflowing_add { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + unsafe { + asm!(" + add $9, $0 + adc $10, $1 + adc $11, $2 + adc $12, $3 + setc %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) + : + : + ); + } + (U256(result), overflow != 0) + }); + (U512, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 8] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 8] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 8] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + + unsafe { + asm!(" + add $15, $0 + adc $16, $1 + adc $17, $2 + adc $18, $3 + lodsq + adc $11, %rax + stosq + lodsq + adc $12, %rax + stosq + lodsq + adc $13, %rax + stosq + lodsq + adc $14, %rax + stosq + setc %al + + ": "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), + + "={al}"(overflow) /* $0 - $4 */ + + : "{rdi}"(&result[4] as *const u64) /* $5 */ + "{rsi}"(&other_t[4] as *const u64) /* $6 */ + "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "m"(self_t[4]), "m"(self_t[5]), "m"(self_t[6]), "m"(self_t[7]), + /* $7 - $14 */ + + "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]), + "m"(other_t[4]), "m"(other_t[5]), "m"(other_t[6]), "m"(other_t[7]) /* $15 - $22 */ + : "rdi", "rsi" + : + ); + } + (U512(result), overflow != 0) + }); + + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( + uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) + ) +} + +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +macro_rules! uint_overflowing_sub { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let res = overflowing!((!$other).overflowing_add(From::from(1u64))); + let res = overflowing!($self_expr.overflowing_add(res)); + (res, $self_expr < $other) + }) +} + +#[cfg(all(feature="x64asm", target_arch="x86_64"))] +macro_rules! uint_overflowing_sub { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + unsafe { + asm!(" + sub $9, $0 + sbb $10, $1 + sbb $11, $2 + sbb $12, $3 + setb %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) + : + : + ); + } + (U256(result), overflow != 0) + }); + (U512, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 8] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 8] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 8] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + + unsafe { + asm!(" + sub $15, $0 + sbb $16, $1 + sbb $17, $2 + sbb $18, $3 + lodsq + sbb $19, %rax + stosq + lodsq + sbb $20, %rax + stosq + lodsq + sbb $21, %rax + stosq + lodsq + sbb $22, %rax + stosq + setb %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), + + "={al}"(overflow) /* $0 - $4 */ + + : "{rdi}"(&result[4] as *const u64) /* $5 */ + "{rsi}"(&self_t[4] as *const u64) /* $6 */ + "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "m"(self_t[4]), "m"(self_t[5]), "m"(self_t[6]), "m"(self_t[7]), + /* $7 - $14 */ + + "m"(other_t[0]), "m"(other_t[1]), "m"(other_t[2]), "m"(other_t[3]), + "m"(other_t[4]), "m"(other_t[5]), "m"(other_t[6]), "m"(other_t[7]) /* $15 - $22 */ + : "rdi", "rsi" + : + ); + } + (U512(result), overflow != 0) + }); + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let res = overflowing!((!$other).overflowing_add(From::from(1u64))); + let res = overflowing!($self_expr.overflowing_add(res)); + (res, $self_expr < $other) + }) +} + +#[cfg(all(feature="x64asm", target_arch="x86_64"))] +macro_rules! uint_overflowing_mul { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u64; + unsafe { + asm!(" + mov $5, %rax + mulq $9 + mov %rax, $0 + mov %rdx, $1 + + mov $6, %rax + mulq $9 + add %rax, $1 + mov %rdx, $2 + + mov $5, %rax + mulq $10 + add %rax, $1 + adc %rdx, $2 + + mov $6, %rax + mulq $10 + add %rax, $2 + mov %rdx, $3 + + mov $7, %rax + mulq $9 + add %rax, $2 + adc %rdx, $3 + + mov $5, %rax + mulq $11 + add %rax, $2 + adc %rdx, $3 + + mov $8, %rax + mulq $9 + adc %rax, $3 + adc $$0, %rdx + mov %rdx, %rcx + + mov $7, %rax + mulq $10 + add %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + + mov $6, %rax + mulq $11 + add %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + + mov $5, %rax + mulq $12 + add %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + + cmpq $$0, %rcx + jne 2f + + popcnt $8, %rcx + jrcxz 12f + + popcnt $12, %rcx + popcnt $11, %rax + add %rax, %rcx + popcnt $10, %rax + add %rax, %rcx + jmp 2f + + 12: + popcnt $12, %rcx + jrcxz 11f + + popcnt $7, %rcx + popcnt $6, %rax + add %rax, %rcx + + cmpq $$0, %rcx + jne 2f + + 11: + popcnt $11, %rcx + jrcxz 2f + popcnt $7, %rcx + + 2: + " + : /* $0 */ "={r8}"(result[0]), /* $1 */ "={r9}"(result[1]), /* $2 */ "={r10}"(result[2]), + /* $3 */ "={r11}"(result[3]), /* $4 */ "={rcx}"(overflow) + + : /* $5 */ "m"(self_t[0]), /* $6 */ "m"(self_t[1]), /* $7 */ "m"(self_t[2]), + /* $8 */ "m"(self_t[3]), /* $9 */ "m"(other_t[0]), /* $10 */ "m"(other_t[1]), + /* $11 */ "m"(other_t[2]), /* $12 */ "m"(other_t[3]) + : "rax", "rdx", "rbx" + : + + ); + } + (U256(result), overflow > 0) + }); + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( + uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) + ) +} + +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +macro_rules! uint_overflowing_mul { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) + }) +} + +macro_rules! uint_overflowing_mul_reg { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut res = $name::from(0u64); + let mut overflow = false; + // TODO: be more efficient about this + for i in 0..(2 * $n_words) { + let v = overflowing!($self_expr.overflowing_mul_u32(($other >> (32 * i)).low_u32()), overflow); + let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); + res = overflowing!(res.overflowing_add(res2), overflow); + } + (res, overflow) + }) +} + macro_rules! overflowing { ($op: expr, $overflow: expr) => ( { @@ -297,50 +630,20 @@ macro_rules! construct_uint { (res, overflow) } + /// Optimized instructions + #[inline(always)] fn overflowing_add(self, other: $name) -> ($name, bool) { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - let mut overflow = false; - - for i in 0..$n_words { - ret[i] = me[i].wrapping_add(you[i]); - - if ret[i] < me[i] { - if i < $n_words - 1 { - carry[i + 1] = 1; - b_carry = true; - } else { - overflow = true; - } - } - } - if b_carry { - let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow); - (ret, overflow) - } else { - ($name(ret), overflow) - } + uint_overflowing_add!($name, $n_words, self, other) } + #[inline(always)] fn overflowing_sub(self, other: $name) -> ($name, bool) { - let res = overflowing!((!other).overflowing_add(From::from(1u64))); - let res = overflowing!(self.overflowing_add(res)); - (res, self < other) + uint_overflowing_sub!($name, $n_words, self, other) } + #[inline(always)] fn overflowing_mul(self, other: $name) -> ($name, bool) { - let mut res = $name::from(0u64); - let mut overflow = false; - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - let v = overflowing!(self.overflowing_mul_u32((other >> (32 * i)).low_u32()), overflow); - let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); - res = overflowing!(res.overflowing_add(res2), overflow); - } - (res, overflow) + uint_overflowing_mul!($name, $n_words, self, other) } fn overflowing_div(self, other: $name) -> ($name, bool) { @@ -391,6 +694,7 @@ macro_rules! construct_uint { } impl $name { + #[allow(dead_code)] // not used when multiplied with inline assembly /// Multiplication by u32 fn mul_u32(self, other: u32) -> Self { let $name(ref arr) = self; @@ -412,6 +716,7 @@ macro_rules! construct_uint { $name(ret) + $name(carry) } + #[allow(dead_code)] // not used when multiplied with inline assembly /// Overflowing multiplication by u32 fn overflowing_mul_u32(self, other: u32) -> (Self, bool) { let $name(ref arr) = self; @@ -535,23 +840,9 @@ macro_rules! construct_uint { type Output = $name; fn add(self, other: $name) -> $name { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - for i in 0..$n_words { - if i < $n_words - 1 { - ret[i] = me[i].wrapping_add(you[i]); - if ret[i] < me[i] { - carry[i + 1] = 1; - b_carry = true; - } - } else { - ret[i] = me[i] + you[i]; - } - } - if b_carry { $name(ret) + $name(carry) } else { $name(ret) } + let (result, overflow) = self.overflowing_add(other); + panic_on_overflow!(overflow); + result } } @@ -560,9 +851,9 @@ macro_rules! construct_uint { #[inline] fn sub(self, other: $name) -> $name { - panic_on_overflow!(self < other); - let res = overflowing!((!other).overflowing_add(From::from(1u64))); - overflowing!(self.overflowing_add(res)) + let (result, overflow) = self.overflowing_sub(other); + panic_on_overflow!(overflow); + result } } @@ -570,15 +861,9 @@ macro_rules! construct_uint { type Output = $name; fn mul(self, other: $name) -> $name { - let mut res = $name::from(0u64); - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - let v = self.mul_u32((other >> (32 * i)).low_u32()); - let (r, overflow) = v.overflowing_shl(32 * i as u32); - panic_on_overflow!(overflow); - res = res + r; - } - res + let (result, overflow) = self.overflowing_mul(other); + panic_on_overflow!(overflow); + result } } @@ -1171,8 +1456,6 @@ mod tests { ); } - - #[test] #[should_panic] pub fn uint256_mul_overflow_panic() { @@ -1291,5 +1574,173 @@ mod tests { fn display_uint_zero() { assert_eq!(format!("{}", U256::from(0)), "0"); } + + #[test] + fn u512_multi_adds() { + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); + + let (result, _) = U512([1, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([1, 0, 0, 0, 0, 0, 0, 1])); + assert_eq!(result, U512([2, 0, 0, 0, 0, 0, 0, 2])); + + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 1])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 2])); + + let (result, _) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 5, 2])); + + let (result, _) = U512([1, 2, 3, 4, 5, 6, 7, 8]).overflowing_add(U512([9, 10, 11, 12, 13, 14, 15, 16])); + assert_eq!(result, U512([10, 12, 14, 16, 18, 20, 22, 24])); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); + assert!(!overflow); + + let (_, overflow) = U512([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_add(U512([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX]) + .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX]) + .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert!(!overflow); + } + + #[test] + fn u256_multi_adds() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_add(U256([0, 0, 0, 0])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_add(U256([0, 0, 0, 1])); + assert_eq!(result, U256([0, 0, 0, 2])); + + let (result, overflow) = U256([0, 0, 2, 1]).overflowing_add(U256([0, 0, 3, 1])); + assert_eq!(result, U256([0, 0, 5, 2])); + assert!(!overflow); + + let (_, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_add(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_add(U256([0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + } + + + #[test] + fn u256_multi_subs() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_sub(U256([0, 0, 0, 0])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 0, 1])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (_, overflow) = U256([0, 0, 2, 1]).overflowing_sub(U256([0, 0, 3, 1])); + assert!(overflow); + + let (result, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_sub(U256([::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1]), result); + + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 1, 0])); + assert!(!overflow); + assert_eq!(U256([0, 0, ::std::u64::MAX, 0]), result); + + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([1, 0, 0, 0])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]), result); + } + + + #[test] + fn u512_multi_subs() { + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_sub(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); + + let (result, _) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); + assert_eq!(result, U512([1, 1, 1, 1, 1, 1, 1, 1])); + + let (_, overflow) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); + assert!(!overflow); + + let (_, overflow) = U512([9, 8, 7, 6, 5, 4, 3, 2]).overflowing_sub(U512([10, 9, 8, 7, 6, 5, 4, 3])); + assert!(overflow); + } + + #[test] + fn u256_multi_muls() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([1, 0, 0, 0]), result); + + let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0])); + assert_eq!(U256([25, 0, 0, 0]), result); + + let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 25, 0]), result); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 1]), result); + + let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 10]), result); + + let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 0, 5]), result); + + let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 10, 0, 0]), result); + + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX-1, 0, 0]), result); + + let (result, _) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, 0, 0, 0]), result); + } + + #[test] + fn u256_multi_muls_overflow() { + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(!overflow); + + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 1, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([0, 1, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 1, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([0, ::std::u64::MAX, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([10, 0, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX / 2])); + assert!(!overflow); + + let (_, overflow) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert!(overflow); + } }