var _ = require('lodash'); var d3 = require('d3'); var MAX_HISTORY = 2000; var MAX_PEER_PROPAGATION = 40; var MIN_PROPAGATION_RANGE = 0; var MAX_PROPAGATION_RANGE = 10000; var MAX_UNCLES = 1000; var MAX_UNCLES_PER_BIN = 25; var MAX_BINS = 40; var History = function History(data) { this._items = []; this._callback = null; } History.prototype.add = function(block, id, trusted, addingHistory) { var changed = false; if( !_.isUndefined(block) && !_.isUndefined(block.number) && !_.isUndefined(block.uncles) && !_.isUndefined(block.transactions) && !_.isUndefined(block.difficulty) && block.number > 0 ) { trusted = (process.env.LITE === 'true' ? true : trusted); var historyBlock = this.search(block.number); var forkIndex = -1; var now = _.now(); block.trusted = trusted; block.arrived = now; block.received = now; block.propagation = 0; block.fork = 0; if( historyBlock ) { // We already have a block with this height in collection // Check if node already checked this block height var propIndex = _.findIndex( historyBlock.propagTimes, { node: id } ); // Check if node already check a fork with this height forkIndex = compareForks(historyBlock, block); if( propIndex === -1 ) { // Node didn't submit this block before if( forkIndex >= 0 && !_.isUndefined(historyBlock.forks[forkIndex]) ) { // Found fork => update data block.arrived = historyBlock.forks[forkIndex].arrived; block.propagation = now - historyBlock.forks[forkIndex].received; } else { // No fork found => add a new one var prevBlock = this.prevMaxBlock(block.number); if( prevBlock ) { block.time = Math.max(block.arrived - prevBlock.block.arrived, 0); if(block.number < this.bestBlock().height) block.time = Math.max((block.timestamp - prevBlock.block.timestamp) * 1000, 0); } else { block.time = 0; } forkIndex = historyBlock.forks.push(block) - 1; historyBlock.forks[forkIndex].fork = forkIndex; } // Push propagation time historyBlock.propagTimes.push({ node: id, trusted: trusted, fork: forkIndex, received: now, propagation: block.propagation }); } else { // Node submited the block before if( forkIndex >= 0 && !_.isUndefined(historyBlock.forks[forkIndex]) ) { // Matching fork found => update data block.arrived = historyBlock.forks[forkIndex].arrived; if( forkIndex === historyBlock.propagTimes[propIndex].fork ) { // Fork index is the same block.received = historyBlock.propagTimes[propIndex].received; block.propagation = historyBlock.propagTimes[propIndex].propagation; } else { // Fork index is different historyBlock.propagTimes[propIndex].fork = forkIndex; historyBlock.propagTimes[propIndex].propagation = block.propagation = now - historyBlock.forks[forkIndex].received; } } else { // No matching fork found => replace old one block.received = historyBlock.propagTimes[propIndex].received; block.propagation = historyBlock.propagTimes[propIndex].propagation; var prevBlock = this.prevMaxBlock(block.number); if( prevBlock ) { block.time = Math.max(block.arrived - prevBlock.block.arrived, 0); if(block.number < this.bestBlock().height) block.time = Math.max((block.timestamp - prevBlock.block.timestamp) * 1000, 0); } else { block.time = 0; } forkIndex = historyBlock.forks.push(block) - 1; historyBlock.forks[forkIndex].fork = forkIndex; } } if( trusted && !compareBlocks(historyBlock.block, historyBlock.forks[forkIndex]) ) { // If source is trusted update the main block historyBlock.forks[forkIndex].trusted = trusted; historyBlock.block = historyBlock.forks[forkIndex]; } block.fork = forkIndex; changed = true; } else { // Couldn't find block with this height // Getting previous max block var prevBlock = this.prevMaxBlock(block.number); if( prevBlock ) { block.time = Math.max(block.arrived - prevBlock.block.arrived, 0); if(block.number < this.bestBlock().height) block.time = Math.max((block.timestamp - prevBlock.block.timestamp) * 1000, 0); } else { block.time = 0; } var item = { height: block.number, block: block, forks: [block], propagTimes: [] } if( this._items.length === 0 || (this._items.length > 0 && block.number > this.worstBlockNumber()) || (this._items.length < MAX_HISTORY && block.number < this.bestBlockNumber() && addingHistory) ) { item.propagTimes.push({ node: id, trusted: trusted, fork: 0, received: now, propagation: block.propagation }); this._save(item); changed = true; } } return { block: block, changed: changed }; } return false; } function compareBlocks(block1, block2) { if( block1.hash !== block2.hash || block1.parentHash !== block2.parentHash || block1.sha3Uncles !== block2.sha3Uncles || block1.transactionsRoot !== block2.transactionsRoot || block1.stateRoot !== block2.stateRoot || block1.miner !== block2.miner || block1.difficulty !== block2.difficulty || block1.totalDifficulty !== block2.totalDifficulty) return false; return true; } function compareForks(historyBlock, block2) { if( _.isUndefined(historyBlock) ) return -1; if( _.isUndefined(historyBlock.forks) || historyBlock.forks.length === 0 ) return -1; for(var x = 0; x < historyBlock.forks.length; x++) if(compareBlocks(historyBlock.forks[x], block2)) return x; return -1; } History.prototype._save = function(block) { this._items.unshift(block); this._items = _.orderBy( this._items, 'height', "desc" ); if(this._items.length > MAX_HISTORY) { this._items.pop(); } } History.prototype.clean = function(max) { if(max > 0 && this._items.length > 0 && max < this.bestBlockNumber()) { console.log("MAX:", max); console.log("History items before:", this._items.length); this._items = _(this._items).filter(function(item) { return (item.height <= max && item.block.trusted === false); }).value(); console.log("History items after:", this._items.length); } } History.prototype.search = function(number) { var index = _.findIndex( this._items, { height: number } ); if(index < 0) return false; return this._items[index]; } History.prototype.prevMaxBlock = function(number) { var index = _.findIndex(this._items, function (item) { return item.height < number; }); if(index < 0) return false; return this._items[index]; } History.prototype.bestBlock = function() { return _.maxBy(this._items, 'height'); } History.prototype.bestBlockNumber = function() { var best = this.bestBlock(); if( !_.isUndefined(best) && !_.isUndefined(best.height) ) return best.height; return 0; } History.prototype.worstBlock = function() { return _.minBy(this._items, 'height'); } History.prototype.worstBlockNumber = function(trusted) { var worst = this.worstBlock(); if( !_.isUndefined(worst) && !_.isUndefined(worst.height) ) return worst.height; return 0; } History.prototype.getNodePropagation = function(id) { var propagation = new Array( MAX_PEER_PROPAGATION ); var bestBlock = this.bestBlockNumber(); var lastBlocktime = _.now(); _.fill(propagation, -1); var sorted = _( this._items ) .sortBy( 'height', false ) .slice( 0, MAX_PEER_PROPAGATION ) .forEach(function (item, key) { var index = MAX_PEER_PROPAGATION - 1 - bestBlock + item.height; if(index >= 0) { var tmpPropagation = _.result(_.find(item.propagTimes, 'node', id), 'propagation', false); if (_.result(_.find(item.propagTimes, 'node', id), 'propagation', false) !== false) { propagation[index] = tmpPropagation; lastBlocktime = item.block.arrived; } else { propagation[index] = Math.max(0, lastBlocktime - item.block.arrived); } } }) .reverse(); return propagation; } History.prototype.getBlockPropagation = function() { var propagation = []; var avgPropagation = 0; _.forEach(this._items, function (n, key) { _.forEach(n.propagTimes, function (p, i) { var prop = Math.min(MAX_PROPAGATION_RANGE, _.result(p, 'propagation', -1)); if(prop >= 0) propagation.push(prop); }); }); if(propagation.length > 0) { var avgPropagation = Math.round( _.sum(propagation) / propagation.length ); } var data = d3.layout.histogram() .frequency( false ) .range([ MIN_PROPAGATION_RANGE, MAX_PROPAGATION_RANGE ]) .bins( MAX_BINS ) ( propagation ); var freqCum = 0; var histogram = data.map(function (val) { freqCum += val.length; var cumPercent = ( freqCum / Math.max(1, propagation.length) ); return { x: val.x, dx: val.dx, y: val.y, frequency: val.length, cumulative: freqCum, cumpercent: cumPercent }; }); return { histogram: histogram, avg: avgPropagation }; } History.prototype.getUncleCount = function() { var uncles = _( this._items ) .sortBy( 'height', false ) // .filter(function (item) // { // return item.block.trusted; // }) .slice(0, MAX_UNCLES) .map(function (item) { return item.block.uncles.length; }) .value(); var uncleBins = _.fill( Array(MAX_BINS), 0 ); var sumMapper = function (array, key) { uncleBins[key] = _.sum(array); return _.sum(array); }; _.map(_.chunk( uncles, MAX_UNCLES_PER_BIN ), sumMapper); return uncleBins; } History.prototype.getBlockTimes = function() { var blockTimes = _( this._items ) .sortBy( 'height', false ) // .filter(function (item) // { // return item.block.trusted; // }) .slice(0, MAX_BINS) .reverse() .map(function (item) { return item.block.time / 1000; }) .value(); return blockTimes; } History.prototype.getAvgBlocktime = function() { var blockTimes = _( this._items ) .sortBy( 'height', false ) // .filter(function (item) // { // return item.block.trusted; // }) // .slice(0, MAX_BINS) .reverse() .map(function (item) { return item.block.time / 1000; }) .value(); return _.sum(blockTimes) / (blockTimes.length === 0 ? 1 : blockTimes.length); } History.prototype.getGasLimit = function() { var gasLimitHistory = _( this._items ) .sortBy( 'height', false ) // .filter(function (item) // { // return item.block.trusted; // }) .slice(0, MAX_BINS) .reverse() .map(function (item) { return item.block.gasLimit; }) .value(); return gasLimitHistory; } History.prototype.getDifficulty = function() { var difficultyHistory = _( this._items ) .sortBy( 'height', false ) .filter(function (item) { return item.block.trusted; }) .slice(0, MAX_BINS) .reverse() .map(function (item) { return item.block.difficulty; }) .value(); return difficultyHistory; } History.prototype.getTransactionsCount = function() { var txCount = _( this._items ) .sortBy( 'height', false ) .filter(function (item) { return item.block.trusted; }) .slice(0, MAX_BINS) .reverse() .map(function (item) { return item.block.transactions.length; }) .value(); return txCount; } History.prototype.getGasSpending = function() { var gasSpending = _( this._items ) .sortBy( 'height', false ) .filter(function (item) { return item.block.trusted; }) .slice(0, MAX_BINS) .reverse() .map(function (item) { return item.block.gasUsed; }) .value(); return gasSpending; } History.prototype.getAvgHashrate = function() { if( _.isEmpty(this._items) ) return 0; var blocktimeHistory = _( this._items ) .sortBy( 'height', false ) // .filter(function (item) // { // return item.block.trusted; // }) .slice(0, 64) .map(function (item) { return item.block.time; }) .value(); var avgBlocktime = (_.sum(blocktimeHistory) / blocktimeHistory.length)/1000; return this.bestBlock().block.difficulty / avgBlocktime; } History.prototype.getMinersCount = function() { var miners = _( this._items ) .sortBy( 'height', false ) // .filter(function (item) // { // return item.block.trusted; // }) .slice(0, MAX_BINS) .map(function (item) { return item.block.miner; }) .value(); var minerCount = []; _.forEach( _.countBy(miners), function (cnt, miner) { minerCount.push({ miner: miner, name: false, blocks: cnt }); }); return _(minerCount) .sortBy( 'blocks', false ) .slice(0, 2) .value(); } History.prototype.setCallback = function(callback) { this._callback = callback; } History.prototype.getCharts = function() { if(this._callback !== null) { var chartHistory = _( this._items ) .orderBy( 'height', "desc" ) // .filter(function (item) // { // return item.block.trusted; // }) .slice(0, MAX_BINS) .reverse() .map(function (item) { return { height: item.height, blocktime: item.block.time / 1000, difficulty: item.block.difficulty, uncles: item.block.uncles.length, transactions: item.block.transactions ? item.block.transactions.length : 0, gasSpending: item.block.gasUsed, gasLimit: item.block.gasLimit, miner: item.block.miner }; }) .value(); this._callback(null, { height : _.map( chartHistory, 'height' ), blocktime : _.map( chartHistory, 'blocktime' ), // avgBlocktime : _.sum(_.map( chartHistory, 'blocktime' )) / (chartHistory.length === 0 ? 1 : chartHistory.length), avgBlocktime : this.getAvgBlocktime(), difficulty : _.map( chartHistory, 'difficulty' ), uncles : _.map( chartHistory, 'uncles' ), transactions : _.map( chartHistory, 'transactions' ), gasSpending : _.map( chartHistory, 'gasSpending' ), gasLimit : _.map( chartHistory, 'gasLimit' ), miners : this.getMinersCount(), propagation : this.getBlockPropagation(), uncleCount : this.getUncleCount(), avgHashrate : this.getAvgHashrate() }); } } History.prototype.requiresUpdate = function() { // return ( this._items.length < MAX_HISTORY && !_.isEmpty(this._items) ); return ( this._items.length < MAX_HISTORY ); } History.prototype.getHistoryRequestRange = function() { if( this._items.length < 2 ) return false; var blocks = _.map( this._items, 'height' ); var best = _.max( blocks ); var range = _.range( _.max([ 0, best - MAX_HISTORY ]), best + 1); var missing = _.difference( range, blocks ); var max = _.max(missing); var min = max - Math.min( 50, (MAX_HISTORY - this._items.length + 1) ) + 1; return { max: max, min: min, list: _( missing ).reverse().slice(0, 50).reverse().value() }; } module.exports = History;