logging improvements

This commit is contained in:
cubedro
2015-06-03 05:06:09 +03:00
parent 1d5eed21b5
commit 7bd8d450ae
6 changed files with 183 additions and 30 deletions

227
lib/collection.js Normal file
View File

@@ -0,0 +1,227 @@
var _ = require('lodash');
var Blockchain = require('./history');
var Node = require('./node');
var Collection = function Collection()
{
this._items = [];
this._blockchain = new Blockchain();
return this;
}
Collection.prototype.add = function(data)
{
var node = this.getNodeOrNew({ id : data.id }, data);
node.setInfo(data);
return node.getInfo();
}
Collection.prototype.update = function(id, stats)
{
var node = this.getNode({ id: id });
if (!node)
return false;
var block = this._blockchain.add(stats.block, id);
if (!block)
return false;
var propagationHistory = this._blockchain.getNodePropagation(id);
stats.block.arrived = block.arrived;
stats.block.received = block.received;
stats.block.propagation = block.propagation;
return node.setStats(stats, propagationHistory);
}
Collection.prototype.addBlock = function(id, block)
{
var node = this.getNode({ id: id });
if (!node)
return false;
var block = this._blockchain.add(block, id);
if (!block)
return false;
var propagationHistory = this._blockchain.getNodePropagation(id);
block.arrived = block.arrived;
block.received = block.received;
block.propagation = block.propagation;
return node.setBlock(block, propagationHistory);
}
Collection.prototype.updatePending = function(id, stats)
{
var node = this.getNode({ id: id });
if (!node)
return false;
return node.setPending(stats);
}
Collection.prototype.updateStats = function(id, stats)
{
var node = this.getNode({ id: id });
if (!node)
return false;
return node.setBasicStats(stats);
}
Collection.prototype.addHistory = function(id, blocks)
{
var node = this.getNode({ id: id });
if (!node)
return false;
blocks = blocks.reverse();
for (var i = 0; i <= blocks.length - 1; i++)
{
this._blockchain.add(blocks[i], id);
};
return this.getCharts();
}
Collection.prototype.updateLatency = function(id, latency)
{
var node = this.getNode({ id: id });
if (!node)
return false;
return node.setLatency(latency);
}
Collection.prototype.inactive = function(id)
{
var node = this.getNode({ spark: id });
if (!node)
return false;
node.setState(false);
return node.getStats();
}
Collection.prototype.getIndex = function(search)
{
return _.findIndex(this._items, search);
}
Collection.prototype.getNode = function(search)
{
var index = this.getIndex(search);
if(index >= 0)
return this._items[index];
return false;
}
Collection.prototype.getNodeByIndex = function(index)
{
if(this._items[index])
return this._items[index];
return false;
}
Collection.prototype.getIndexOrNew = function(search, data)
{
var index = this.getIndex(search);
return (index >= 0 ? index : this._items.push(new Node(data)) - 1);
}
Collection.prototype.getNodeOrNew = function(search, data)
{
return this.getNodeByIndex(this.getIndexOrNew(search, data));
}
Collection.prototype.all = function()
{
this.removeOldNodes();
return this._items;
}
Collection.prototype.removeOldNodes = function()
{
var deleteList = []
for(var i = this._items.length - 1; i >= 0; i--)
{
if( this._items[i].isInactiveAndOld() )
{
deleteList.push(i);
}
}
if(deleteList.length > 0)
{
for(var i = 0; i < deleteList.length; i++)
{
this._items.splice(deleteList[i], 1);
}
}
}
Collection.prototype.blockPropagationChart = function()
{
return this._blockchain.getBlockPropagation();
}
Collection.prototype.getUncleCount = function()
{
return this._blockchain.getUncleCount();
}
Collection.prototype.getCharts = function()
{
return this._blockchain.getCharts();
}
Collection.prototype.getHistory = function()
{
return this._blockchain;
}
Collection.prototype.canNodeUpdate = function(id)
{
var node = this.getNode({id: id});
if(!node)
return false;
if(node.canUpdate())
{
var diff = this._blockchain.bestBlockNumber() - node.getBlockNumber();
return (diff <= 0);
}
return false;
}
Collection.prototype.requiresUpdate = function(id)
{
return ( this.canNodeUpdate(id) && this._blockchain.requiresUpdate() );
}
module.exports = Collection;

450
lib/history.js Normal file
View File

@@ -0,0 +1,450 @@
var _ = require('lodash');
var d3 = require('d3');
var MAX_HISTORY = 1000;
var MAX_PEER_PROPAGATION = 36;
var MIN_PROPAGATION_RANGE = 0;
var MAX_PROPAGATION_RANGE = 10000;
var MAX_UNCLES_PER_BIN = 25;
var MAX_BINS = 40;
var History = function History(data)
{
this._items = [];
var item = {
height: 0,
block: {
number: 0,
hash: '0x?',
arrived: 0,
received: 0,
propagation: 0,
difficulty: 0,
gasUsed: 0,
transactions: [],
uncles: []
},
propagTimes: []
};
}
History.prototype.add = function(block, id)
{
if( !_.isUndefined(block) && !_.isUndefined(block.number) && !_.isUndefined(block.uncles) && !_.isUndefined(block.transactions) && !_.isUndefined(block.difficulty) && block.number > 0 )
{
var historyBlock = this.search(block.number);
var now = _.now();
block.arrived = now;
block.received = now;
block.propagation = 0;
if( historyBlock )
{
var propIndex = _.findIndex( historyBlock.propagTimes, { node: id } );
if( propIndex === -1 )
{
block.arrived = historyBlock.block.arrived;
block.received = now;
block.propagation = now - historyBlock.block.received;
historyBlock.propagTimes.push({
node: id,
received: now,
propagation: block.propagation
});
}
else
{
block.arrived = historyBlock.block.arrived;
block.received = historyBlock.propagTimes[propIndex].received;
block.propagation = historyBlock.propagTimes[propIndex].propagation;
if(historyBlock.hash !== block.hash || historyBlock.totalDifficulty !== block.totalDifficulty || historyBlock.transactions.length !== block.transactions.length)
{
index = _.findIndex( this._items, { height: block.number } );
this._items[index].hash = block.hash;
this._items[index].parentHash = block.parentHash;
this._items[index].nonce = block.nonce;
this._items[index].sha3Uncles = block.sha3Uncles;
this._items[index].transactionsRoot = block.transactionsRoot;
this._items[index].stateRoot = block.stateRoot;
this._items[index].miner = block.miner;
this._items[index].difficulty = block.difficulty;
this._items[index].totalDifficulty = block.totalDifficulty;
this._items[index].size = block.size;
this._items[index].extraData = block.extraData;
this._items[index].gasLimit = block.gasLimit;
this._items[index].gasUsed = block.gasUsed;
this._items[index].timestamp = block.timestamp;
this._items[index].transactions = block.transactions;
this._items[index].uncles = block.uncles;
}
}
}
else
{
var prevBlock = this.prevMaxBlock(block.number);
if( prevBlock )
{
block.time = Math.max(block.arrived - prevBlock.block.arrived, 0);
if(block.number < this.bestBlock().height)
block.time = Math.max((block.timestamp - prevBlock.block.timestamp) * 1000, 0);
}
else
{
block.time = 0;
}
var item = {
height: block.number,
block: block,
propagTimes: []
}
if( this._items.length === 0 || block.number >= (this.bestBlockNumber() - MAX_HISTORY + 1) )
{
item.propagTimes.push({
node: id,
received: now,
propagation: block.propagation
});
this._save(item);
}
}
return block;
}
return false;
}
History.prototype._save = function(block)
{
this._items.unshift(block);
this._items = _.sortByOrder( this._items, 'height', false );
if(this._items.length > MAX_HISTORY)
{
this._items.pop();
}
}
History.prototype.search = function(number)
{
var index = _.findIndex( this._items, { height: number } );
if(index < 0)
return false;
return this._items[index];
}
History.prototype.prevMaxBlock = function(number)
{
var index = _.findIndex(this._items, function (item) {
return item.height < number;
});
if(index < 0)
return false;
return this._items[index];
}
History.prototype.bestBlock = function()
{
return _.max(this._items, 'height');
}
History.prototype.bestBlockNumber = function()
{
var best = this.bestBlock();
if( !_.isUndefined(best.height) )
return best.height;
return 0;
}
History.prototype.getNodePropagation = function(id)
{
var propagation = new Array( MAX_PEER_PROPAGATION );
var bestBlock = this.bestBlockNumber();
var lastBlocktime = _.now();
_.fill(propagation, -1);
var sorted = _( this._items )
.sortByOrder( 'height', false )
.slice( 0, MAX_PEER_PROPAGATION )
.forEach(function (item, key)
{
var index = MAX_PEER_PROPAGATION - 1 - bestBlock + item.height;
if(index >= 0)
{
var tmpPropagation = _.result(_.find(item.propagTimes, 'node', id), 'propagation', false);
if (_.result(_.find(item.propagTimes, 'node', id), 'propagation', false) !== false)
{
propagation[index] = tmpPropagation;
lastBlocktime = item.block.arrived;
}
else
{
propagation[index] = Math.max(0, lastBlocktime - item.block.arrived);
}
}
})
.reverse()
.value();
return propagation;
}
History.prototype.getBlockPropagation = function()
{
var propagation = [];
var avgPropagation = 0;
_.forEach(this._items, function (n, key)
{
_.forEach(n.propagTimes, function (p, i)
{
var prop = Math.min(MAX_PROPAGATION_RANGE, _.result(p, 'propagation', -1));
if(prop >= 0)
propagation.push(prop);
});
});
if(propagation.length > 0)
{
var avgPropagation = Math.round( _.sum(propagation) / propagation.length );
}
var data = d3.layout.histogram()
.frequency( false )
.range([ MIN_PROPAGATION_RANGE, MAX_PROPAGATION_RANGE ])
.bins( MAX_BINS )
( propagation );
var freqCum = 0;
var histogram = data.map(function (val) {
freqCum += val.length;
var cumPercent = ( freqCum / Math.max(1, propagation.length) );
return {
x: val.x,
dx: val.dx,
y: val.y,
frequency: val.length,
cumulative: freqCum,
cumpercent: cumPercent
};
});
return {
histogram: histogram,
avg: avgPropagation
};
}
History.prototype.getUncleCount = function()
{
var uncles = _( this._items )
.sortByOrder( 'height', false )
.map(function (item)
{
return item.block.uncles.length;
})
.value();
var uncleBins = _.fill( Array(MAX_BINS), 0 );
var sumMapper = function (array, key)
{
uncleBins[key] = _.sum(array);
return _.sum(array);
};
_.map(_.chunk( uncles, MAX_UNCLES_PER_BIN ), sumMapper);
return uncleBins;
}
History.prototype.getBlockTimes = function()
{
var blockTimes = _( this._items )
.sortByOrder( 'height', false )
.slice(0, MAX_BINS)
.reverse()
.map(function (item)
{
return item.block.time;
})
.value();
return blockTimes;
}
History.prototype.getDifficulty = function()
{
var difficultyHistory = _( this._items )
.sortByOrder( 'height', false )
.slice(0, MAX_BINS)
.reverse()
.map(function (item)
{
return item.block.difficulty;
})
.value();
return difficultyHistory;
}
History.prototype.getTransactionsCount = function()
{
var txCount = _( this._items )
.sortByOrder( 'height', false )
.slice(0, MAX_BINS)
.reverse()
.map(function (item)
{
return item.block.transactions.length;
})
.value();
return txCount;
}
History.prototype.getGasSpending = function()
{
var gasSpending = _( this._items )
.sortByOrder( 'height', false )
.slice(0, MAX_BINS)
.reverse()
.map(function (item)
{
return item.block.gasUsed;
})
.value();
return gasSpending;
}
History.prototype.getAvgHashrate = function()
{
if( _.isEmpty(this._items) )
return 0;
var blocktimeHistory = _( this._items )
.sortByOrder( 'height', false )
.slice(0, 64)
.map(function (item)
{
return item.block.time;
})
.value();
var avgBlocktime = (_.sum(blocktimeHistory) / blocktimeHistory.length)/1000;
return this.bestBlock().block.difficulty / avgBlocktime;
}
History.prototype.getMinersCount = function()
{
var miners = _( this._items )
.sortByOrder( 'height', false )
.slice(0, MAX_BINS)
.map(function (item)
{
return item.block.miner;
})
.value();
var minerCount = [];
_.forEach( _.countBy(miners), function (cnt, miner)
{
minerCount.push({ miner: miner, name: false, blocks: cnt });
});
return _(minerCount)
.sortByOrder( 'blocks', false )
.slice(0, 5)
.value();
}
History.prototype.getCharts = function(callback)
{
var chartHistory = _( this._items )
.sortByOrder( 'height', false )
.slice(0, MAX_BINS)
.reverse()
.map(function (item)
{
return {
height: item.height,
blocktime: item.block.time / 1000,
difficulty: item.block.difficulty,
uncles: item.block.uncles.length,
transactions: item.block.transactions.length,
gasSpending: item.block.gasUsed,
miner: item.block.miner
};
})
.value();
return {
height : _.pluck( chartHistory, 'height' ),
blocktime : _.pluck( chartHistory, 'blocktime' ),
avgBlocktime : _.sum(_.pluck( chartHistory, 'blocktime' )) / (chartHistory.length === 0 ? 1 : chartHistory.length),
difficulty : _.pluck( chartHistory, 'difficulty' ),
uncles : _.pluck( chartHistory, 'uncles' ),
transactions : _.pluck( chartHistory, 'transactions' ),
gasSpending : _.pluck( chartHistory, 'gasSpending' ),
miners : this.getMinersCount(),
propagation : this.getBlockPropagation(),
uncleCount : this.getUncleCount(),
avgHashrate : this.getAvgHashrate()
};
}
History.prototype.requiresUpdate = function()
{
return ( this._items.length < MAX_HISTORY && !_.isEmpty(this._items) );
}
History.prototype.getHistoryRequestRange = function()
{
if( _.isEmpty(this._items) )
return false;
var blocks = _.pluck( this._items, 'height' );
var best = _.max( blocks );
var range = _.range( _.max([ 0, best - MAX_HISTORY ]), best + 1);
var missing = _.difference( range, blocks );
var max = _.max(missing);
var min = max - Math.min( 50, (MAX_HISTORY - this._items.length + 1) ) + 1;
return {
max: max,
min: min,
list: _( missing ).reverse().slice(0, 50).reverse().value()
};
}
module.exports = History;

335
lib/node.js Normal file
View File

@@ -0,0 +1,335 @@
var geoip = require('geoip-lite');
var _ = require('lodash');
var MAX_HISTORY = 40;
var MAX_INACTIVE_TIME = 1000*60*60*4;
var Node = function(data)
{
this.id = null;
this.info = {};
this.geo = {}
this.stats = {
active: false,
mining: false,
hashrate: 0,
peers: 0,
pending: 0,
gasPrice: 0,
block: {
number: 0,
hash: '0x0000000000000000000000000000000000000000000000000000000000000000',
difficulty: 0,
totalDifficulty: 0,
gasLimit: 0,
timestamp: 0,
time: 0,
arrival: 0,
received: 0,
propagation: 0,
transactions: [],
uncles: []
},
propagationAvg: 0,
latency: 0,
uptime: 100
};
this.history = new Array(MAX_HISTORY);
this.uptime = {
started: null,
up: 0,
down: 0,
lastStatus: null,
lastUpdate: null
};
this.init(data);
return this;
}
Node.prototype.init = function(data)
{
_.fill(this.history, -1);
if( this.id === null && this.uptime.started === null )
this.setState(true);
this.id = _.result(data, 'id', this.id);
if( !_.isUndefined(data.latency) )
this.stats.latency = data.latency;
this.setInfo(data);
}
Node.prototype.setInfo = function(data)
{
if( !_.isUndefined(data.info) )
{
this.info = data.info;
if( !_.isUndefined(data.info.canUpdateHistory) )
{
this.info.canUpdateHistory = _.result(data, 'info.canUpdateHistory', false);
}
}
if( !_.isUndefined(data.ip) )
{
this.setGeo(data.ip);
}
this.spark = _.result(data, 'spark', null);
this.setState(true);
}
Node.prototype.setGeo = function(ip)
{
this.info.ip = ip;
this.geo = geoip.lookup(ip);
}
Node.prototype.getInfo = function()
{
return {
id: this.id,
info: this.info,
stats: {
active: this.stats.active,
mining: this.stats.mining,
hashrate: this.stats.hashrate,
peers: this.stats.peers,
gasPrice: this.stats.gasPrice,
block: this.stats.block,
propagationAvg: this.stats.propagationAvg,
uptime: this.stats.uptime,
latency: this.stats.latency,
pending: this.stats.pending,
},
history: this.history,
geo: this.geo
};
}
Node.prototype.setStats = function(stats, history)
{
if( !_.isUndefined(stats) )
{
this.setBlock( _.result(stats, 'block', this.stats.block), history );
this.setBasicStats(stats);
this.setPending( _.result(stats, 'pending', this.stats.pending) );
return this.getStats();
}
return false;
}
Node.prototype.setBlock = function(block, history)
{
if( !_.isUndefined(block) && !_.isUndefined(block.number) && ( !_.isEqual(history, this.history) || !_.isEqual(block, this.stats.block) ))
{
if(block.number !== this.stats.block.number && block.hash !== this.stats.block.hash)
{
this.stats.block = block;
}
this.setHistory(history);
return this.getBlockStats();
}
return false;
}
Node.prototype.setHistory = function(history)
{
if( _.isEqual(history, this.history) )
{
return false;
}
if( !_.isArray(history) )
{
this.history = _.fill( new Array(MAX_HISTORY), -1 );
this.stats.propagationAvg = 0;
return true;
}
this.history = history;
var positives = _.filter(history, function(p) {
return p >= 0;
});
this.stats.propagationAvg = ( positives.length > 0 ? Math.round( _.sum(positives) / positives.length ) : 0 );
positives = null;
return true;
}
Node.prototype.setPending = function(stats)
{
if( !_.isUndefined(stats) && !_.isUndefined(stats.pending) && !_.isEqual(stats.pending, this.stats.pending))
{
this.stats.pending = stats.pending;
return {
id: this.id,
pending: this.stats.pending
};
}
return false;
}
Node.prototype.setBasicStats = function(stats)
{
if( !_.isUndefined(stats) && !_.isEqual(stats, {
active: this.stats.active,
mining: this.stats.mining,
hashrate: this.stats.hashrate,
peers: this.stats.peers,
gasPrice: this.stats.gasPrice,
uptime: this.stats.uptime
}) )
{
this.stats.active = stats.active;
this.stats.mining = stats.mining;
this.stats.hashrate = stats.hashrate;
this.stats.peers = stats.peers;
this.stats.gasPrice = stats.gasPrice;
this.stats.uptime = stats.uptime;
return this.getBasicStats();
}
return false;
}
Node.prototype.setLatency = function(latency)
{
if( !_.isUndefined(latency) && !_.isEqual(latency, this.stats.latency) )
{
this.stats.latency = latency;
return {
id: this.id,
latency: latency
};
}
return false;
}
Node.prototype.getStats = function()
{
return {
id: this.id,
stats: {
active: this.stats.active,
mining: this.stats.mining,
hashrate: this.stats.hashrate,
peers: this.stats.peers,
gasPrice: this.stats.gasPrice,
block: this.stats.block,
propagationAvg: this.stats.propagationAvg,
uptime: this.stats.uptime,
pending: this.stats.pending
},
history: this.history
};
}
Node.prototype.getBlockStats = function()
{
return {
id: this.id,
block: this.stats.block,
propagationAvg: this.stats.propagationAvg,
history: this.history
};
}
Node.prototype.getBasicStats = function()
{
return {
id: this.id,
stats: {
active: this.stats.active,
mining: this.stats.mining,
hashrate: this.stats.hashrate,
peers: this.stats.peers,
gasPrice: this.stats.gasPrice,
uptime: this.stats.uptime
}
};
}
Node.prototype.setState = function(active)
{
var now = _.now();
if(this.uptime.started !== null)
{
if(this.uptime.lastStatus === active)
{
this.uptime[(active ? 'up' : 'down')] += now - this.uptime.lastUpdate;
}
else
{
this.uptime[(active ? 'down' : 'up')] += now - this.uptime.lastUpdate;
}
}
else
{
this.uptime.started = now;
}
this.stats.active = active;
this.uptime.lastStatus = active;
this.uptime.lastUpdate = now;
this.stats.uptime = this.calculateUptime();
now = undefined;
}
Node.prototype.calculateUptime = function()
{
if(this.uptime.lastUpdate === this.uptime.started)
{
return 100;
}
return Math.round( this.uptime.up / (this.uptime.lastUpdate - this.uptime.started) * 100);
}
Node.prototype.getBlockNumber = function()
{
return this.stats.block.number;
}
Node.prototype.canUpdate = function()
{
return this.info.canUpdateHistory || false;
}
Node.prototype.isInactiveAndOld = function()
{
if( this.uptime.lastStatus === false && this.uptime.lastUpdate !== null && (_.now() - this.uptime.lastUpdate) > MAX_INACTIVE_TIME )
{
return true;
}
return false;
}
module.exports = Node;

150
lib/utils/logger.js Normal file
View File

@@ -0,0 +1,150 @@
'use strict';
var util = require('util');
var chalk = require('chalk');
var sections = [
'API',
'COL',
'SYS'
];
var types = [
'CON',
'CHR',
'UDP',
'BLK',
'TXS',
'STA',
'HIS',
'PIN'
];
var typeColors = {
'CON': chalk.reset.bold.yellow,
'CHR': chalk.reset.bold.red,
'UDP': chalk.reset.bold.green,
'BLK': chalk.reset.bold.blue,
'TXS': chalk.reset.bold.cyan,
'STA': chalk.reset.bold.red,
'HIS': chalk.reset.bold.magenta,
'PIN': chalk.reset.bold.yellow,
};
[
{
name: "info",
sign: '=i=',
signColor: chalk.blue,
messageColor: chalk.bold,
formatter: function (sign, message)
{
return [sign, message];
}
},
{
name: "success",
inherit: 'log',
sign: '=✓=',
signColor: chalk.green,
messageColor: chalk.bold.green,
formatter: function (sign, message)
{
return [sign, message];
}
},
{
name: "warn",
sign: '=!=',
signColor: chalk.yellow,
messageColor: chalk.bold.yellow,
formatter: function (sign, message)
{
return [sign, message];
}
},
{
name: "error",
sign: '=✘=',
signColor: chalk.red,
messageColor: chalk.bold.red,
formatter: function (sign, message)
{
return [sign, message];
}
},
{
name: "time",
sign: '=T=',
signColor: chalk.cyan,
messageColor: chalk.bold,
formatter: function (sign, message)
{
return [util.format.apply(util, [sign, message])];
}
},
{
name: "timeEnd",
sign: '=T=',
signColor: chalk.cyan,
messageColor: chalk.bold,
formatter: function (sign, message)
{
return [util.format.apply(util, [sign, message])];
}
}
].forEach( function (item)
{
if(item.inherit !== undefined)
console[item.name] = console[item.inherit];
var fn = console[item.name];
console[item.name] = function ()
{
var args = Array.prototype.slice.call(arguments);
var type,
sign,
time;
var section = 'API';
var message = '';
if (args[0].indexOf(new Date().getFullYear()) >= 0)
{
time = args.splice(0, 1);
}
if (sections.indexOf(args[0]) >= 0)
{
section = args.splice(0, 1);
}
if (types.indexOf(args[0]) >= 0)
{
type = args.splice(0, 1);
}
sign = item.signColor.bold( '[' ) + chalk.reset.bold.white( section ) + item.signColor.bold( ']' ) + " " + item.signColor.bold( '[' ) + typeColors[type](type) + item.signColor.bold( ']' );
if(item.name !== "time" && item.name !== "timeEnd")
{
time = (new Date()).toJSON().replace("T", " ").replace("Z", " ");
}
else
{
time = time.toString().replace("T", " ").replace("Z", "");
}
sign = chalk.reset.magenta(time) + sign;
if (typeof args[0] === 'object')
{
message = util.inspect( args[0], { depth: null, colors: true } );
}
else {
message = item.messageColor( util.format.apply(util, args) );
}
return fn.apply( this, item.formatter(sign, message) );
}
});