From 28821abc89d42f1adbe1325d435530da96c1e3c9 Mon Sep 17 00:00:00 2001 From: cubedro Date: Mon, 4 May 2015 23:16:02 +0300 Subject: [PATCH 1/4] made stats calls async --- lib/node.js | 326 +++++++++++++++++++++++++++++++++++---------------- package.json | 1 + 2 files changed, 225 insertions(+), 102 deletions(-) diff --git a/lib/node.js b/lib/node.js index bff0496..7725cfd 100644 --- a/lib/node.js +++ b/lib/node.js @@ -1,6 +1,7 @@ 'use strict'; var web3 = require('web3'); +var async = require('async'); var _ = require('lodash'); var os = require('os'); var shelljs = require('shelljs'); @@ -97,7 +98,7 @@ function Node() this.pingInterval = false; this.connectionInterval = false; - this._lastLatestLog = 0; + this._lastChainLog = 0; this._lastPendingLog = 0; this._called = 0 @@ -293,11 +294,15 @@ Node.prototype.getBlock = function(number) try { block = web3.eth.getBlock(number, true); + console.log('====> block:', block); - if( block.hash != '?' && !_.isUndefined(block.difficulty) ) + if( block.hash != '?' && !_.isUndefined(block.difficulty) && !_.isUndefined(block.totalDifficulty) ) { block.difficulty = web3.toDecimal( block.difficulty ); + block.totalDifficulty = web3.toDecimal( block.totalDifficulty ); } + + console.log('====> block:', block); } catch (err) { console.error("getBlock(" + number + "):", err); @@ -309,9 +314,10 @@ Node.prototype.getBlock = function(number) block = web3.eth.getBlock(number, true); - if(block.hash !== '?' && !_.isUndefined(block.difficulty) ) + if( block.hash !== '?' && !_.isUndefined(block.difficulty) && !_.isUndefined(block.totalDifficulty) ) { block.difficulty = web3.toDecimal( block.difficulty ); + block.totalDifficulty = web3.toDecimal( block.totalDifficulty ); } } catch (err) { @@ -357,72 +363,171 @@ Node.prototype.uptime = function() Node.prototype.getStats = function() { - var start = _.now(); - console.log('==> statsstart: ', start); + var self = this; + if(this._socket) this._lastStats = JSON.stringify(this.stats); if(this._web3) { - var blockStart = _.now(); - console.log('==> blockstart: ', blockStart); - console.log('==> blockstart diff: ', (blockStart - start)); + async.parallel({ + start: function (callback) + { + callback(null, _.now()); + }, + peers: function (callback) + { + setTimeout(function () { + var peers; + + try { + peers = web3.toDecimal(web3.net.peerCount); + } + catch (err) { + console.error('xx> PeerCount failed: ', err); + callback(err, null); + } + + callback(null, peers); + }, 1); + }, + pending: function (callback) + { + setTimeout(function() { + try { + web3.eth.getBlockTransactionCount('pending', callback); + } + catch (err) { + console.error('xx> Pending failed: ', err); + callback(err, null); + } + }, 1); + }, + mining: function (callback) + { + setTimeout(function () { + var mining; + + try { + mining = web3.eth.mining; + } + catch (err) { + console.error('xx> Mining failed: ', err); + callback(err, null); + } + + callback(null, mining); + }, 1); + }, + hashrate: function (callback) + { + setTimeout(function () { + var hashrate; + + try { + hashrate = web3.eth.hashrate; + } + catch (err) { + console.error('xx> Hashrate failed: ', err); + callback(err, null); + } + + callback(null, hashrate); + }, 1); + }, + gasprice: function (callback) + { + setTimeout(function () { + var gasPrice; + + try { + gasPrice = web3.toBigNumber(web3.eth.gasPrice).toString(10); + } + catch (err) { + console.error('xx> gasPrice failed: ', err); + callback(err, null); + } + + callback(null, gasPrice); + }, 1); + } + }, + function (err, results) + { + self._tries++; + + if (err) { + console.error('xx> getStats error: ', err); + + self.stats.peers = 0; + self.stats.active = false; + self.stats.pending = 0; + self.stats.mining = false; + self.stats.hashrate = 0; + self.stats.gasPrice = 0; + self._down++; + + return false; + } + + results.end = _.now(); + results.diff = results.end - results.start; + + // console.log('==> Got getStats results: ', results); + console.log('==> Got getStats results in', results.diff, 'ms'); + + if(results.peers !== null) + { + self.stats.peers = results.peers; + self.stats.active = true; + self.stats.pending = results.pending; + self.stats.mining = results.mining; + self.stats.hashrate = results.hashrate; + self.stats.gasPrice = results.gasPrice; + } + else { + self.stats.peers = 0; + self.stats.active = false; + self.stats.pending = 0; + self.stats.mining = false; + self.stats.hashrate = 0; + self.stats.gasPrice = 0; + self._down++; + } + + self.uptime(); + + self.sendUpdate(); + }); + } +} + +Node.prototype.getStatsBlock = function () +{ + if(this._socket) + this._lastStats = JSON.stringify(this.stats); + + if(this._web3) + { + console.log("==> Getting block"); var block = this.getBlock(); - var blockEnd = _.now(); - console.log('==> blockend: ', blockEnd); - console.log('==> block diff: ', (blockEnd - blockStart)); if( !_.isUndefined(block) && !_.isUndefined(block.number) && !_.isUndefined(block.hash) && block.hash !== '?' ) { + console.log("==> Got block:", block.number); this.stats.block = block; - this.sendUpdate(); - - this.isActive(); - - if(PENDING_WORKS) { - try - { - this.stats.pending = web3.eth.getBlockTransactionCount('pending'); - } - catch (err) - { - PENDING_WORKS = false; - console.error("getBlockTransactionCount('pending'):", err); - } - } - - this.stats.mining = web3.eth.mining; - - if(this.stats.mining) - { - try { - this.stats.hashrate = web3.eth.hashrate; - } - catch (err) - { - console.error('hashrate: ', err); - this.stats.hashrate = 0; - } - } - else - this.stats.hashrate = 0; - - this.stats.gasPrice = web3.toBigNumber(web3.eth.gasPrice).toString(10); } else { - console.error("getStats: couldn't fetch block..."); + console.error("xx> getStatsBlock: couldn't fetch block..."); } } this.uptime(); - - var end = _.now(); - console.log('==> end: ', end); - console.log('==> stats diff: ', (end - start)); + this.sendUpdate(); } -Node.prototype.getHistory = function(range) +Node.prototype.getHistory = function (range) { var history = []; var interv = {}; @@ -443,7 +548,7 @@ Node.prototype.getHistory = function(range) }; } - for(var i = interv.min; i <= interv.max; i++) + for (var i = interv.min; i <= interv.max; i++) { var block = this.getBlock(( !_.isUndefined(range.list) ? range.list[i] : i)); @@ -456,21 +561,32 @@ Node.prototype.getHistory = function(range) return history.reverse(); } -Node.prototype.changed = function() +Node.prototype.updateBlock = function() +{ + console.log('==> Getting block stats'); + + this.getStatsBlock(); + + return this; +}; + +Node.prototype.update = function() +{ + console.log('==> Getting stats'); + + this.getStats(); + + return this; +}; + +Node.prototype.changed = function () { var changed = ! _.isEqual( this._lastStats, JSON.stringify(this.stats) ); - if(this._tries - this._lastSent > 5) - { - this._lastSent = this._tries; - - return true; - } - return changed; } -Node.prototype.prepareStats = function() +Node.prototype.prepareStats = function () { return { id: this.id, @@ -478,36 +594,12 @@ Node.prototype.prepareStats = function() }; } -Node.prototype.sendUpdate = function(force) +Node.prototype.sendUpdate = function (force) { if( this.changed() || force ) this.emit('update', this.prepareStats()); } -Node.prototype.update = function() -{ - this.getStats(); - - this.sendUpdate(); - - return this.stats; -}; - -Node.prototype.updatePending = function() -{ - if(PENDING_WORKS) - { - try { - this.stats.pending = web3.eth.getBlockTransactionCount('pending'); - this.sendUpdate(); - } - catch (err) { - PENDING_WORKS = false; - console.error("getBlockTransactionCount('pending'):", err); - } - } -} - Node.prototype.ping = function() { this._latency = _.now(); @@ -519,34 +611,63 @@ Node.prototype.setWatches = function() var self = this; try { - this.pendingFilter = web3.eth.filter('chain'); - this.pendingFilter.watch( function (log) + this.chainFilter = web3.eth.filter('chain'); + this.chainFilter.watch( function (log) { - if(PENDING_WORKS) { - var now = _.now(); - var time = now - self._lastPendingLog; + var now = _.now(); + var time = now - self._lastChainLog; + self._lastChainLog = now; - if(time > 50) - { - self.update(); - } - else - { - debounce(function() { - self.updatePending(); - }, 50); - } + console.log('>>> Chain Filter triggered: ', now); + console.log('>>> Last chain Filter triggered: ', time); - self._lastPendingLog = now; + if(time > 50) + { + self.updateBlock(); + } + else + { + debounce(function() { + self.updateBlock(); + }, 50); } }); } catch (err) { - console.error("Couldn't set up pending filter"); + console.error("Couldn't set up chain filter"); console.error(err); } + // try { + // this.pendingFilter = web3.eth.filter('pending'); + // this.pendingFilter.watch( function (log) + // { + // var now = _.now(); + // var time = now - self._lastPendingLog; + + // console.log('>>> Pending Filter triggered: ', now); + + // if(time > 50) + // { + // self.update(); + // } + // else + // { + // debounce(function() { + // self.update(); + // }, 50); + // } + + // self._lastPendingLog = now; + // }); + // } + // catch (err) + // { + // console.error("Couldn't set up pending filter"); + // console.error(err); + // } + this.updateInterval = setInterval( function(){ self.update(); }, UPDATE_INTERVAL); @@ -575,6 +696,7 @@ Node.prototype.init = function() this.startSocketConnection(); this.installContract(); this.setWatches(); + this.updateBlock(); this.update(); } diff --git a/package.json b/package.json index 9478efa..9d630b2 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "lib": "./lib" }, "dependencies": { + "async": "^0.9.0", "debounce": "1.0.0", "debug": "2.1.3", "lodash": "3.7.0", From 76b4870c90260fbd7913e18f213cef76fd4b983d Mon Sep 17 00:00:00 2001 From: cubedro Date: Tue, 5 May 2015 00:51:16 +0300 Subject: [PATCH 2/4] improve d --- lib/node.js | 157 +++++++++++++++++++++++----------------------------- 1 file changed, 69 insertions(+), 88 deletions(-) diff --git a/lib/node.js b/lib/node.js index 7725cfd..9654e9d 100644 --- a/lib/node.js +++ b/lib/node.js @@ -79,6 +79,7 @@ function Node() }; this._lastStats = JSON.stringify(this.stats); + this._lastFetch = 0; this._tries = 0; this._down = 0; @@ -232,32 +233,18 @@ Node.prototype.emit = function(message, payload) } } -Node.prototype.isActive = function() +Node.prototype.setInactive = function() { - this._tries++; - - try { - var peers = web3.toDecimal(web3.net.peerCount); - - if(peers !== null) - { - this.stats.peers = peers; - this.stats.active = true; - - return true; - } - } - catch (err) { - console.error("peerCount:", err); - } - this.stats.active = false; - this.stats.listening = false; - this.stats.mining = false; this.stats.peers = 0; + this.stats.pending = 0; + this.stats.mining = false; + this.stats.hashrate = 0; + this.stats.gasPrice = 0; + this.stats.miner = false; this._down++; - return false; + return this; } Node.prototype.getInfo = function() @@ -294,36 +281,17 @@ Node.prototype.getBlock = function(number) try { block = web3.eth.getBlock(number, true); - console.log('====> block:', block); if( block.hash != '?' && !_.isUndefined(block.difficulty) && !_.isUndefined(block.totalDifficulty) ) { block.difficulty = web3.toDecimal( block.difficulty ); block.totalDifficulty = web3.toDecimal( block.totalDifficulty ); } - - console.log('====> block:', block); } catch (err) { console.error("getBlock(" + number + "):", err); - if(number > 0) - { - try { - number--; - - block = web3.eth.getBlock(number, true); - - if( block.hash !== '?' && !_.isUndefined(block.difficulty) && !_.isUndefined(block.totalDifficulty) ) - { - block.difficulty = web3.toDecimal( block.difficulty ); - block.totalDifficulty = web3.toDecimal( block.totalDifficulty ); - } - } - catch (err) { - console.error("getBlock(" + number + "):", err); - } - } + return false; } return block; @@ -331,26 +299,26 @@ Node.prototype.getBlock = function(number) Node.prototype.getMinerName = function(miner) { - var result = _.find(this._knownMiners, {miner: miner}); + var result = _.find(this._knownMiners, { miner: miner }); - if(result !== undefined) + if (result !== undefined) { return result.name; } else { - if(this._Registrar !== null) + if (this._Registrar !== null) { var name = this._Registrar.name(miner); if(name.length > 0) { - this._knownMiners.push({miner: miner, name: name}); + this._knownMiners.push({ miner: miner, name: name }); return name; } } - this._knownMiners.push({miner: miner, name: false}); + this._knownMiners.push({ miner: miner, name: false }); } return false; @@ -361,15 +329,20 @@ Node.prototype.uptime = function() this.stats.uptime = ((this._tries - this._down) / this._tries) * 100; } -Node.prototype.getStats = function() +Node.prototype.getStats = function(forced) { var self = this; + var now = _.now(); + var lastFetchAgo = now - this._lastFetch; + this._lastFetch = now; - if(this._socket) + if (this._socket) this._lastStats = JSON.stringify(this.stats); - if(this._web3) + if (this._web3 && (lastFetchAgo >= UPDATE_INTERVAL || forced === true)) { + console.log('==> Getting stats; last update:', lastFetchAgo, '- forced:', (forced === true)); + async.parallel({ start: function (callback) { @@ -377,7 +350,7 @@ Node.prototype.getStats = function() }, peers: function (callback) { - setTimeout(function () { + async.nextTick(function () { var peers; try { @@ -389,11 +362,11 @@ Node.prototype.getStats = function() } callback(null, peers); - }, 1); + }); }, pending: function (callback) { - setTimeout(function() { + async.nextTick(function () { try { web3.eth.getBlockTransactionCount('pending', callback); } @@ -401,11 +374,11 @@ Node.prototype.getStats = function() console.error('xx> Pending failed: ', err); callback(err, null); } - }, 1); + }); }, mining: function (callback) { - setTimeout(function () { + async.nextTick(function () { var mining; try { @@ -417,11 +390,11 @@ Node.prototype.getStats = function() } callback(null, mining); - }, 1); + }); }, hashrate: function (callback) { - setTimeout(function () { + async.nextTick(function () { var hashrate; try { @@ -433,11 +406,11 @@ Node.prototype.getStats = function() } callback(null, hashrate); - }, 1); + }); }, - gasprice: function (callback) + gasPrice: function (callback) { - setTimeout(function () { + async.nextTick(function () { var gasPrice; try { @@ -449,7 +422,23 @@ Node.prototype.getStats = function() } callback(null, gasPrice); - }, 1); + }); + }, + minerName: function (callback) + { + async.nextTick(function () { + var minerName; + + try { + minerName = self.getMinerName(self.stats.block.miner); + } + catch (err) { + console.error('xx> minerName failed: ', err); + callback(err, null); + } + + callback(null, minerName); + }); } }, function (err, results) @@ -459,13 +448,7 @@ Node.prototype.getStats = function() if (err) { console.error('xx> getStats error: ', err); - self.stats.peers = 0; - self.stats.active = false; - self.stats.pending = 0; - self.stats.mining = false; - self.stats.hashrate = 0; - self.stats.gasPrice = 0; - self._down++; + self.setInactive(); return false; } @@ -478,21 +461,15 @@ Node.prototype.getStats = function() if(results.peers !== null) { - self.stats.peers = results.peers; self.stats.active = true; + self.stats.peers = results.peers; self.stats.pending = results.pending; self.stats.mining = results.mining; self.stats.hashrate = results.hashrate; self.stats.gasPrice = results.gasPrice; } else { - self.stats.peers = 0; - self.stats.active = false; - self.stats.pending = 0; - self.stats.mining = false; - self.stats.hashrate = 0; - self.stats.gasPrice = 0; - self._down++; + self.setInactive(); } self.uptime(); @@ -509,12 +486,13 @@ Node.prototype.getStatsBlock = function () if(this._web3) { - console.log("==> Getting block"); + var start = _.now(); var block = this.getBlock(); + var end = _.now(); if( !_.isUndefined(block) && !_.isUndefined(block.number) && !_.isUndefined(block.hash) && block.hash !== '?' ) { - console.log("==> Got block:", block.number); + console.log("==> Got block:", block.number, 'in', end - start, 'ms'); this.stats.block = block; } else @@ -523,7 +501,6 @@ Node.prototype.getStatsBlock = function () } } - this.uptime(); this.sendUpdate(); } @@ -563,18 +540,16 @@ Node.prototype.getHistory = function (range) Node.prototype.updateBlock = function() { - console.log('==> Getting block stats'); - this.getStatsBlock(); + this.update(true); + return this; }; -Node.prototype.update = function() +Node.prototype.update = function(forced) { - console.log('==> Getting stats'); - - this.getStats(); + this.getStats(forced); return this; }; @@ -618,8 +593,7 @@ Node.prototype.setWatches = function() var time = now - self._lastChainLog; self._lastChainLog = now; - console.log('>>> Chain Filter triggered: ', now); - console.log('>>> Last chain Filter triggered: ', time); + console.log('>>> Chain Filter triggered: ', now, '- last trigger:', time); if(time > 50) { @@ -692,12 +666,20 @@ Node.prototype.installContract = function() Node.prototype.init = function() { + // Fetch node info this.getInfo(); + + // Start socket connection this.startSocketConnection(); + + // Install Registrar contract this.installContract(); + + // Set filters this.setWatches(); + + // Update stats and send info this.updateBlock(); - this.update(); } Node.prototype.stop = function() @@ -705,7 +687,6 @@ Node.prototype.stop = function() if(this._socket) socket.end(); - if(this.updateInterval) clearInterval(this.updateInterval); From 019507a44f944960f06940873f51334a87cc43b6 Mon Sep 17 00:00:00 2001 From: cubedro Date: Tue, 5 May 2015 01:00:01 +0300 Subject: [PATCH 3/4] re-added pending filter --- lib/node.js | 57 +++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/lib/node.js b/lib/node.js index 9654e9d..82b8fcc 100644 --- a/lib/node.js +++ b/lib/node.js @@ -542,8 +542,6 @@ Node.prototype.updateBlock = function() { this.getStatsBlock(); - this.update(true); - return this; }; @@ -613,34 +611,34 @@ Node.prototype.setWatches = function() console.error(err); } - // try { - // this.pendingFilter = web3.eth.filter('pending'); - // this.pendingFilter.watch( function (log) - // { - // var now = _.now(); - // var time = now - self._lastPendingLog; + try { + this.pendingFilter = web3.eth.filter('pending'); + this.pendingFilter.watch( function (log) + { + var now = _.now(); + var time = now - self._lastPendingLog; - // console.log('>>> Pending Filter triggered: ', now); + console.log('>>> Pending Filter triggered: ', now); - // if(time > 50) - // { - // self.update(); - // } - // else - // { - // debounce(function() { - // self.update(); - // }, 50); - // } + if(time > 50) + { + self.update(true); + } + else + { + debounce(function() { + self.update(true); + }, 50); + } - // self._lastPendingLog = now; - // }); - // } - // catch (err) - // { - // console.error("Couldn't set up pending filter"); - // console.error(err); - // } + self._lastPendingLog = now; + }); + } + catch (err) + { + console.error("Couldn't set up pending filter"); + console.error(err); + } this.updateInterval = setInterval( function(){ self.update(); @@ -678,8 +676,11 @@ Node.prototype.init = function() // Set filters this.setWatches(); - // Update stats and send info + // Update block and send info this.updateBlock(); + + // Update stats and send info + this.update(); } Node.prototype.stop = function() From c2f8a103bdcf5ab5ea8c3e28de1e6bc37eed0301 Mon Sep 17 00:00:00 2001 From: cubedro Date: Tue, 5 May 2015 01:31:19 +0300 Subject: [PATCH 4/4] optimization --- lib/node.js | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/node.js b/lib/node.js index 82b8fcc..08f68dd 100644 --- a/lib/node.js +++ b/lib/node.js @@ -177,9 +177,10 @@ Node.prototype.setupSockets = function() .on('ready', function() { self._socket = true; - self.sendUpdate(true); - console.info('==> The connection has been established.'); + + self.updateBlock(); + self.update(true); }) .on('data', function incoming(data) { @@ -216,7 +217,7 @@ Node.prototype.setupSockets = function() }) .on('reconnecting', function reconnecting(opts) { - console.warn('We are scheduling a reconnect operation', opts); + console.warn('!!!', 'We are scheduling a reconnect operation', opts); }); } @@ -249,6 +250,10 @@ Node.prototype.setInactive = function() Node.prototype.getInfo = function() { + var start = _.now(); + + console.info('==>', 'Getting info'); + try { this.info.coinbase = web3.eth.coinbase; this.info.node = web3.version.client; @@ -256,7 +261,8 @@ Node.prototype.getInfo = function() this.info.protocol = web3.toDecimal(web3.version.ethereum); this.info.api = web3.version.api; - console.info(this.info); + console.info('==>', 'Got info in', _.now() - start, 'ms'); + console.info(' i ', this.info); return true; } @@ -273,7 +279,8 @@ Node.prototype.getBlock = function(number) number: 0, hash: '?', difficulty: 0, - timestamp: 0 + timestamp: 0, + miner: '' }; if( _.isUndefined(number) ) @@ -617,8 +624,9 @@ Node.prototype.setWatches = function() { var now = _.now(); var time = now - self._lastPendingLog; + self._lastPendingLog = now; - console.log('>>> Pending Filter triggered: ', now); + console.log('>>> Pending Filter triggered: ', now, '- last trigger:', time); if(time > 50) { @@ -630,8 +638,6 @@ Node.prototype.setWatches = function() self.update(true); }, 50); } - - self._lastPendingLog = now; }); } catch (err) @@ -651,13 +657,17 @@ Node.prototype.setWatches = function() Node.prototype.installContract = function() { + var start = _.now(); + try { Contract = web3.eth.contract( registrar.desc ); this._Registrar = new Contract( registrar.address ); + + console.log('==>', 'Installed Registrar contract in', _.now() - start, 'ms'); } catch (err) { - console.error("Couldn't set up registrar contract"); + console.error("!!!", "Couldn't set up registrar contract"); console.error(err); } } @@ -667,20 +677,14 @@ Node.prototype.init = function() // Fetch node info this.getInfo(); - // Start socket connection - this.startSocketConnection(); - // Install Registrar contract this.installContract(); + // Start socket connection + this.startSocketConnection(); + // Set filters this.setWatches(); - - // Update block and send info - this.updateBlock(); - - // Update stats and send info - this.update(); } Node.prototype.stop = function()