Smarter Tokens fetching (#3546)

* Don't auto-subscribe for contracts #3240

* Smarter Balance : don't re-instantiate contracts, fetch tokens #3240

* Smarter Balance Tokens fetching (update image when needed) #3240

* Attaching to TokenReg Events instead of fetching for each block #3240

* Unsubscribe from shapeshit... #3240

* Unsubscribe from EthFilter if used once (resolved) #3240

* Remove warning

* PR review fix

* Typo

* Better contract JS API : one subscribe for all #3546

* Fixed test
This commit is contained in:
Nicolas Gotchac 2016-11-23 10:30:47 +01:00 committed by Jaco Greeff
parent a969c008d1
commit 33dd49160f
10 changed files with 440 additions and 150 deletions

View File

@ -15,7 +15,8 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
export default function (rpc) {
const subscriptions = [];
let subscriptions = [];
let pollStatusIntervalId = null;
function getCoins () {
return rpc.get('getcoins');
@ -45,6 +46,24 @@ export default function (rpc) {
callback,
idx
});
// Only poll if there are subscriptions...
if (!pollStatusIntervalId) {
pollStatusIntervalId = setInterval(_pollStatus, 2000);
}
}
function unsubscribe (depositAddress) {
const newSubscriptions = []
.concat(subscriptions)
.filter((sub) => sub.depositAddress !== depositAddress);
subscriptions = newSubscriptions;
if (subscriptions.length === 0) {
clearInterval(pollStatusIntervalId);
pollStatusIntervalId = null;
}
}
function _getSubscriptionStatus (subscription) {
@ -81,13 +100,12 @@ export default function (rpc) {
subscriptions.forEach(_getSubscriptionStatus);
}
setInterval(_pollStatus, 2000);
return {
getCoins,
getMarketInfo,
getStatus,
shift,
subscribe
subscribe,
unsubscribe
};
}

View File

@ -48,7 +48,11 @@ export default class Contract {
this._instance[fn.signature] = fn;
});
this._sendSubscriptionChanges();
this._subscribedToPendings = false;
this._pendingsSubscriptionId = null;
this._subscribedToBlock = false;
this._blockSubscriptionId = null;
}
get address () {
@ -239,44 +243,71 @@ export default class Contract {
return event;
}
subscribe (eventName = null, options = {}, callback) {
return new Promise((resolve, reject) => {
let event = null;
_findEvent (eventName = null) {
const event = eventName
? this._events.find((evt) => evt.name === eventName)
: null;
if (eventName) {
event = this._events.find((evt) => evt.name === eventName);
if (eventName && !event) {
const events = this._events.map((evt) => evt.name).join(', ');
throw new Error(`${eventName} is not a valid eventName, subscribe using one of ${events} (or null to include all)`);
}
if (!event) {
const events = this._events.map((evt) => evt.name).join(', ');
reject(new Error(`${eventName} is not a valid eventName, subscribe using one of ${events} (or null to include all)`));
return;
}
}
return event;
}
return this._subscribe(event, options, callback).then(resolve).catch(reject);
_createEthFilter (event = null, _options) {
const optionTopics = _options.topics || [];
const signature = event && event.signature || null;
// If event provided, remove the potential event signature
// as the first element of the topics
const topics = signature
? [ signature ].concat(optionTopics.filter((t, idx) => idx > 0 || t !== signature))
: optionTopics;
const options = Object.assign({}, _options, {
address: this._address,
topics
});
return this._api.eth.newFilter(options);
}
subscribe (eventName = null, options = {}, callback) {
try {
const event = this._findEvent(eventName);
return this._subscribe(event, options, callback);
} catch (e) {
return Promise.reject(e);
}
}
_subscribe (event = null, _options, callback) {
const subscriptionId = nextSubscriptionId++;
const options = Object.assign({}, _options, {
address: this._address,
topics: [event ? event.signature : null]
});
const { skipInitFetch } = _options;
delete _options['skipInitFetch'];
return this._api.eth
.newFilter(options)
return this
._createEthFilter(event, _options)
.then((filterId) => {
this._subscriptions[subscriptionId] = {
options: _options,
callback,
filterId
};
if (skipInitFetch) {
this._subscribeToChanges();
return subscriptionId;
}
return this._api.eth
.getFilterLogs(filterId)
.then((logs) => {
callback(null, this.parseEventLogs(logs));
this._subscriptions[subscriptionId] = {
options,
callback,
filterId
};
this._subscribeToChanges();
return subscriptionId;
});
});
@ -285,19 +316,89 @@ export default class Contract {
unsubscribe (subscriptionId) {
return this._api.eth
.uninstallFilter(this._subscriptions[subscriptionId].filterId)
.then(() => {
delete this._subscriptions[subscriptionId];
})
.catch((error) => {
console.error('unsubscribe', error);
})
.then(() => {
delete this._subscriptions[subscriptionId];
this._unsubscribeFromChanges();
});
}
_sendSubscriptionChanges = () => {
_subscribeToChanges = () => {
const subscriptions = Object.values(this._subscriptions);
const timeout = () => setTimeout(this._sendSubscriptionChanges, 1000);
Promise
const pendingSubscriptions = subscriptions
.filter((s) => s.options.toBlock && s.options.toBlock === 'pending');
const otherSubscriptions = subscriptions
.filter((s) => !(s.options.toBlock && s.options.toBlock === 'pending'));
if (pendingSubscriptions.length > 0 && !this._subscribedToPendings) {
this._subscribedToPendings = true;
this._subscribeToPendings();
}
if (otherSubscriptions.length > 0 && !this._subscribedToBlock) {
this._subscribedToBlock = true;
this._subscribeToBlock();
}
}
_unsubscribeFromChanges = () => {
const subscriptions = Object.values(this._subscriptions);
const pendingSubscriptions = subscriptions
.filter((s) => s.options.toBlock && s.options.toBlock === 'pending');
const otherSubscriptions = subscriptions
.filter((s) => !(s.options.toBlock && s.options.toBlock === 'pending'));
if (pendingSubscriptions.length === 0 && this._subscribedToPendings) {
this._subscribedToPendings = false;
clearTimeout(this._pendingsSubscriptionId);
}
if (otherSubscriptions.length === 0 && this._subscribedToBlock) {
this._subscribedToBlock = false;
this._api.unsubscribe(this._blockSubscriptionId);
}
}
_subscribeToBlock = () => {
this._api
.subscribe('eth_blockNumber', (error) => {
if (error) {
console.error('::_subscribeToBlock', error, error && error.stack);
}
const subscriptions = Object.values(this._subscriptions)
.filter((s) => !(s.options.toBlock && s.options.toBlock === 'pending'));
this._sendSubscriptionChanges(subscriptions);
})
.then((blockSubId) => {
this._blockSubscriptionId = blockSubId;
})
.catch((e) => {
console.error('::_subscribeToBlock', e, e && e.stack);
});
}
_subscribeToPendings = () => {
const subscriptions = Object.values(this._subscriptions)
.filter((s) => s.options.toBlock && s.options.toBlock === 'pending');
const timeout = () => setTimeout(() => this._subscribeFromPendings(), 1000);
this._sendSubscriptionChanges(subscriptions)
.then(() => {
this._pendingsSubscriptionId = timeout();
});
}
_sendSubscriptionChanges = (subscriptions) => {
return Promise
.all(
subscriptions.map((subscription) => {
return this._api.eth.getFilterChanges(subscription.filterId);
@ -315,12 +416,9 @@ export default class Contract {
console.error('_sendSubscriptionChanges', error);
}
});
timeout();
})
.catch((error) => {
console.error('_sendSubscriptionChanges', error);
timeout();
});
}
}

View File

@ -437,6 +437,7 @@ describe('api/contract/Contract', () => {
]
}
];
const logs = [{
address: '0x22bff18ec62281850546a664bb63a5c06ac5f76c',
blockHash: '0xa9280530a3b47bee2fc80f2862fd56502ae075350571d724d6442ea4c597347b',
@ -450,6 +451,7 @@ describe('api/contract/Contract', () => {
transactionHash: '0xca16f537d761d13e4e80953b754e2b15541f267d6cad9381f750af1bae1e4917',
transactionIndex: '0x0'
}];
const parsed = [{
address: '0x22bfF18ec62281850546a664bb63a5C06AC5F76C',
blockHash: '0xa9280530a3b47bee2fc80f2862fd56502ae075350571d724d6442ea4c597347b',
@ -466,11 +468,13 @@ describe('api/contract/Contract', () => {
sender: { type: 'address', value: '0x63Cf90D3f0410092FC0fca41846f596223979195' }
},
topics: [
'0x954ba6c157daf8a26539574ffa64203c044691aa57251af95f4b48d85ec00dd5', '0x0000000000000000000000000000000000000000000000000001000000004fe0'
'0x954ba6c157daf8a26539574ffa64203c044691aa57251af95f4b48d85ec00dd5',
'0x0000000000000000000000000000000000000000000000000001000000004fe0'
],
transactionHash: '0xca16f537d761d13e4e80953b754e2b15541f267d6cad9381f750af1bae1e4917',
transactionIndex: new BigNumber(0)
}];
let contract;
beforeEach(() => {
@ -496,18 +500,19 @@ describe('api/contract/Contract', () => {
scope = mockHttp([
{ method: 'eth_newFilter', reply: { result: '0x123' } },
{ method: 'eth_getFilterLogs', reply: { result: logs } },
{ method: 'eth_getFilterChanges', reply: { result: logs } },
{ method: 'eth_newFilter', reply: { result: '0x123' } },
{ method: 'eth_getFilterLogs', reply: { result: logs } }
]);
cbb = sinon.stub();
cbe = sinon.stub();
return contract.subscribe('Message', {}, cbb);
return contract.subscribe('Message', { toBlock: 'pending' }, cbb);
});
it('sets the subscriptionId returned', () => {
return contract
.subscribe('Message', {}, cbe)
.subscribe('Message', { toBlock: 'pending' }, cbe)
.then((subscriptionId) => {
expect(subscriptionId).to.equal(1);
});
@ -515,7 +520,7 @@ describe('api/contract/Contract', () => {
it('creates a new filter and retrieves the logs on it', () => {
return contract
.subscribe('Message', {}, cbe)
.subscribe('Message', { toBlock: 'pending' }, cbe)
.then((subscriptionId) => {
expect(scope.isDone()).to.be.true;
});
@ -523,7 +528,7 @@ describe('api/contract/Contract', () => {
it('returns the logs to the callback', () => {
return contract
.subscribe('Message', {}, cbe)
.subscribe('Message', { toBlock: 'pending' }, cbe)
.then((subscriptionId) => {
expect(cbe).to.have.been.calledWith(null, parsed);
});

View File

@ -15,6 +15,7 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
import BigNumber from 'bignumber.js';
import { range } from 'lodash';
import { isArray, isHex, isInstanceOf, isString } from '../util/types';
@ -50,14 +51,19 @@ export function inHash (hash) {
return inHex(hash);
}
export function pad (input, length) {
const value = inHex(input).substr(2, length * 2);
return '0x' + value + range(length * 2 - value.length).map(() => '0').join('');
}
export function inTopics (_topics) {
let topics = (_topics || [])
.filter((topic) => topic)
.map(inHex);
.filter((topic) => topic === null || topic)
.map((topic) => topic === null ? null : pad(topic, 32));
while (topics.length < 4) {
topics.push(null);
}
// while (topics.length < 4) {
// topics.push(null);
// }
return topics;
}

View File

@ -29,3 +29,7 @@ export function hex2Ascii (_hex) {
return str;
}
export function asciiToHex (string) {
return '0x' + string.split('').map((s) => s.charCodeAt(0).toString(16)).join('');
}

View File

@ -16,7 +16,7 @@
import { isAddress as isAddressValid, toChecksumAddress } from '../../abi/util/address';
import { decodeCallData, decodeMethodInput, methodToAbi } from './decode';
import { bytesToHex, hex2Ascii } from './format';
import { bytesToHex, hex2Ascii, asciiToHex } from './format';
import { fromWei, toWei } from './wei';
import { sha3 } from './sha3';
import { isArray, isFunction, isHex, isInstanceOf, isString } from './types';
@ -31,6 +31,7 @@ export default {
isString,
bytesToHex,
hex2Ascii,
asciiToHex,
createIdentityImg,
decodeCallData,
decodeMethodInput,

View File

@ -19,17 +19,34 @@ export const checkIfVerified = (contract, account) => {
};
export const checkIfRequested = (contract, account) => {
let subId = null;
let resolved = false;
return new Promise((resolve, reject) => {
contract.subscribe('Requested', {
fromBlock: 0, toBlock: 'pending'
}, (err, logs) => {
if (err) {
return reject(err);
}
const e = logs.find((l) => {
return l.type === 'mined' && l.params.who && l.params.who.value === account;
contract
.subscribe('Requested', {
fromBlock: 0, toBlock: 'pending'
}, (err, logs) => {
if (err) {
return reject(err);
}
const e = logs.find((l) => {
return l.type === 'mined' && l.params.who && l.params.who.value === account;
});
resolve(e ? e.transactionHash : false);
resolved = true;
if (subId) {
contract.unsubscribe(subId);
}
})
.then((_subId) => {
subId = _subId;
if (resolved) {
contract.unsubscribe(subId);
}
});
resolve(e ? e.transactionHash : false);
});
});
};

View File

@ -63,6 +63,16 @@ export default class Shapeshift extends Component {
this.retrieveCoins();
}
componentWillUnmount () {
this.unsubscribe();
}
unsubscribe () {
// Unsubscribe from Shapeshit
const { depositAddress } = this.state;
shapeshift.unsubscribe(depositAddress);
}
render () {
const { error, stage } = this.state;
@ -205,6 +215,10 @@ export default class Shapeshift extends Component {
console.log('onShift', result);
const depositAddress = result.deposit;
if (this.state.depositAddress) {
this.unsubscribe();
}
shapeshift.subscribe(depositAddress, this.onExchangeInfo);
this.setState({ depositAddress });
})

View File

@ -31,13 +31,23 @@ export default class Balances {
constructor (store, api) {
this._api = api;
this._store = store;
this._tokens = {};
this._images = {};
this._accountsInfo = null;
this._tokens = [];
this._tokenreg = null;
this._fetchingTokens = false;
this._fetchedTokens = false;
this._tokenregSubId = null;
this._tokenregMetaSubId = null;
}
start () {
this._subscribeBlockNumber();
this._subscribeAccountsInfo();
this._retrieveTokens();
}
_subscribeAccountsInfo () {
@ -48,10 +58,7 @@ export default class Balances {
}
this._accountsInfo = accountsInfo;
this._retrieveBalances();
})
.then((subscriptionId) => {
console.log('_subscribeAccountsInfo', 'subscriptionId', subscriptionId);
this._retrieveTokens();
})
.catch((error) => {
console.warn('_subscribeAccountsInfo', error);
@ -62,21 +69,22 @@ export default class Balances {
this._api
.subscribe('eth_blockNumber', (error) => {
if (error) {
return;
return console.warn('_subscribeBlockNumber', error);
}
this._retrieveTokens();
})
.then((subscriptionId) => {
console.log('_subscribeBlockNumber', 'subscriptionId', subscriptionId);
})
.catch((error) => {
console.warn('_subscribeBlockNumber', error);
});
}
_retrieveTokens () {
this._api.parity
getTokenRegistry () {
if (this._tokenreg) {
return Promise.resolve(this._tokenreg);
}
return this._api.parity
.registryAddress()
.then((registryAddress) => {
const registry = this._api.newContract(abis.registry, registryAddress);
@ -85,60 +93,49 @@ export default class Balances {
})
.then((tokenregAddress) => {
const tokenreg = this._api.newContract(abis.tokenreg, tokenregAddress);
this._tokenreg = tokenreg;
this.attachToTokens();
return tokenreg;
});
}
_retrieveTokens () {
if (this._fetchingTokens) {
return;
}
if (this._fetchedTokens) {
return this._retrieveBalances();
}
this._fetchingTokens = true;
this._fetchedTokens = false;
this
.getTokenRegistry()
.then((tokenreg) => {
return tokenreg.instance.tokenCount
.call()
.then((numTokens) => {
const promisesTokens = [];
const promisesImages = [];
const promises = [];
while (promisesTokens.length < numTokens.toNumber()) {
const index = promisesTokens.length;
promisesTokens.push(tokenreg.instance.token.call({}, [index]));
promisesImages.push(tokenreg.instance.meta.call({}, [index, 'IMG']));
for (let i = 0; i < numTokens.toNumber(); i++) {
promises.push(this.fetchTokenInfo(tokenreg, i));
}
return Promise.all([
Promise.all(promisesTokens),
Promise.all(promisesImages)
]);
return Promise.all(promises);
});
})
.then(([_tokens, images]) => {
const tokens = {};
this._tokens = _tokens
.map((_token, index) => {
const [address, tag, format, name] = _token;
.then(() => {
this._fetchingTokens = false;
this._fetchedTokens = true;
const token = {
address,
name,
tag,
format: format.toString(),
contract: this._api.newContract(abis.eip20, address)
};
tokens[address] = token;
this._store.dispatch(setAddressImage(address, images[index]));
return token;
})
.sort((a, b) => {
if (a.tag < b.tag) {
return -1;
} else if (a.tag > b.tag) {
return 1;
}
return 0;
});
this._store.dispatch(getTokens(tokens));
this._store.dispatch(getTokens(this._tokens));
this._retrieveBalances();
})
.catch((error) => {
console.warn('_retrieveTokens', error);
this._retrieveBalances();
console.warn('balances::_retrieveTokens', error);
});
}
@ -147,48 +144,20 @@ export default class Balances {
return;
}
const addresses = Object.keys(this._accountsInfo);
const addresses = Object
.keys(this._accountsInfo)
.filter((address) => {
const account = this._accountsInfo[address];
return !account.meta || !account.meta.deleted;
});
this._balances = {};
Promise
.all(
addresses.map((address) => Promise.all([
this._api.eth.getBalance(address),
this._api.eth.getTransactionCount(address)
]))
)
.then((balanceTxCount) => {
return Promise.all(
balanceTxCount.map(([value, txCount], idx) => {
const address = addresses[idx];
this._balances[address] = {
txCount,
tokens: [{
token: ETH,
value
}]
};
return Promise.all(
this._tokens.map((token) => {
return token.contract.instance.balanceOf.call({}, [address]);
})
);
})
);
})
.then((tokenBalances) => {
addresses.forEach((address, idx) => {
const balanceOf = tokenBalances[idx];
const balance = this._balances[address];
this._tokens.forEach((token, tidx) => {
balance.tokens.push({
token,
value: balanceOf[tidx]
});
});
.all(addresses.map((a) => this.fetchAccountBalance(a)))
.then((balances) => {
addresses.forEach((a, idx) => {
this._balances[a] = balances[idx];
});
this._store.dispatch(getBalances(this._balances));
@ -197,4 +166,161 @@ export default class Balances {
console.warn('_retrieveBalances', error);
});
}
attachToTokens () {
this.attachToTokenMetaChange();
this.attachToNewToken();
}
attachToNewToken () {
if (this._tokenregSubId) {
return;
}
this._tokenreg
.instance
.Registered
.subscribe({
fromBlock: 0,
toBlock: 'latest',
skipInitFetch: true
}, (error, logs) => {
if (error) {
return console.error('balances::attachToNewToken', 'failed to attach to tokenreg Registered', error.toString(), error.stack);
}
const promises = logs.map((log) => {
const id = log.params.id.value.toNumber();
return this.fetchTokenInfo(this._tokenreg, id);
});
return Promise.all(promises);
})
.then((tokenregSubId) => {
this._tokenregSubId = tokenregSubId;
})
.catch((e) => {
console.warn('balances::attachToNewToken', e);
});
}
attachToTokenMetaChange () {
if (this._tokenregMetaSubId) {
return;
}
this._tokenreg
.instance
.MetaChanged
.subscribe({
fromBlock: 0,
toBlock: 'latest',
topics: [ null, this._api.util.asciiToHex('IMG') ],
skipInitFetch: true
}, (error, logs) => {
if (error) {
return console.error('balances::attachToTokenMetaChange', 'failed to attach to tokenreg MetaChanged', error.toString(), error.stack);
}
// In case multiple logs for same token
// in one block. Take the last value.
const tokens = logs
.filter((log) => log.type === 'mined')
.reduce((_tokens, log) => {
const id = log.params.id.value.toNumber();
const image = log.params.value.value;
const token = Object.values(this._tokens).find((c) => c.id === id);
const { address } = token;
_tokens[address] = { address, id, image };
return _tokens;
}, {});
Object
.values(tokens)
.forEach((token) => {
const { address, image } = token;
if (this._images[address] !== image.toString()) {
this._store.dispatch(setAddressImage(address, image));
this._images[address] = image.toString();
}
});
})
.then((tokenregMetaSubId) => {
this._tokenregMetaSubId = tokenregMetaSubId;
})
.catch((e) => {
console.warn('balances::attachToTokenMetaChange', e);
});
}
fetchTokenInfo (tokenreg, tokenId) {
return Promise
.all([
tokenreg.instance.token.call({}, [tokenId]),
tokenreg.instance.meta.call({}, [tokenId, 'IMG'])
])
.then(([ tokenData, image ]) => {
const [ address, tag, format, name ] = tokenData;
const contract = this._api.newContract(abis.eip20, address);
if (this._images[address] !== image.toString()) {
this._store.dispatch(setAddressImage(address, image));
this._images[address] = image.toString();
}
const token = {
format: format.toString(),
id: tokenId,
address,
tag,
name,
contract
};
this._tokens[address] = token;
return token;
})
.catch((e) => {
console.warn('balances::fetchTokenInfo', `couldn't fetch token #${tokenId}`, e);
});
}
/**
* TODO?: txCount is only shown on an address page, so we
* might not need to fetch it for each address for each block,
* but only for one address when the user is on the account
* view.
*/
fetchAccountBalance (address) {
const _tokens = Object.values(this._tokens);
const tokensPromises = _tokens
.map((token) => {
return token.contract.instance.balanceOf.call({}, [ address ]);
});
return Promise
.all([
this._api.eth.getTransactionCount(address),
this._api.eth.getBalance(address)
].concat(tokensPromises))
.then(([ txCount, ethBalance, ...tokensBalance ]) => {
const tokens = []
.concat(
{ token: ETH, value: ethBalance },
_tokens
.map((token, index) => ({
token,
value: tokensBalance[index]
}))
);
const balance = { txCount, tokens };
return balance;
});
}
}

View File

@ -23,9 +23,10 @@ export const TEST_HTTP_URL = 'http://localhost:6688';
export const TEST_WS_URL = 'ws://localhost:8866';
export function mockHttp (requests) {
nock.cleanAll();
let scope = nock(TEST_HTTP_URL);
requests.forEach((request) => {
requests.forEach((request, index) => {
scope = scope
.post('/')
.reply(request.code || 200, (uri, body) => {