Add block-sync to application.

This commit is contained in:
Spencer Ofwiti
2020-11-08 09:31:52 +03:00
parent cec92d25b9
commit ca9ca61ae1
22 changed files with 2885 additions and 306 deletions

View File

@@ -0,0 +1,319 @@
import Web3 from "web3";
if (Web3 === undefined) {
Web3 = require('web3');
}
// TODO: Load these from json external source
const abis = {
bancor: {
'network':[
],
'contract_registry': [
{
"inputs": [
{
"internalType": "bytes32",
"name": "_contractName",
"type": "bytes32"
}
],
"name": "getAddress",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
}
],
'converter_registry': [
{
"inputs": [],
"name": "getConvertibleTokens",
"outputs": [
{
"internalType": "address[]",
"name": "",
"type": "address[]"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
],
},
common: {
'erc20': [
{
"inputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"name": "balanceOf",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "decimals",
"outputs": [
{
"internalType": "uint8",
"name": "",
"type": "uint8"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "name",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "symbol",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
]
}
};
const topics = {
erc20: {
'transfer': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
'convert': '0x7154b38b5dd31bb3122436a96d4e09aba5b323ae1fd580025fab55074334c095'
}
};
function Token(address, name, symbol) {
let self = this;
self.address = address;
self.name = name;
self.symbol = symbol;
}
Token.prototype.toString = function() {
let self = this;
return 'Token: ' + self.name + ' (' + self.symbol + ')';
}
export function Registry(w3, address) {
let self = this;
self.w3 = w3;
self.contracts = {
'bancor_contract_registry': new self.w3.eth.Contract(abis.bancor['contract_registry'], address),
};
self.contracts_r = {};
self.contracts_r[address] = self.contracts['bancor_contract_registry'];
self.tokens = [];
self.tokens_s = {};
self.tokens_r = {};
self.init = {
network: [1, 3], // current index, target index
tokens: [0, -1], // current index, target index
}
self.ontokensload= function(n) {
console.debug('tokens loaded', n);
}
self.onregistryload= function() {
console.debug('registry loaded');
}
}
Registry.prototype.load = async function() {
console.debug('loading registry');
let self = this;
const cr = self.contracts['bancor_contract_registry'];
let crid_hex = self.w3.utils.toHex('BancorConverterRegistry');
let shaid = self.w3.eth.abi.encodeParameter('bytes32', crid_hex)
cr.methods.getAddress(shaid).call().then((address) => {
self.contracts['bancor_converter_registry'] = new self.w3.eth.Contract(abis.bancor['converter_registry'], address);
self.contracts_r[address] = self.contracts['bancor_converter_registry'];
console.log('bancor converter registry', address);
self.load_tokens();
self.init.network[0]++;
if (self.init.network[0] == self.init.network[1]) {
self.onregistryload(self.init.network[0]);
}
});
crid_hex = self.w3.utils.toHex('BancorNetwork');
shaid = self.w3.eth.abi.encodeParameter('bytes32', crid_hex)
cr.methods.getAddress(shaid).call().then((address) => {
self.contracts['bancor_network'] = new self.w3.eth.Contract(abis.bancor['network'], address);
self.contracts_r[address] = self.contracts['bancor_network'];
console.log('bancor network', address);
self.init.network[0]++;
if (self.init.network[0] == self.init.network[1]) {
self.onregistryload(self.init.network[0]);
}
});
};
Registry.prototype.load_tokens = async function() {
console.debug('loading tokens');
let self = this;
const cr = self.contracts['bancor_converter_registry'];
cr.methods.getConvertibleTokens().call().then(async (addresses) => {
self.init.tokens[1] = addresses.length;
for (const address of addresses) {
await self.add_token(address);
console.debug('l ', self.tokens.length, addresses.length);
if (self.tokens.length === addresses.length) {
// self.onload();
}
}
});
};
Registry.prototype.add_token = async function(address) {
let self = this;
const ct = new self.w3.eth.Contract(abis.common['erc20'], address);
const symbol = await ct.methods.symbol().call();
const name = await ct.methods.name().call();
const t = new Token(address, symbol, name);
const ti = self.tokens.length;
self.tokens.push(t);
self.tokens[t.symbol] = self.tokens[ti];
self.tokens_r[address] = self.tokens[ti];
self.init.tokens[0]++;
console.log('added token', t.toString(), ti, address);
if (self.init.tokens[0] == self.init.tokens[1]) {
self.ontokenload(self.init.tokens[0]);
}
};
function Tx(block, tx_index, tx_hash, timestamp, success) {
let self = this;
self.block = block;
self.txIndex = tx_index;
self.txHash = tx_hash;
self.timestamp = timestamp;
self.success = success;
}
function Transfer(tx, token, from, to, value, timestamp, block, transaction) {
let self = this;
self.from = from;
self.to = to;
self.token = token;
self.value = value;
self.tx = tx;
}
function Conversion(tx, sourceToken, destinationToken, trader, fromValue, toValue) {
let self = this;
self.sourceToken = sourceToken;
self.destinationToken = destinationToken;
self.trader = trader;
self.fromValue = fromValue;
self.toValue = toValue;
self.tx = tx;
}
export function TransactionHelper(w3, registry) {
let self = this;
self.registry = registry;
self.w3 = w3;
self.ontransfer = (t) => {
console.debug('transfer ', t);
}
self.onconversion = (c) => {
console.debug('convert ', c);
}
}
TransactionHelper.prototype.processTokenTransactionLog = async function(success, t, log) {
let self = this;
if (log.topics[0] == topics.erc20['transfer']) {
const block = await self.w3.eth.getBlock(log.blockNumber);
const from = self.w3.utils.toChecksumAddress(log.topics[1].substring(26, 66));
const to = self.w3.utils.toChecksumAddress(log.topics[2].substring(26, 66));
const value = self.w3.utils.hexToNumber(log.data);
const tx = new Tx(log.blockNumber, log.transactionIndex, log.transactionHash, block.timestamp, success);
const transfer = new Transfer(tx, t, from, to, value);
self.ontransfer(transfer);
}
}
TransactionHelper.prototype.processConvertTransactionLog = async function(success, log) {
let self = this;
if (log.topics[0] == topics.erc20['convert']) {
const block = await self.w3.eth.getBlock(log.blockNumber);
const sourceToken_address = self.w3.utils.toChecksumAddress('0x' + log.topics[1].substring(26, 66));
const sourceToken = self.registry.tokens_r[sourceToken_address];
const destinationToken_address = self.w3.utils.toChecksumAddress('0x' + log.topics[2].substring(26, 66));
const destinationToken = self.registry.tokens_r[destinationToken_address];
const fromValue = self.w3.utils.hexToNumber(log.data.substring(0, 66));
const toValue = self.w3.utils.hexToNumber('0x' + log.data.substring(66, 130));
const trader = self.w3.utils.toChecksumAddress('0x' + log.data.substring(154));
const tx = new Tx(log.blockNumber, log.transactionIndex, log.transactionHash, block.timestamp, success);
const cv = new Conversion(tx, sourceToken, destinationToken, trader, fromValue, toValue);
self.onconversion(cv);
}
}
TransactionHelper.prototype.processReceipt = async function(r) {
let self = this;
const logs = r.logs;
for (let i = 0; i < logs.length; i++) {
const contract_address = logs[i].address;
const t = self.registry.tokens_r[contract_address];
if (t !== undefined) {
self.processTokenTransactionLog(r.status, t, logs[i]);
} else {
if (self.registry.contracts_r[contract_address] !== undefined) {
self.processConvertTransactionLog(r.status, logs[i]);
}
}
}
}

168
src/assets/js/moolb.js Normal file
View File

@@ -0,0 +1,168 @@
let crypto = undefined;
(function() {
if (typeof module !== 'undefined' && typeof exports !== 'undefined') {
const nodeCrypto = require('crypto');
function hashWrapper(nodeCrypto, alg) {
this.alg = alg;
this.crypto = nodeCrypto.createHash(alg);
}
hashWrapper.prototype.update = function(d) {
this.crypto.update(d);
}
hashWrapper.prototype.digest = async function() {
const z = this.crypto.digest(this.data);
return new Uint8Array(z);
}
function cryptoWrapper(nodeCrypto) {
this.crypto = nodeCrypto
}
cryptoWrapper.prototype.createHash = function(alg) {
return new hashWrapper(this.crypto, alg);
}
module.exports = {
Bloom: Bloom,
fromBytes: fromBytes,
};
crypto = new cryptoWrapper(nodeCrypto);
} else {
function hashWrapper(webCrypto, alg) {
this.alg = alg;
this.crypto = webCrypto;
this.data = undefined;
}
hashWrapper.prototype.update = function(d) {
if (this.data != undefined) {
throw "cannot append";
}
this.data = d;
}
hashWrapper.prototype.digest = async function() {
const z = await this.crypto.subtle.digest('SHA-256', this.data);
return new Uint8Array(z);
}
function cryptoWrapper(webCrypto) {
this.crypto = webCrypto
}
cryptoWrapper.prototype.createHash = function(alg) {
return new hashWrapper(this.crypto, alg);
}
crypto = new cryptoWrapper(window.crypto);
window.Bloom = Bloom;
window.bloomFromBytes = fromBytes;
}
})()
// block numbers 6000000
// false positive probability 2%
//
// m = ceil((n * log(p)) / log(1 / pow(2, log(2))));
// m = ceil((6000000 * log(0.1)) / log(1 / pow(2, log(2))))
// = 3917675
// Creates a new bloom object.
// \param size of filter in bits, aligned to byte boundary
// \param number of rounds to hash
// \param hasher function, which must take two Uint8Array parameters 'data' and 'salt'. If not sef, hashBloomDefault will be used.
function Bloom(bits, rounds, hasher) {
this.bits = bits;
this.bytes = parseInt(bits / 8, 10);
this.rounds = rounds;
if (this.hasher === undefined) {
this.hasher = hashBloomDefault;
}
if (this.bytes * 8 != this.bits) {
console.error('number of bits must be on byte boundary');
return false;
} else {
this.filter = new Uint8Array(this.bytes);
}
}
// add entry to bloom filter
// \param value to add
Bloom.prototype.add = async function(v) {
let a = new ArrayBuffer(v.byteLength + 4);
let iw = new DataView(a);
for (let i = 0; i < v.byteLength; i++) {
iw.setUint8(i, v[i]);
}
//console.debug(iw, v);
for (var i = 0; i < this.rounds; i++) {
iw.setInt32(v.byteLength, i);
let result = await this.hasher(iw);
let resultHex = Array.prototype.map.call(new Uint8Array(result), x => ('00' + x.toString(16)).slice(-2)).join('');
let resultInt = parseInt(BigInt('0x'+resultHex) % BigInt(this.bits), 10);
let bytepos = parseInt(resultInt / 8, 10);
let bitpos = parseInt(resultInt, 10) % 8;
this.filter[bytepos] |= 1 << bitpos;
//console.log("setpos ", bytepos, bitpos);
}
};
// checks if the block number has been added to the bloom filter
// \param value to check for
// \return false if not found in filter
Bloom.prototype.check = async function(v) {
let a = new ArrayBuffer(v.byteLength + 4);
let iw = new DataView(a);
for (let i = 0; i < v.byteLength; i++) {
iw.setUint8(i, v[i]);
}
for (let j = 0; j < this.filter.byteLength; j++) {
if (v[3] == 20 && this.filter[j] > 0) {
console.log('filter ', j, this.filter[j]);
}
}
for (let i = 0; i < this.rounds; i++) {
iw.setInt32(v.byteLength, i);
let result = await this.hasher(iw);
//console.log('result', result);
let resultHex = Array.prototype.map.call(new Uint8Array(result), x => ('00' + x.toString(16)).slice(-2)).join('');
let resultInt = parseInt(BigInt('0x'+resultHex) % BigInt(this.bits), 10);
let bytepos = parseInt(resultInt / 8, 10);
//console.log("setpos ", v, bytepos, resultInt % 8);
if (this.filter[bytepos] === undefined) {
console.error('byte pos ' + bytepos + ' is undefined (filter length ' + this.filter.byteLength + ')');
return false;
} else {
let test = 1 << (0xff & (resultInt % 8));
if (v[3] == 20) {
console.log('test', test, bytepos, resultInt % 8);
}
if ((this.filter[bytepos] & test) == 0) {
return false;
}
}
}
return true;
};
// return the verbatim filter
// \return Uint8Array filter
Bloom.prototype.bytes = function() {
return this.filter;
};
// Default hashing function used in Bloom.add() - sha256
// \param data to insert in filter
// \param salt, typically a sequence number
// \return Uint8Array digest
async function hashBloomDefault(data, salt) {
const h = crypto.createHash('sha256');
h.update(data);
const z = await h.digest();
return Uint8Array.from(z);
}
function fromBytes(bytes, rounds, hasher) {
const bits = bytes.byteLength * 8;
let b = new Bloom(bits, rounds, hasher);
b.filter = bytes;
return b;
}

31
src/assets/js/plugin.js Normal file
View File

@@ -0,0 +1,31 @@
export function fetcher(settings) {
let xhr = new XMLHttpRequest();
xhr.responseType = 'json';
xhr.open('GET', 'http://localhost:5555/tx/0/100');
xhr.addEventListener('load', async (e) => {
const d = xhr.response;
//const digest = await crypto.subtle.digest('SHA-256', ArrayBuffer.from(d.block_filter))
//console.log('block filter digest', digest)
const block_filter_binstr = window.atob(d.block_filter);
let b_one = new Uint8Array(block_filter_binstr.length);
b_one.map(function(e, i, v) {
v[i] = block_filter_binstr.charCodeAt([i]);
});
const blocktx_filter_binstr = window.atob(d.blocktx_filter);
let b_two = new Uint8Array(blocktx_filter_binstr.length);
b_two.map(function(e, i, v) {
v[i] = blocktx_filter_binstr.charCodeAt([i]);
});
for (let i = 0; i < block_filter_binstr.length; i++) {
if (b_one[i] > 0 ) {
console.debug('blocktx value on', i);
}
}
settings.scanFilter(settings, d.low, d.high, b_one, b_two, d.filter_rounds)
});
xhr.send();
}

30
src/assets/js/sync.js Normal file
View File

@@ -0,0 +1,30 @@
function sync_by_filter_block(block, count, buf, bloom_blocktx, result_callback) {
for (let j = 0; j < count; j++) {
let w = new DataView(buf);
w.setInt32(4, j);
const r = new Uint8Array(buf);
bloom_blocktx.check(r).then(function(ok) {
if (ok) {
console.debug('match in block' + block + ' tx ' + j);
result_callback(block, j);
}
});
}
}
function sync_by_filter(caller, lo, hi, bloom_block, bloom_blocktx, tx_count_getter, result_callback) {
for (let i = lo; i <= hi; i++) {
let a = new ArrayBuffer(8);
let w = new DataView(a);
w.setInt32(0, i);
const r = new Uint8Array(a.slice(0, 4));
bloom_block.check(r).then(function(ok) {
if (ok) {
console.debug('match in block ' + i);
tx_count_getter(i).then(function(n) {
sync_by_filter_block(i, n, a, bloom_blocktx, result_callback);
});
}
});
}
}

1
src/assets/js/web3.min.js vendored Normal file

File diff suppressed because one or more lines are too long

63
src/assets/js/worker.js Normal file
View File

@@ -0,0 +1,63 @@
let window = self;
importScripts('./moolb.js');
importScripts('./sync.js');
importScripts('./web3.min.js');
let s = undefined;
// TODO: as worker has its own scope we don't need the object
function Driver(messager, provider, lo, filters, syncer) {
this.w3 = new Web3(provider);
this.lo = lo;
this.hi = lo;
this.filters = filters;
this.syncer = syncer;
this.messager = messager;
}
Driver.prototype.start = function(hi) {
let self = this;
if (hi !== undefined) {
self.sync(hi);
return;
}
s.w3.eth.getBlockNumber().then(function(n) {
self.sync(n);
});
};
Driver.prototype.sync = function(n) {
this.hi = n;
this.syncer(this, this.lo, this.hi, this.filters[0], this.filters[1], this.getCount, this.process);
};
Driver.prototype.process = function(b, t) {
s.w3.eth.getTransactionFromBlock(b, t).then((t) => {
s.w3.eth.getTransactionReceipt(t.hash).then((r) => {
this.postMessage(r);
});
}).catch(function(e) {
//this.postMessage(['failed getTransactionFromBlock(' + b + ', ' + t + ')']);
console.error('failed getTransactionFromBlock(' + b + ', ' + t + ')');
});
}
Driver.prototype.getCount = async function (b) {
const n = s.w3.eth.getBlockTransactionCount(b);
return n;
};
onmessage = function(o) {
const filters = [
bloomFromBytes(o.data.filters[0], o.data.filter_rounds),
bloomFromBytes(o.data.filters[1], o.data.filter_rounds),
];
s = new Driver(postMessage, o.data.w3_provider, o.data.lo, filters, sync_by_filter);
let hi = undefined;
if (o.data.hi > 0) {
hi = o.data.hi;
}
s.start(hi);
};