mirror of
git://holbrook.no/eth-monitor.git
synced 2024-11-21 20:06:46 +01:00
Factor out tx, block store
This commit is contained in:
parent
4b372481d4
commit
8454b0eaaf
@ -4,77 +4,29 @@ import logging
|
||||
import json
|
||||
|
||||
# external imports
|
||||
from chainsyncer.backend.file import chain_dir_for
|
||||
from leveldir.numeric import NumDir
|
||||
from leveldir.hex import HexDir
|
||||
from hexathon import strip_0x
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
base_dir = '/var/lib'
|
||||
|
||||
|
||||
class RuledFilter:
|
||||
|
||||
cache_root = None
|
||||
chain_dir = None
|
||||
cache_dir = None
|
||||
block_num_path = None
|
||||
block_src_path = None
|
||||
block_hash_path = None
|
||||
block_num_dir = None
|
||||
block_src_dir = None
|
||||
block_hash_dir = None
|
||||
address_path = None
|
||||
address_dir = None
|
||||
tx_path = None
|
||||
tx_dir = None
|
||||
tx_raw_dir = None
|
||||
|
||||
|
||||
def __init__(self, rules_filter=None):
|
||||
if self.chain_dir == None:
|
||||
raise RuntimeError('filter must be initialized. call RuledFilter.init() first')
|
||||
if self.store.chain_dir == None:
|
||||
raise RuntimeError('store must be initialized. call RuledFilter.init() first')
|
||||
self.rules_filter = rules_filter
|
||||
|
||||
|
||||
@staticmethod
|
||||
def init(chain_spec, cache_root=base_dir, rules_filter=None, include_block_data=False, include_tx_data=False):
|
||||
RuledFilter.cache_root = os.path.join(
|
||||
cache_root,
|
||||
'eth_monitor',
|
||||
chain_spec.engine(),
|
||||
chain_spec.fork(),
|
||||
str(chain_spec.chain_id()),
|
||||
)
|
||||
RuledFilter.chain_dir = chain_dir_for(RuledFilter.cache_root)
|
||||
RuledFilter.cache_dir = os.path.join(RuledFilter.chain_dir, 'cache')
|
||||
RuledFilter.block_src_path = os.path.join(RuledFilter.cache_dir, 'block', 'src')
|
||||
RuledFilter.block_src_dir = NumDir(RuledFilter.block_src_path, [100000, 1000])
|
||||
RuledFilter.block_num_path = os.path.join(RuledFilter.cache_dir, 'block', 'num')
|
||||
RuledFilter.block_num_dir = NumDir(RuledFilter.block_num_path, [100000, 1000])
|
||||
RuledFilter.block_hash_path = os.path.join(RuledFilter.cache_dir, 'block', 'hash')
|
||||
RuledFilter.block_hash_dir = HexDir(RuledFilter.block_hash_path, 32, levels=2)
|
||||
RuledFilter.tx_path = os.path.join(RuledFilter.cache_dir, 'tx', 'src')
|
||||
RuledFilter.tx_raw_path = os.path.join(RuledFilter.cache_dir, 'tx', 'raw')
|
||||
RuledFilter.tx_dir = HexDir(RuledFilter.tx_path, 32, levels=2)
|
||||
RuledFilter.tx_raw_dir = HexDir(RuledFilter.tx_raw_path, 32, levels=2)
|
||||
RuledFilter.address_path = os.path.join(RuledFilter.cache_dir, 'address')
|
||||
RuledFilter.address_dir = HexDir(RuledFilter.address_path, 20, levels=2)
|
||||
RuledFilter.chain_spec = chain_spec
|
||||
def init(store, include_block_data=False, include_tx_data=False):
|
||||
RuledFilter.store = store
|
||||
RuledFilter.include_block_data = include_block_data
|
||||
RuledFilter.include_tx_data = include_tx_data
|
||||
|
||||
|
||||
@classmethod
|
||||
def block_callback(cls, block, extra=None):
|
||||
hash_bytes = bytes.fromhex(strip_0x(block.hash))
|
||||
cls.block_num_dir.add(block.number, hash_bytes)
|
||||
num_bytes = block.number.to_bytes(8, 'big')
|
||||
cls.block_hash_dir.add(hash_bytes, num_bytes)
|
||||
if cls.include_block_data:
|
||||
src = json.dumps(block.src()).encode('utf-8')
|
||||
cls.block_src_dir.add(hash_bytes, src)
|
||||
cls.store.put_block(block, include_data=cls.include_block_data)
|
||||
|
||||
|
||||
def filter(self, conn, block, tx, db_session=None):
|
||||
|
@ -1,14 +1,5 @@
|
||||
# standard imports
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
|
||||
# external imports
|
||||
from hexathon import strip_0x
|
||||
from chainlib.eth.tx import (
|
||||
Tx,
|
||||
pack,
|
||||
)
|
||||
|
||||
# local imports
|
||||
from eth_monitor.filters import RuledFilter
|
||||
@ -19,14 +10,4 @@ logg = logging.getLogger(__name__)
|
||||
class Filter(RuledFilter):
|
||||
|
||||
def ruled_filter(self, conn, block, tx, db_session=None):
|
||||
raw = pack(tx.src(), self.chain_spec)
|
||||
tx_hash_dirnormal = strip_0x(tx.hash).upper()
|
||||
tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
|
||||
self.tx_raw_dir.add(tx_hash_bytes, raw)
|
||||
address = bytes.fromhex(strip_0x(tx.inputs[0]))
|
||||
self.address_dir.add_dir(tx_hash_dirnormal, address, b'')
|
||||
address = bytes.fromhex(strip_0x(tx.outputs[0]))
|
||||
self.address_dir.add_dir(tx_hash_dirnormal, address, b'')
|
||||
if self.include_tx_data:
|
||||
src = json.dumps(tx.src()).encode('utf-8')
|
||||
self.tx_dir.add(bytes.fromhex(strip_0x(tx.hash)), src)
|
||||
self.store.put_tx(tx, include_data=self.include_tx_data)
|
||||
|
@ -24,10 +24,10 @@ class EtherscanImporter:
|
||||
|
||||
|
||||
def get(self, address):
|
||||
#f = open('sample_import.json', 'r')
|
||||
#o = json.load(f)
|
||||
#f.close()
|
||||
o = self.get_api(address)
|
||||
f = open('sample_import.json', 'r')
|
||||
o = json.load(f)
|
||||
f.close()
|
||||
#o = self.get_api(address)
|
||||
|
||||
for v in o['result']:
|
||||
o = block_by_hash(v['blockHash'])
|
||||
|
@ -12,6 +12,7 @@ from chainlib.chain import ChainSpec
|
||||
# local imports
|
||||
from eth_monitor.filters.cache import Filter as CacheFilter
|
||||
from eth_monitor.filters import RuledFilter
|
||||
from eth_monitor.store.file import FileStore
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
@ -21,6 +22,7 @@ normalize_address = TxHexNormalizer().wallet_address
|
||||
argparser = argparse.ArgumentParser('master eth events monitor')
|
||||
argparser.add_argument('--api-key-file', dest='api_key_file', type=str, help='File to read API key from')
|
||||
argparser.add_argument('--cache-dir', dest='cache_dir', type=str, help='Directory to store tx data')
|
||||
argparser.add_argument('--include-data', dest='include_data', action='store_true', help='Include data objects')
|
||||
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='evm:ethereum:1', help='Chain specification string')
|
||||
argparser.add_argument('-f', '--address-file', dest='address_file', default=[], type=str, action='append', help='Add addresses from file')
|
||||
argparser.add_argument('-a', '--address', default=[], type=str, action='append', help='Add address')
|
||||
@ -80,12 +82,14 @@ def collect_addresses(addresses=[], address_files=[]):
|
||||
|
||||
|
||||
def setup_filter(chain_spec, cache_dir):
|
||||
store = FileStore(chain_spec, cache_dir)
|
||||
cache_dir = os.path.realpath(cache_dir)
|
||||
if cache_dir == None:
|
||||
import tempfile
|
||||
cache_dir = tempfile.mkdtemp()
|
||||
logg.info('using chain spec {} and dir {}'.format(chain_spec, cache_dir))
|
||||
RuledFilter.init(chain_spec, cache_dir)
|
||||
include_data = bool(args.include_data)
|
||||
RuledFilter.init(store, include_tx_data=include_data, include_block_data=include_data)
|
||||
|
||||
|
||||
def main():
|
||||
|
70
eth_monitor/store/file.py
Normal file
70
eth_monitor/store/file.py
Normal file
@ -0,0 +1,70 @@
|
||||
# standard imports
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from hexathon import strip_0x
|
||||
from chainlib.eth.tx import (
|
||||
Tx,
|
||||
pack,
|
||||
)
|
||||
from chainsyncer.backend.file import chain_dir_for
|
||||
from leveldir.numeric import NumDir
|
||||
from leveldir.hex import HexDir
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
base_dir = '/var/lib'
|
||||
|
||||
|
||||
class FileStore:
|
||||
|
||||
def put_tx(self, tx, include_data=False):
|
||||
raw = pack(tx.src(), self.chain_spec)
|
||||
tx_hash_dirnormal = strip_0x(tx.hash).upper()
|
||||
tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
|
||||
self.tx_raw_dir.add(tx_hash_bytes, raw)
|
||||
address = bytes.fromhex(strip_0x(tx.inputs[0]))
|
||||
self.address_dir.add_dir(tx_hash_dirnormal, address, b'')
|
||||
address = bytes.fromhex(strip_0x(tx.outputs[0]))
|
||||
self.address_dir.add_dir(tx_hash_dirnormal, address, b'')
|
||||
if include_data:
|
||||
src = json.dumps(tx.src()).encode('utf-8')
|
||||
self.tx_dir.add(bytes.fromhex(strip_0x(tx.hash)), src)
|
||||
|
||||
|
||||
def put_block(self, block, include_data=False):
|
||||
hash_bytes = bytes.fromhex(strip_0x(block.hash))
|
||||
self.block_num_dir.add(block.number, hash_bytes)
|
||||
num_bytes = block.number.to_bytes(8, 'big')
|
||||
self.block_hash_dir.add(hash_bytes, num_bytes)
|
||||
if include_data:
|
||||
src = json.dumps(block.src()).encode('utf-8')
|
||||
self.block_src_dir.add(hash_bytes, src)
|
||||
|
||||
|
||||
def __init__(self, chain_spec, cache_root=base_dir):
|
||||
self.cache_root = os.path.join(
|
||||
cache_root,
|
||||
'eth_monitor',
|
||||
chain_spec.engine(),
|
||||
chain_spec.fork(),
|
||||
str(chain_spec.chain_id()),
|
||||
)
|
||||
self.cache_root = os.path.realpath(self.cache_root)
|
||||
self.chain_dir = chain_dir_for(self.cache_root)
|
||||
self.cache_dir = os.path.join(self.chain_dir, 'cache')
|
||||
self.block_src_path = os.path.join(self.cache_dir, 'block', 'src')
|
||||
self.block_src_dir = HexDir(self.block_src_path, 32, levels=2)
|
||||
self.block_num_path = os.path.join(self.cache_dir, 'block', 'num')
|
||||
self.block_num_dir = NumDir(self.block_num_path, [100000, 1000])
|
||||
self.block_hash_path = os.path.join(self.cache_dir, 'block', 'hash')
|
||||
self.block_hash_dir = HexDir(self.block_hash_path, 32, levels=2)
|
||||
self.tx_path = os.path.join(self.cache_dir, 'tx', 'src')
|
||||
self.tx_raw_path = os.path.join(self.cache_dir, 'tx', 'raw')
|
||||
self.tx_dir = HexDir(self.tx_path, 32, levels=2)
|
||||
self.tx_raw_dir = HexDir(self.tx_raw_path, 32, levels=2)
|
||||
self.address_path = os.path.join(self.cache_dir, 'address')
|
||||
self.address_dir = HexDir(self.address_path, 20, levels=2)
|
||||
self.chain_spec = chain_spec
|
Loading…
Reference in New Issue
Block a user