mirror of
git://holbrook.no/eth-monitor.git
synced 2024-11-27 05:36:46 +01:00
467 lines
15 KiB
Python
467 lines
15 KiB
Python
# standard imports
|
||
import logging
|
||
import os
|
||
import uuid
|
||
import importlib
|
||
import tempfile
|
||
|
||
# external imports
|
||
from chainlib.settings import ChainSettings
|
||
from chainlib.eth.connection import EthHTTPConnection
|
||
from chainsyncer.settings import *
|
||
from eth_monitor.chain import EthChainInterface
|
||
from chainlib.eth.address import is_address
|
||
from eth_cache.rpc import CacheRPC
|
||
from eth_cache.store.file import FileStore
|
||
from chainsyncer.settings import process_sync_range
|
||
|
||
|
||
# local imports
|
||
from eth_monitor.rules import (
|
||
AddressRules,
|
||
RuleSimple,
|
||
RuleMethod,
|
||
RuleData,
|
||
RuleHash,
|
||
)
|
||
from eth_monitor.cli.rules import to_config_names
|
||
from eth_monitor.callback import (
|
||
state_change_callback,
|
||
filter_change_callback,
|
||
BlockCallbackFilter,
|
||
)
|
||
from eth_monitor.filters import RuledFilter
|
||
from eth_monitor.filters.cache import Filter as CacheFilter
|
||
from eth_monitor.config import override, list_from_prefix
|
||
from eth_monitor.filters.out import OutFilter
|
||
from eth_monitor.filters.block import Filter as BlockFilter
|
||
from eth_monitor.filters.run import Filter as RunFilter
|
||
from eth_monitor.cache import from_cache_spec
|
||
|
||
logg = logging.getLogger(__name__)
|
||
|
||
|
||
def process_monitor_session(settings, config):
|
||
session_id = config.get('_SESSION_ID')
|
||
if session_id == None:
|
||
if config.get('_SINGLE'):
|
||
session_id = str(uuid.uuid4())
|
||
else:
|
||
session_id = 'default'
|
||
|
||
settings.set('SESSION_ID', session_id)
|
||
settings.set('SESSION_OK', True)
|
||
return settings
|
||
|
||
|
||
def process_monitor_rundir(settings, config):
|
||
settings.set('RUN_OUT', False)
|
||
if config.get('_RUN_DIR') == None:
|
||
return settings
|
||
|
||
run_dir = config.get('_RUN_DIR')
|
||
try:
|
||
os.makedirs(run_dir, exist_ok=True)
|
||
except Exception as e:
|
||
logg.error('could not create run dir, deactivating run output: ' + str(e))
|
||
return settings
|
||
|
||
lockfile = os.path.join(run_dir, '.lock')
|
||
try:
|
||
f = open(lockfile, 'x')
|
||
f.close()
|
||
except FileExistsError:
|
||
logg.error('run dir {} is already in use, deactivating run output'.format(run_dir))
|
||
return settings
|
||
|
||
settings.set('RUN_OUT', True)
|
||
settings.set('RUN_DIR', run_dir)
|
||
return settings
|
||
|
||
|
||
def process_monitor_session_dir(settings, config):
|
||
syncer_store_module = None
|
||
syncer_store_class = None
|
||
sync_store = None
|
||
session_id = settings.get('SESSION_ID')
|
||
state_dir = None
|
||
if config.get('SYNCER_BACKEND') == 'mem':
|
||
syncer_store_module = importlib.import_module('chainsyncer.store.mem')
|
||
syncer_store_class = getattr(syncer_store_module, 'SyncMemStore')
|
||
sync_store = syncer_store_class(
|
||
session_id=session_id,
|
||
state_event_callback=state_change_callback,
|
||
filter_state_event_callback=filter_change_callback,
|
||
)
|
||
|
||
else:
|
||
if config.get('SYNCER_BACKEND') == 'fs':
|
||
syncer_store_module = importlib.import_module('chainsyncer.store.fs')
|
||
syncer_store_class = getattr(syncer_store_module, 'SyncFsStore')
|
||
elif config.get('SYNCER_BACKEND') == 'rocksdb':
|
||
syncer_store_module = importlib.import_module('chainsyncer.store.rocksdb')
|
||
syncer_store_class = getattr(syncer_store_module, 'SyncRocksDbStore')
|
||
else:
|
||
syncer_store_module = importlib.import_module(config.get('SYNCER_BACKEND'))
|
||
syncer_store_class = getattr(syncer_store_module, 'SyncStore')
|
||
state_dir = os.path.join(config.get('ETHMONITOR_STATE_DIR'), config.get('SYNCER_BACKEND'))
|
||
os.makedirs(state_dir, exist_ok=True)
|
||
session_dir = os.path.join(state_dir, session_id)
|
||
sync_store = syncer_store_class(
|
||
session_dir,
|
||
session_id=session_id,
|
||
state_event_callback=state_change_callback,
|
||
filter_state_event_callback=filter_change_callback,
|
||
)
|
||
settings.set('SESSION_DIR', session_dir)
|
||
|
||
logg.info('using engine {} module {}.{}'.format(config.get('SYNCER_BACKEND'), syncer_store_module.__file__, syncer_store_class.__name__))
|
||
|
||
settings.set('STATE_DIR', state_dir)
|
||
settings.set('SYNC_STORE', sync_store)
|
||
|
||
return settings
|
||
|
||
|
||
def process_address_arg_rules(settings, config):
|
||
rules = settings.get('RULES')
|
||
category = {
|
||
'input': {
|
||
'i': [],
|
||
'x': [],
|
||
},
|
||
'output': {
|
||
'i': [],
|
||
'x': [],
|
||
},
|
||
'exec': {
|
||
'i': [],
|
||
'x': [],
|
||
},
|
||
'hash': {
|
||
'i': [],
|
||
'x': [],
|
||
},
|
||
}
|
||
for rules_arg in [
|
||
'input',
|
||
'output',
|
||
'exec',
|
||
]:
|
||
(vy, vn) = to_config_names(rules_arg)
|
||
for address in config.get(vy):
|
||
if not is_address(address):
|
||
raise ValueError('invalid address in config {}: {}'.format(vy, address))
|
||
category[rules_arg]['i'].append(address)
|
||
for address in config.get(vn):
|
||
if not is_address(address):
|
||
raise ValueError('invalid address in config {}: {}'.format(vn, address))
|
||
category[rules_arg]['x'].append(address)
|
||
|
||
includes = RuleSimple(
|
||
category['output']['i'],
|
||
category['input']['i'],
|
||
category['exec']['i'],
|
||
description='INCLUDE',
|
||
match_all=settings.get('MATCH_ALL'),
|
||
)
|
||
rules.include(includes)
|
||
|
||
excludes = RuleSimple(
|
||
category['output']['x'],
|
||
category['input']['x'],
|
||
category['exec']['x'],
|
||
description='EXCLUDE',
|
||
)
|
||
rules.exclude(excludes)
|
||
|
||
return settings
|
||
|
||
|
||
def process_data_arg_rules(settings, config):
|
||
rules = settings.get('RULES')
|
||
|
||
include_data = []
|
||
for v in config.get('ETHMONITOR_DATA'):
|
||
include_data.append(v.lower())
|
||
exclude_data = []
|
||
for v in config.get('ETHMONITOR_X_DATA'):
|
||
exclude_data.append(v.lower())
|
||
|
||
includes = RuleMethod(include_data, description='INCLUDE')
|
||
rules.include(includes)
|
||
|
||
excludes = RuleMethod(exclude_data, description='EXCLUDE')
|
||
rules.exclude(excludes)
|
||
|
||
include_data = []
|
||
for v in config.get('ETHMONITOR_DATA_IN'):
|
||
include_data.append(v.lower())
|
||
exclude_data = []
|
||
for v in config.get('ETHMONITOR_X_DATA_IN'):
|
||
exclude_data.append(v.lower())
|
||
|
||
includes = RuleData(include_data, description='INCLUDE', match_all=settings.get('MATCH_ALL'))
|
||
rules.include(includes)
|
||
|
||
excludes = RuleData(exclude_data, description='EXCLUDE')
|
||
rules.exclude(excludes)
|
||
|
||
return settings
|
||
|
||
|
||
def process_tx_hash_rules(settings, config):
|
||
rules = settings.get('RULES')
|
||
|
||
include_tx = []
|
||
for v in config.get('ETHMONITOR_TXHASH'):
|
||
include_tx.append(v.lower())
|
||
exclude_tx = []
|
||
for v in config.get('ETHMONITOR_X_TXHASH'):
|
||
exclude_tx.append(v.lower())
|
||
|
||
includes = RuleHash(include_tx, description='INCLUDE')
|
||
rules.include(includes)
|
||
|
||
excludes = RuleHash(exclude_tx, description='EXCLUDE')
|
||
rules.exclude(excludes)
|
||
|
||
return settings
|
||
|
||
|
||
def process_address_file_rules(settings, config): #rules, includes_file=None, excludes_file=None, include_default=False, include_block_default=False):
|
||
rules = settings.get('RULES')
|
||
includes_file = config.get('ETHMONITOR_INCLUDES_FILE')
|
||
if includes_file != None:
|
||
f = open(includes_file, 'r')
|
||
logg.debug('reading includes rules from {}'.format(os.path.realpath(includes_file)))
|
||
while True:
|
||
r = f.readline()
|
||
if r == '':
|
||
break
|
||
r = r.rstrip()
|
||
v = r.split("\t")
|
||
|
||
sender = []
|
||
recipient = []
|
||
executable = []
|
||
|
||
try:
|
||
if v[0] != '':
|
||
sender = v[0].split(',')
|
||
except IndexError:
|
||
pass
|
||
|
||
try:
|
||
if v[1] != '':
|
||
recipient = v[1].split(',')
|
||
except IndexError:
|
||
pass
|
||
|
||
try:
|
||
if v[2] != '':
|
||
executable = v[2].split(',')
|
||
except IndexError:
|
||
pass
|
||
|
||
rule = RuleSimple(sender, recipient, executable, match_all=settings.get('MATCH_ALL'))
|
||
rules.include(rule)
|
||
|
||
excludes_file = config.get('ETHMONITOR_EXCLUDES_FILE')
|
||
if excludes_file != None:
|
||
f = open(includes_file, 'r')
|
||
logg.debug('reading excludes rules from {}'.format(os.path.realpath(excludes_file)))
|
||
while True:
|
||
r = f.readline()
|
||
if r == '':
|
||
break
|
||
r = r.rstrip()
|
||
v = r.split("\t")
|
||
|
||
sender = None
|
||
recipient = None
|
||
executable = None
|
||
|
||
if v[0] != '':
|
||
sender = v[0].strip(',')
|
||
if v[1] != '':
|
||
recipient = v[1].strip(',')
|
||
if v[2] != '':
|
||
executable = v[2].strip(',')
|
||
|
||
rule = RuleSimple(sender, recipient, executable)
|
||
rules.exclude(rule)
|
||
return settings
|
||
|
||
|
||
def process_arg_rules(settings, config):
|
||
address_rules = AddressRules(include_by_default=config.get('ETHMONITOR_INCLUDE_DEFAULT'))
|
||
settings.set('MATCH_ALL', config.true('ETHMONITOR_MATCH_ALL'))
|
||
settings.set('RULES', address_rules)
|
||
settings = process_address_arg_rules(settings, config)
|
||
settings = process_data_arg_rules(settings, config)
|
||
settings = process_address_file_rules(settings, config)
|
||
settings = process_tx_hash_rules(settings, config)
|
||
return settings
|
||
|
||
|
||
def process_cache_store(settings, config):
|
||
cache_spec = config.get('_CACHE_SPEC')
|
||
store = from_cache_spec(settings.get('CHAIN_SPEC'), cache_spec)
|
||
if store == None:
|
||
cache_dir = config.get('_CACHE_DIR')
|
||
if cache_dir == None:
|
||
logg.warning('no cache dir specified, will discard everything!!')
|
||
from eth_cache.store.null import NullStore
|
||
store = NullStore()
|
||
else:
|
||
store = FileStore(settings.get('CHAIN_SPEC'), cache_dir)
|
||
cache_dir = os.path.realpath(cache_dir)
|
||
if cache_dir == None:
|
||
import tempfile
|
||
cache_dir = tempfile.mkdtemp()
|
||
|
||
logg.info('using cache store {}'.format(store))
|
||
settings.set('CACHE_STORE', store)
|
||
|
||
return settings
|
||
|
||
|
||
def process_cache_filter(settings, config):
|
||
cache_store = settings.get('CACHE_STORE')
|
||
cache_rules = AddressRules(include_by_default=True)
|
||
if str(cache_store) != 'Nullstore':
|
||
cache_rules = settings.o['RULES']
|
||
fltr = CacheFilter(cache_store, rules_filter=cache_rules, include_tx_data=config.true('ETHCACHE_STORE_TX'))
|
||
sync_store = settings.get('SYNC_STORE')
|
||
sync_store.register(fltr)
|
||
|
||
fltr = BlockFilter(cache_store, include_block_data=config.true('ETHCACHE_STORE_BLOCK'))
|
||
hndlr = settings.get('BLOCK_HANDLER')
|
||
hndlr.register(fltr)
|
||
|
||
return settings
|
||
|
||
|
||
def process_run_filter(settings, config):
|
||
if not settings.get('RUN_OUT'):
|
||
return settings
|
||
fltr = RunFilter(settings.get('RUN_DIR'))
|
||
hndlr = settings.get('BLOCK_HANDLER')
|
||
hndlr.register(fltr)
|
||
return settings
|
||
|
||
def process_tx_filter(settings, config):
|
||
for fltr in list_from_prefix(config, 'filter'):
|
||
m = importlib.import_module(fltr)
|
||
fltr_object = m.Filter(rules_filter=settings.get('RULES'))
|
||
store = settings.get('SYNC_STORE')
|
||
store.register(fltr_object)
|
||
logg.info('using filter module {}'.format(fltr))
|
||
return settings
|
||
|
||
|
||
def process_block_filter(settings, config):
|
||
block_filter_handler = BlockCallbackFilter()
|
||
for block_filter in list_from_prefix(config, 'block_filter'):
|
||
m = importlib.import_module(block_filter)
|
||
block_filter_handler.register(m)
|
||
logg.info('using block filter module {}'.format(block_filter))
|
||
|
||
settings.set('BLOCK_HANDLER', block_filter_handler)
|
||
return settings
|
||
|
||
|
||
def process_out_filter(settings, config):
|
||
out_filter = OutFilter(
|
||
settings.o['CHAIN_SPEC'],
|
||
rules_filter=settings.o['RULES'],
|
||
renderers=settings.o['RENDERER'],
|
||
)
|
||
store = settings.get('SYNC_STORE')
|
||
store.register(out_filter)
|
||
return settings
|
||
|
||
|
||
def process_arg_filter(settings, config):
|
||
store = settings.get('SYNC_STORE')
|
||
if config.get('ETHMONITOR_FILTER') != None:
|
||
for k in config.get('ETHMONITOR_FILTER'):
|
||
m = importlib.import_module(k)
|
||
fltr = m.Filter()
|
||
store.register(fltr)
|
||
return settings
|
||
|
||
|
||
def process_filter(settings, config):
|
||
settings.set('FILTER', [])
|
||
settings = process_renderer(settings, config)
|
||
settings = process_block_filter(settings, config)
|
||
settings = process_cache_filter(settings, config)
|
||
settings = process_run_filter(settings, config)
|
||
settings = process_tx_filter(settings, config)
|
||
settings = process_out_filter(settings, config)
|
||
settings = process_arg_filter(settings, config)
|
||
return settings
|
||
|
||
|
||
def process_renderer(settings, config):
|
||
renderers_mods = []
|
||
for renderer in config.get('ETHMONITOR_RENDERER'):
|
||
m = importlib.import_module(renderer)
|
||
renderers_mods.append(m)
|
||
logg.info('using renderer module {}'.format(renderer))
|
||
settings.set('RENDERER', renderers_mods)
|
||
return settings
|
||
|
||
|
||
def process_cache_rpc(settings, config):
|
||
if str(settings.get('CACHE_STORE')) == 'Nullstore':
|
||
logg.debug('cache store is null, cache rpc proxy will be deactivated')
|
||
return settings
|
||
if not config.true('_FRESH'):
|
||
rpc = CacheRPC(settings.get('CONN'), settings.get('CACHE_STORE'))
|
||
settings.set('CONN', rpc)
|
||
return settings
|
||
|
||
|
||
def process_sync(settings, config):
|
||
dialect_filter = settings.get('RPC_DIALECT_FILTER')
|
||
settings.set('SYNCER_INTERFACE', EthChainInterface(dialect_filter=dialect_filter, batch_limit=settings.get('RPC_BATCH_LIMIT')))
|
||
settings = process_sync_range(settings, config)
|
||
return settings
|
||
|
||
|
||
def process_cache(settings, config):
|
||
settings = process_cache_store(settings, config)
|
||
settings = process_cache_rpc(settings, config)
|
||
return settings
|
||
|
||
|
||
def process_user_context(settings, config):
|
||
ctx_usr = {}
|
||
ks = config.get('ETHMONITOR_CONTEXT_KEY')
|
||
if ks != None:
|
||
for kv in ks:
|
||
(k, v) = kv.split('=', 1)
|
||
ctx_usr[k] = v
|
||
ctx = {
|
||
'driver': 'eth-monitor',
|
||
'rundir': settings.get('RUN_DIR'),
|
||
'usr': ctx_usr,
|
||
}
|
||
settings.set('SYNCER_CONTEXT', ctx)
|
||
return settings
|
||
|
||
|
||
def process_settings(settings, config):
|
||
settings = process_monitor_session(settings, config)
|
||
settings = process_monitor_session_dir(settings, config)
|
||
settings = process_monitor_rundir(settings, config)
|
||
settings = process_arg_rules(settings, config)
|
||
settings = process_sync(settings, config)
|
||
settings = process_cache(settings, config)
|
||
settings = process_user_context(settings, config)
|
||
settings = process_filter(settings, config)
|
||
return settings
|