From c99259b2ed28e88681f5c30003af64af1152a6e1 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 17 Aug 2023 12:48:33 +0100 Subject: [PATCH] Add match-all criteria and flag --- CHANGELOG | 3 + eth_monitor/cli/arg.py | 1 + eth_monitor/cli/config.py | 2 + eth_monitor/data/config/monitor.ini | 1 + eth_monitor/error.py | 2 + eth_monitor/rules.py | 57 ++++++++-- eth_monitor/settings.py | 6 +- run_tests.sh | 17 +++ tests/rules/test_base.py | 160 ++++++++++++++++++++++++++++ tests/rules/test_greedy.py | 90 ++++++++++++++++ tests/test_filter.py | 40 ------- 11 files changed, 329 insertions(+), 50 deletions(-) create mode 100644 eth_monitor/error.py create mode 100644 run_tests.sh create mode 100644 tests/rules/test_base.py create mode 100644 tests/rules/test_greedy.py delete mode 100644 tests/test_filter.py diff --git a/CHANGELOG b/CHANGELOG index a6ced6f..490cc54 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +- 0.8.8 + * Add match-all flag to rule processing + * Add match-all flag to CLI to toggle setting match_all flag to rule processing for include criteria - 0.8.7 * Upgrade chainsyncer (and shep) to avoid state deletion on partial filter list interrupts - 0.8.6 diff --git a/eth_monitor/cli/arg.py b/eth_monitor/cli/arg.py index 7b83fd9..bc4359f 100644 --- a/eth_monitor/cli/arg.py +++ b/eth_monitor/cli/arg.py @@ -30,6 +30,7 @@ def process_args(argparser, args, flags): argparser.add_argument('--store-tx-data', action='store_true', dest='store_tx_data', help='Store tx data in cache store') argparser.add_argument('--store-block-data', action='store_true', dest='store_block_data', help='Store block data in cache store') argparser.add_argument('--fresh', action='store_true', help='Do not read block and tx data from cache, even if available') + argparser.add_argument('--match-all', action='store_true', dest='match_all', help='Match all include filter criteria') # misc flags argparser.add_argument('-k', '--context-key', dest='context_key', action='append', type=str, help='Add a key-value pair to be added to the context') diff --git a/eth_monitor/cli/config.py b/eth_monitor/cli/config.py index 40f4295..59a069b 100644 --- a/eth_monitor/cli/config.py +++ b/eth_monitor/cli/config.py @@ -40,6 +40,8 @@ def process_config(config, arg, args, flags): arg_override['ETHMONITOR_CONTEXT_KEY'] = getattr(args, 'context_key') + arg_override['ETHMONITOR_MATCH_ALL'] = getattr(args, 'match_all') + arg_override['ETHCACHE_STORE_BLOCK'] = getattr(args, 'store_block_data') arg_override['ETHCACHE_STORE_TX'] = getattr(args, 'store_tx_data') diff --git a/eth_monitor/data/config/monitor.ini b/eth_monitor/data/config/monitor.ini index ec993c1..e3d3fcc 100644 --- a/eth_monitor/data/config/monitor.ini +++ b/eth_monitor/data/config/monitor.ini @@ -19,3 +19,4 @@ block_filter = include_default = 0 state_dir = ./.eth-monitor context_key = +match_all = 0 diff --git a/eth_monitor/error.py b/eth_monitor/error.py new file mode 100644 index 0000000..b795811 --- /dev/null +++ b/eth_monitor/error.py @@ -0,0 +1,2 @@ +class RuleFail(Exception): + pass diff --git a/eth_monitor/rules.py b/eth_monitor/rules.py index 09074a4..b4dfaa8 100644 --- a/eth_monitor/rules.py +++ b/eth_monitor/rules.py @@ -4,6 +4,7 @@ import uuid # external imports from chainlib.eth.address import is_same_address +from .error import RuleFail logg = logging.getLogger() @@ -11,14 +12,17 @@ logg = logging.getLogger() class RuleData: - def __init__(self, fragments, description=None): + def __init__(self, fragments, description=None, match_all=False): self.fragments = fragments self.description = description if self.description == None: self.description = str(uuid.uuid4()) + self.match_all = match_all def check(self, sender, recipient, data, tx_hash): + have_fail = False + have_match = False if len(self.fragments) == 0: return False @@ -28,9 +32,16 @@ class RuleData: continue if fragment in data: logg.debug('tx {} rule {} match in DATA FRAGMENT {}'.format(tx_hash, self.description, fragment)) - return True + if not self.match_all: + return True + have_match = True + else: + logg.debug('data match all {}'.format(self.match_all)) + if self.match_all: + return False + have_fail = True - return False + return have_match def __str__(self): @@ -41,11 +52,13 @@ class RuleData: class RuleMethod: - def __init__(self, methods, description=None): + def __init__(self, methods, description=None, match_all=False): self.methods = methods self.description = description if self.description == None: self.description = str(uuid.uuid4()) + if match_all: + logg.warning('match_all ignord for RuleMethod rule') def check(self, sender, recipient, data, tx_hash): @@ -82,22 +95,51 @@ class RuleSimple: def check(self, sender, recipient, data, tx_hash): + r = None + try: + r = self.__check(sender, recipient, data, tx_hash) + except RuleFail: + return False + return r + + + def __check(self, sender, recipient, data, tx_hash): have_fail = False have_match = False for rule in self.outputs: if rule != None and is_same_address(sender, rule): logg.debug('tx {} rule {} match in SENDER {}'.format(tx_hash, self.description, sender)) - return True + if not self.match_all: + return True + have_match = True + else: + if self.match_all: + raise RuleFail(rule) + have_fail = True if recipient == None: return False for rule in self.inputs: if rule != None and is_same_address(recipient, rule): logg.debug('tx {} rule {} match in RECIPIENT {}'.format(tx_hash, self.description, recipient)) - return True + if not self.match_all: + return True + have_match = True + else: + if self.match_all: + raise RuleFail(rule) + have_fail = True for rule in self.executables: if rule != None and is_same_address(recipient, rule): logg.debug('tx {} rule {} match in EXECUTABLE {}'.format(tx_hash, self.description, recipient)) - return True + if not self.match_all: + return True + have_match = True + else: + if self.match_all: + raise RuleFail(rule) + have_fail = True + + return have_match def __str__(self): @@ -131,7 +173,6 @@ class AddressRules: return self.apply_rules_addresses(tx.outputs[0], tx.inputs[0], tx.payload, tx.hash) - # TODO: rename def apply_rules_addresses(self, sender, recipient, data, tx_hash): v = self.include_by_default have_fail = False diff --git a/eth_monitor/settings.py b/eth_monitor/settings.py index 30355b7..0704377 100644 --- a/eth_monitor/settings.py +++ b/eth_monitor/settings.py @@ -130,6 +130,7 @@ def process_address_arg_rules(settings, config): category['input']['i'], category['exec']['i'], description='INCLUDE', + match_all=settings.get('MATCH_ALL'), ) rules.include(includes) @@ -167,7 +168,7 @@ def process_data_arg_rules(settings, config): for v in config.get('ETHMONITOR_X_DATA_IN'): exclude_data.append(v.lower()) - includes = RuleData(include_data, description='INCLUDE') + includes = RuleData(include_data, description='INCLUDE', match_all=settings.get('MATCH_ALL')) rules.include(includes) excludes = RuleData(exclude_data, description='EXCLUDE') @@ -211,7 +212,7 @@ def process_address_file_rules(settings, config): #rules, includes_file=None, ex except IndexError: pass - rule = RuleSimple(sender, recipient, executable) + rule = RuleSimple(sender, recipient, executable, match_all=settings.get('MATCH_ALL')) rules.include(rule) excludes_file = config.get('ETHMONITOR_EXCLUDES_FILE') @@ -243,6 +244,7 @@ def process_address_file_rules(settings, config): #rules, includes_file=None, ex def process_arg_rules(settings, config): address_rules = AddressRules(include_by_default=config.get('ETHMONITOR_INCLUDE_DEFAULT')) + settings.set('MATCH_ALL', config.true('ETHMONITOR_MATCH_ALL')) settings.set('RULES', address_rules) settings = process_address_arg_rules(settings, config) settings = process_data_arg_rules(settings, config) diff --git a/run_tests.sh b/run_tests.sh new file mode 100644 index 0000000..a751c1c --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -a +set -e +set -x +default_pythonpath=$PYTHONPATH:. +export PYTHONPATH=${default_pythonpath:-.} +>&2 echo using pythonpath $PYTHONPATH +for f in `ls tests/*.py`; do + python $f +done +for f in `ls tests/rules/*.py`; do + python $f +done +set +x +set +e +set +a diff --git a/tests/rules/test_base.py b/tests/rules/test_base.py new file mode 100644 index 0000000..aa06ca6 --- /dev/null +++ b/tests/rules/test_base.py @@ -0,0 +1,160 @@ +# standard imports +import logging +import unittest +import os + +# local imports +from eth_monitor.rules import * + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + + +class TestRule(unittest.TestCase): + + def setUp(self): + self.alice = os.urandom(20).hex() + self.bob = os.urandom(20).hex() + self.carol = os.urandom(20).hex() + self.dave = os.urandom(20).hex() + self.x = os.urandom(20).hex() + self.y = os.urandom(20).hex() + self.hsh = os.urandom(32).hex() + + + def test_address_include(self): + data = b'' + outs = [self.alice] + ins = [] + execs = [] + rule = RuleSimple(outs, ins, execs) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertTrue(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertFalse(r) + + outs = [] + ins = [self.alice] + execs = [] + rule = RuleSimple(outs, ins, execs) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertTrue(r) + + outs = [] + ins = [] + execs = [self.x] + rule = RuleSimple(outs, ins, execs) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh) + self.assertTrue(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertFalse(r) + + data = b'deadbeef0123456789' + data_match = [data[:8]] + rule = RuleMethod(data_match) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh) + self.assertTrue(r) + r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh) + self.assertFalse(r) + + rule = RuleData(data_match) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh) + self.assertTrue(r) + r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh) + self.assertTrue(r) + + + def test_address_exclude(self): + data = b'' + outs = [self.alice] + ins = [] + execs = [] + rule = RuleSimple(outs, ins, execs) + + c = AddressRules() + c.exclude(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertFalse(r) + + c = AddressRules(include_by_default=True) + c.exclude(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertTrue(r) + + outs = [] + ins = [self.alice] + execs = [] + rule = RuleSimple(outs, ins, execs) + c = AddressRules(include_by_default=True) + c.exclude(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertTrue(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertFalse(r) + + outs = [] + ins = [] + execs = [self.x] + rule = RuleSimple(outs, ins, execs) + c = AddressRules(include_by_default=True) + c.exclude(rule) + r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertTrue(r) + + data = b'deadbeef0123456789' + data_match = [data[:8]] + rule = RuleMethod(data_match) + c = AddressRules(include_by_default=True) + c.exclude(rule) + r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh) + self.assertTrue(r) + + rule = RuleData(data_match) + c = AddressRules(include_by_default=True) + c.exclude(rule) + r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, b'abcd', self.hsh) + self.assertTrue(r) + + + def test_address_include_exclude(self): + data = b'' + outs = [self.alice] + ins = [] + execs = [] + rule = RuleSimple(outs, ins, execs) + c = AddressRules() + c.include(rule) + c.exclude(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertFalse(r) + + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/rules/test_greedy.py b/tests/rules/test_greedy.py new file mode 100644 index 0000000..5e7216a --- /dev/null +++ b/tests/rules/test_greedy.py @@ -0,0 +1,90 @@ +import logging +import unittest +import os + +# local imports +from eth_monitor.rules import * + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + + +class TestRule(unittest.TestCase): + + def setUp(self): + self.alice = os.urandom(20).hex() + self.bob = os.urandom(20).hex() + self.carol = os.urandom(20).hex() + self.dave = os.urandom(20).hex() + self.x = os.urandom(20).hex() + self.y = os.urandom(20).hex() + self.hsh = os.urandom(32).hex() + + + def test_greedy_includes(self): + data = b'' + outs = [self.alice] + ins = [self.carol] + execs = [] + rule = RuleSimple(outs, ins, execs, match_all=True) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.carol, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.alice, self.carol, data, self.hsh) + self.assertTrue(r) + + rule = RuleSimple(outs, ins, execs) + c = AddressRules(match_all=True) + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertTrue(r) + r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) + self.assertFalse(r) + r = c.apply_rules_addresses(self.bob, self.carol, data, self.hsh) + self.assertTrue(r) + r = c.apply_rules_addresses(self.alice, self.carol, data, self.hsh) + self.assertTrue(r) + + + def test_greedy_data(self): + data = os.urandom(128).hex() + data_match_one = data[4:8] + data_match_two = data[32:42] + data_match_fail = os.urandom(64).hex() + data_match = [data_match_one] + + rule = RuleData(data_match, match_all=True) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertTrue(r) + + data_match = [data_match_two] + rule = RuleData(data_match, match_all=True) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertTrue(r) + + data_match = [data_match_two, data_match_one] + rule = RuleData(data_match, match_all=True) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertTrue(r) + + data_match = [data_match_two, data_match_fail, data_match_one] + rule = RuleData(data_match, match_all=True) + c = AddressRules() + c.include(rule) + r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) + self.assertFalse(r) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_filter.py b/tests/test_filter.py deleted file mode 100644 index b28c84e..0000000 --- a/tests/test_filter.py +++ /dev/null @@ -1,40 +0,0 @@ -# standard imports -import logging -import unittest -import os - -# local imports -from eth_monitor.rules import * - -logging.basicConfig(level=logging.DEBUG) -logg = logging.getLogger() - - -class TestRule(unittest.TestCase): - - def setUp(self): - self.alice = os.urandom(20).hex() - self.bob = os.urandom(20).hex() - self.carol = os.urandom(20).hex() - self.dave = os.urandom(20).hex() - self.x = os.urandom(20).hex() - self.y = os.urandom(20).hex() - self.hsh = os.urandom(32).hex() - - - def test_address_include(self): - outs = [self.alice] - ins = [] - execs = [] - rule = RuleSimple(outs, ins, execs) - c = AddressRules() - c.include(rule) - data = b'' - r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh) - self.assertTrue(r) - r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh) - self.assertFalse(r) - - -if __name__ == '__main__': - unittest.main()