Bump version

CHANGELOG

@@ -1,3 +1,19 @@
+- 0.8.8
+	* Skip rules filter processing for cache when deactivated
+	* Add match-all flag to rule processing
+	* Add match-all flag to CLI to toggle setting match_all flag to rule processing for include criteria
+	* Implement state dir (rundir), and last synced block output
+- 0.8.7
+	* Upgrade chainsyncer (and shep) to avoid state deletion on partial filter list interrupts
+- 0.8.6
+	* Handle crash on conrtact creation when recipient filter is active
+- 0.8.5
+	* Instantiate constructor for chain interface superclass
+	* Remove unused settings transform method for sync interface
+- 0.8.4
+	* Update man pages with rpc batch limit setting
+- 0.8.3
+	* Update man pages
 - 0.8.2
 	* Handle undefined content-key argument
 - 0.8.1

eth_monitor/chain.py

@@ -8,15 +8,19 @@ from chainlib.eth.block import (
 from chainlib.eth.tx import (
         receipt,
         Tx,
+        transaction,
         )
 
 class EthChainInterface(ChainInterface):
 
     def __init__(self, dialect_filter=None, batch_limit=1):
+        super(EthChainInterface, self).__init__(dialect_filter=dialect_filter, batch_limit=batch_limit)
         self.batch_limit = batch_limit
         self._block_latest = block_latest
         self._block_by_number = block_by_number
         self._block_from_src = Block.from_src
+        self._tx_from_src = Tx.from_src
         self._tx_receipt = receipt
         self._src_normalize = Tx.src_normalize
         self._dialect_filter = dialect_filter
+        self._tx_by_hash = transaction

eth_monitor/cli/arg.py

@@ -30,6 +30,8 @@ def process_args(argparser, args, flags):
     argparser.add_argument('--store-tx-data', action='store_true', dest='store_tx_data', help='Store tx data in cache store')
     argparser.add_argument('--store-block-data', action='store_true', dest='store_block_data', help='Store block data in cache store')
     argparser.add_argument('--fresh', action='store_true', help='Do not read block and tx data from cache, even if available')
+    argparser.add_argument('--match-all', action='store_true', dest='match_all', help='Match all include filter criteria')
 
     # misc flags
     argparser.add_argument('-k', '--context-key', dest='context_key', action='append', type=str, help='Add a key-value pair to be added to the context')
+    argparser.add_argument('--run-dir', type=str, dest='run_dir', help='Output key sync and processing state properties to given diretory')

eth_monitor/cli/config.py

@@ -40,6 +40,8 @@ def process_config(config, arg, args, flags):
 
     arg_override['ETHMONITOR_CONTEXT_KEY'] = getattr(args, 'context_key')
 
+    arg_override['ETHMONITOR_MATCH_ALL'] = getattr(args, 'match_all')
+
     arg_override['ETHCACHE_STORE_BLOCK'] = getattr(args, 'store_block_data')
     arg_override['ETHCACHE_STORE_TX'] = getattr(args, 'store_tx_data')
 
@@ -54,6 +56,7 @@ def process_config(config, arg, args, flags):
 
     config.add(getattr(args, 'session_id'), '_SESSION_ID', False)
     config.add(getattr(args, 'cache_dir'), '_CACHE_DIR', False)
+    config.add(getattr(args, 'run_dir'), '_RUN_DIR', False)
     config.add(getattr(args, 'fresh'), '_FRESH', False)
 
     return config

eth_monitor/cli/rules.py

@@ -14,5 +14,3 @@ rules_data_args = [
 def to_config_names(v):
     v = v.upper()
     return ('ETHMONITOR_' + v, 'ETHMONITOR_X_' + v)
-
-

eth_monitor/data/config/monitor.ini

@@ -19,3 +19,4 @@ block_filter =
 include_default = 0
 state_dir = ./.eth-monitor
 context_key =
+match_all = 0

eth_monitor/error.py

@@ -0,0 +1,2 @@
+class RuleFail(Exception):
+    pass

eth_monitor/filters/block.py

@@ -1,3 +1,7 @@
+# standard imports
+import os
+
+
 class Filter:
 
     def __init__(self, store, include_block_data=False):
@@ -5,5 +9,5 @@ class Filter:
         self.include_block_data = include_block_data
 
 
-    def filter(self, conn, block):
+    def filter(self, conn, block, **kwargs):
         self.store.put_block(block, include_data=self.include_block_data)

eth_monitor/filters/run.py

@@ -0,0 +1,16 @@
+# standard imports
+import os
+
+
+class Filter:
+
+    def __init__(self, run_dir):
+        self.run_dir = run_dir
+        self.fp = os.path.join(run_dir, 'block')
+
+
+    def filter(self, conn, block):
+        f = open(self.fp, 'w')
+        f.write(str(block.number))
+        f.close()
+        return False

eth_monitor/rules.py

@@ -4,6 +4,7 @@ import uuid
 
 # external imports
 from chainlib.eth.address import is_same_address
+from .error import RuleFail
 
 logg = logging.getLogger()
 
@@ -11,14 +12,17 @@ logg = logging.getLogger()
 
 class RuleData:
 
-    def __init__(self, fragments, description=None):
+    def __init__(self, fragments, description=None, match_all=False):
         self.fragments = fragments
         self.description = description
         if self.description == None:
             self.description = str(uuid.uuid4())
+        self.match_all = match_all
 
 
     def check(self, sender, recipient, data, tx_hash):
+        have_fail = False
+        have_match = False
         if len(self.fragments) == 0:
             return False
 
@@ -28,9 +32,16 @@ class RuleData:
                 continue
             if fragment in data:
                 logg.debug('tx {} rule {} match in DATA FRAGMENT {}'.format(tx_hash, self.description, fragment))
-                return True
+                if not self.match_all:
+                    return True
+                have_match = True
+            else:
+                logg.debug('data match all {}'.format(self.match_all))
+                if self.match_all:
+                    return False
+                have_fail = True
         
-        return False
+        return have_match
 
 
     def __str__(self):
@@ -41,11 +52,13 @@ class RuleData:
 
 class RuleMethod:
 
-    def __init__(self, methods, description=None):
+    def __init__(self, methods, description=None, match_all=False):
         self.methods = methods
         self.description = description
         if self.description == None:
             self.description = str(uuid.uuid4())
+        if match_all:
+            logg.warning('match_all ignord for RuleMethod rule')
 
 
     def check(self, sender, recipient, data, tx_hash):
@@ -71,28 +84,62 @@ class RuleMethod:
 
 class RuleSimple:
 
-    def __init__(self, outputs, inputs, executables, description=None):
+    def __init__(self, outputs, inputs, executables, description=None, match_all=False):
         self.description = description
         if self.description == None:
             self.description = str(uuid.uuid4())
         self.outputs = outputs
         self.inputs = inputs
         self.executables = executables
+        self.match_all = match_all
 
-
+    
     def check(self, sender, recipient, data, tx_hash):
+        r = None
+        try:
+            r = self.__check(sender, recipient, data, tx_hash)
+        except RuleFail:
+            return False
+        return r
+
+
+    def __check(self, sender, recipient, data, tx_hash):
+        have_fail = False
+        have_match = False
         for rule in self.outputs:
             if rule != None and is_same_address(sender, rule):
                 logg.debug('tx {} rule {} match in SENDER {}'.format(tx_hash, self.description, sender))
-                return True
+                if not self.match_all:
+                    return True
+                have_match = True
+            else:
+                if self.match_all:
+                    raise RuleFail(rule)
+                have_fail = True
+        if recipient == None:
+            return False
         for rule in self.inputs:
             if rule != None and is_same_address(recipient, rule):
                 logg.debug('tx {} rule {} match in RECIPIENT {}'.format(tx_hash, self.description, recipient))
-                return True
+                if not self.match_all:
+                    return True
+                have_match = True
+            else:
+                if self.match_all:
+                    raise RuleFail(rule)
+                have_fail = True
         for rule in self.executables:
             if rule != None and is_same_address(recipient, rule):
                 logg.debug('tx {} rule {} match in EXECUTABLE {}'.format(tx_hash, self.description, recipient))
-                return True
+                if not self.match_all:
+                    return True
+                have_match = True
+            else:
+                if self.match_all:
+                    raise RuleFail(rule)
+                have_fail = True
+
+        return have_match
 
 
     def __str__(self):
@@ -105,10 +152,11 @@ class RuleSimple:
 
 class AddressRules:
 
-    def __init__(self, include_by_default=False):
+    def __init__(self, include_by_default=False, match_all=False):
         self.excludes = []
         self.includes = []
         self.include_by_default = include_by_default
+        self.match_all = match_all
 
 
     def exclude(self, rule):
@@ -125,20 +173,29 @@ class AddressRules:
         return self.apply_rules_addresses(tx.outputs[0], tx.inputs[0], tx.payload, tx.hash)
 
 
-    # TODO: rename
     def apply_rules_addresses(self, sender, recipient, data, tx_hash):
         v = self.include_by_default
+        have_fail = False
+        have_match = False
 
         for rule in self.includes:
             if rule.check(sender, recipient, data, tx_hash):
                 v = True
                 logg.info('match in includes rule: {}'.format(rule))
+                if not self.match_all:
+                    break
+            elif self.match_all:
+                v = False
                 break
 
+        if not v:
+            return v
+
         for rule in self.excludes:
             if rule.check(sender, recipient, data, tx_hash):
                 v = False
                 logg.info('match in excludes rule: {}'.format(rule))
-                break
-
+                if not self.match_all:
+                    break
+    
         return v

eth_monitor/run.py

@@ -0,0 +1,17 @@
+# standard imports
+import os
+import logging
+
+logg = logging.getLogger(__name__)
+
+
+def cleanup_run(settings):
+    if not settings.get('RUN_OUT'):
+        return
+    lockfile = os.path.join(settings.get('RUN_DIR'), '.lock')
+    os.unlink(lockfile)
+    logg.debug('freed rundir {}'.format(settings.get('RUN_DIR')))
+
+
+def cleanup(settings):
+    cleanup_run(settings)

eth_monitor/runnable/sync.py

@@ -47,6 +47,7 @@ from eth_monitor.callback import (
 import eth_monitor.cli
 from eth_monitor.cli.log import process_log
 from eth_monitor.settings import process_settings as process_settings_local
+from eth_monitor.run import cleanup
 
 logg = logging.getLogger()
 
@@ -111,6 +112,8 @@ def main():
     except SyncDone as e:
         sys.stderr.write("sync {} done at block {}\n".format(drv, e))
 
+    cleanup(settings)
+
 
 if __name__ == '__main__':
     main()

eth_monitor/settings.py

@@ -34,6 +34,7 @@ from eth_monitor.filters.cache import Filter as CacheFilter
 from eth_monitor.config import override, list_from_prefix
 from eth_monitor.filters.out import OutFilter
 from eth_monitor.filters.block import Filter as BlockFilter
+from eth_monitor.filters.run import Filter as RunFilter
 
 logg = logging.getLogger(__name__)
 
@@ -47,6 +48,32 @@ def process_monitor_session(settings, config):
             session_id = 'default'
     
     settings.set('SESSION_ID', session_id)
+    settings.set('SESSION_OK', True)
+    return settings
+
+
+def process_monitor_rundir(settings, config):
+    settings.set('RUN_OUT', False)
+    if config.get('_RUN_DIR') == None:
+        return settings
+
+    run_dir = config.get('_RUN_DIR')
+    try:
+        os.makedirs(run_dir, exist_ok=True)
+    except Exception as e:
+        logg.error('could not create run dir, deactivating run output: ' + str(e))
+        return settings
+ 
+    lockfile = os.path.join(run_dir, '.lock')
+    try:
+        f = open(lockfile, 'x')
+        f.close()
+    except FileExistsError:
+        logg.error('run dir {} is already in use, deactivating run output'.format(run_dir))
+        return settings
+  
+    settings.set('RUN_OUT', True)
+    settings.set('RUN_DIR', run_dir)
     return settings
 
 
@@ -130,6 +157,7 @@ def process_address_arg_rules(settings, config):
             category['input']['i'],
             category['exec']['i'],
             description='INCLUDE',
+            match_all=settings.get('MATCH_ALL'),
             )
     rules.include(includes)
 
@@ -167,7 +195,7 @@ def process_data_arg_rules(settings, config):
     for v in config.get('ETHMONITOR_X_DATA_IN'):
         exclude_data.append(v.lower())
 
-    includes = RuleData(include_data, description='INCLUDE')
+    includes = RuleData(include_data, description='INCLUDE', match_all=settings.get('MATCH_ALL'))
     rules.include(includes)
    
     excludes = RuleData(exclude_data, description='EXCLUDE')
@@ -211,7 +239,7 @@ def process_address_file_rules(settings, config): #rules, includes_file=None, ex
             except IndexError:
                 pass
 
-            rule = RuleSimple(sender, recipient, executable)
+            rule = RuleSimple(sender, recipient, executable, match_all=settings.get('MATCH_ALL'))
             rules.include(rule)
 
     excludes_file = config.get('ETHMONITOR_EXCLUDES_FILE')
@@ -243,6 +271,7 @@ def process_address_file_rules(settings, config): #rules, includes_file=None, ex
 
 def process_arg_rules(settings, config):
     address_rules = AddressRules(include_by_default=config.get('ETHMONITOR_INCLUDE_DEFAULT'))
+    settings.set('MATCH_ALL', config.true('ETHMONITOR_MATCH_ALL'))
     settings.set('RULES', address_rules)
     settings = process_address_arg_rules(settings, config)
     settings = process_data_arg_rules(settings, config)
@@ -272,7 +301,10 @@ def process_cache_store(settings, config):
 
 def process_cache_filter(settings, config):
     cache_store = settings.get('CACHE_STORE')
-    fltr = CacheFilter(cache_store, rules_filter=settings.o['RULES'], include_tx_data=config.true('ETHCACHE_STORE_TX'))
+    cache_rules = AddressRules(include_by_default=True)
+    if str(cache_store) != 'Nullstore':
+        cache_rules = settings.o['RULES'] 
+    fltr = CacheFilter(cache_store, rules_filter=cache_rules, include_tx_data=config.true('ETHCACHE_STORE_TX'))
     sync_store = settings.get('SYNC_STORE')
     sync_store.register(fltr)
     
@@ -283,6 +315,14 @@ def process_cache_filter(settings, config):
     return settings
 
 
+def process_run_filter(settings, config):
+    if not settings.get('RUN_OUT'):
+        return settings
+    fltr = RunFilter(settings.get('RUN_DIR'))
+    hndlr = settings.get('BLOCK_HANDLER')
+    hndlr.register(fltr)
+    return settings
+
 def process_tx_filter(settings, config):
     for fltr in list_from_prefix(config, 'filter'):
         m = importlib.import_module(fltr)
@@ -330,6 +370,7 @@ def process_filter(settings, config):
     settings = process_renderer(settings, config)
     settings = process_block_filter(settings, config)
     settings = process_cache_filter(settings, config)
+    settings = process_run_filter(settings, config)
     settings = process_tx_filter(settings, config)
     settings = process_out_filter(settings, config)
     settings = process_arg_filter(settings, config)
@@ -356,22 +397,11 @@ def process_cache_rpc(settings, config):
     return settings
 
 
-def process_sync_interface(settings, config):
-    ifc = EthChainInterface(dialect=settings.get('RPC_DIALECT_FILTER'))
-    settings.set('SYNCER_INTERFACE', ifc)
-    return settings
-
-
 def process_sync(settings, config):
     dialect_filter = settings.get('RPC_DIALECT_FILTER')
     settings.set('SYNCER_INTERFACE', EthChainInterface(dialect_filter=dialect_filter, batch_limit=settings.get('RPC_BATCH_LIMIT')))
     settings = process_sync_range(settings, config)
     return settings
-#def process_sync(settings, config):
-#    settings = process_sync_interface(settings, config)
-#    settings = process_sync_backend(settings, config)
-#    settings = process_sync_range(settings, config)
-#    return settings
 
 
 def process_cache(settings, config):
@@ -389,6 +419,7 @@ def process_user_context(settings, config):
             ctx_usr[k] = v
     ctx = {
         'driver': 'eth-monitor',
+        'rundir': settings.get('RUN_DIR'),
         'usr': ctx_usr,
             }
     settings.set('SYNCER_CONTEXT', ctx)
@@ -398,6 +429,7 @@ def process_user_context(settings, config):
 def process_settings(settings, config):
     settings = process_monitor_session(settings, config)
     settings = process_monitor_session_dir(settings, config)
+    settings = process_monitor_rundir(settings, config)
     settings = process_arg_rules(settings, config)
     settings = process_sync(settings, config)
     settings = process_cache(settings, config)

man/build/eth-monitor-import.1

@@ -0,0 +1,170 @@
+.TH eth-monitor-import	1
+
+
+.SH NAME
+eth-monitor-import \- Import transaction data from an indexing service
+
+
+.SH SYNOPSIS
+.SY eth-monitor-import
+[ -i \fIchain_spec\fP] [ --api-key-file \fIfile\fp ] [ --address-file \fIfile\fP ] [ -a \fIaddress\fP ... ] [ --cache-dir \fIdirectory\fP ] \fIservice\fP
+
+
+.SH DESCRIPTION
+Use an indexing service to retrieve transaction hashes for one or more addresses. Supported services may be listed using the \fB--list-services\fP option.
+.P
+Which addresses to retrieve data for may be defined by the \fB-a\fP \fIaddress\fP option. Alternatively, the \fB--address-file\fP \fIfile\fP option may be used, where addresses are supplied from the given file as a comma-separated list. The address matching mechanism used in transaction processing is the same as for \fBeth-monitor(1)\fP.
+.P
+Only block and transaction hashes are used from the indexing service. The RPC endpoint will be used to retrieve the block and transaction data.
+.P
+If \fB--cache-dir\fP \fIdirectory\fP is defined, data will be cached to the given path using the same caching filter as \fBeth-monitor(1)\fP. \fB--store-tx-data\fP and \fB--store-block-data-\fP define whether also transaction and block data is stored to cache, respectively.
+
+
+.SS OPTIONS
+
+.TP
+\fB-0\fP
+Omit newline to output
+
+.TP
+\fB--address-file \fI\fIfile
+\fP\fP
+Load address include matching rules from file. Addresses must be given as a comma-separated list.
+
+.TP
+\fB-c \fI\fIconfig_dir\fP\fP, \fB--config \fI\fIconfig_dir\fP\fP
+Load configuration files from given directory. All files with an .ini extension will be loaded, of which all must contain valid ini file data.
+
+.TP
+\fB--dumpconfig \fI\fIformat\fP\fP
+Output configuration settings rendered from environment and inputs. Valid arguments are \fIini\fP for ini file output, and \fIenv\fP for environment variable output. See \fBCONFIGURATION\fP.
+
+.TP
+\fB--env-prefix\fP
+Environment prefix for variables to overwrite configuration. Example: If \fB--env-prefix\fP is set to \fBFOO\fP then configuration variable \fBBAR_BAZ\fP would be set by environment variable \fBFOO_BAZ_BAR\fP. Also see \fBENVIRONMENT\fP.
+
+.TP
+\fB--height\fP
+Block height at which to query state for. Does not apply to transactions.
+
+.TP
+\fB-i \fI\fIchain_spec\fP\fP, \fB--chain-spec \fI\fIchain_spec\fP\fP
+Chain specification string, in the format <engine>:<fork>:<chain_id>:<common_name>. Example: "evm:london:1:ethereum". Overrides the \fIRPC_CREDENTIALS\fP configuration setting.
+
+.TP
+\fB--list-services \fI\fI
+\fP\fP
+List all supported services.
+
+.TP
+\fB-n \fI\fInamespace\fP\fP, \fB--namespace \fI\fInamespace\fP\fP
+Load given configuration namespace. Configuration will be loaded from the immediate configuration subdirectory with the same name.
+
+.TP
+\fB--no-logs\fP
+Turn of logging completely. Negates \fB-v\fP and \fB-vv\fP
+
+.TP
+\fB-p\fP, \fB--rpc-provider\fP
+Fully-qualified URL of RPC provider. Overrides the \fIRPC_PROVIDER\fP configuration setting.
+
+.TP
+\fB--raw\fP
+Produce output most optimized for machines.
+
+.TP
+\fB--rpc-batch-limit\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together. Overrides the \fIRPC_BATCH_LIMIT\fP configuration setting.
+
+.TP
+\fB--rpc-dialect\fP
+RPC backend dialect. If specified it \fImay\fP help with encoding and decoding issues. Overrides the \fIRPC_DIALECT\fP configuration setting.
+
+.TP
+\fB--socks-host \fI\fIhost
+\fP\fP
+Connect through the specified socks4a host (e.g. tor)
+
+.TP
+\fB--socks-port \fI\fIport
+\fP\fP
+Connect through the specified socks4a host port (e.g. tor)
+
+.TP
+\fB--store-block-data \fI\fI
+\fP\fP
+Store block data in cache for matching transactions. Requires \fB--cache-dir\fP.
+
+.TP
+\fB--store-tx-data \fI\fI
+\fP\fP
+Store transaction data in cache for matching transactions. Requires \fB--cache-dir\fP.
+
+.TP
+\fB-u\fP, \fB--unsafe\fP
+Allow addresses that do not pass checksum.
+
+.TP
+\fB-v\fP
+Verbose. Show logs for important state changes.
+
+.TP
+\fB-vv\fP
+Very verbose. Show logs with debugging information.
+
+.SH CONFIGURATION
+
+All configuration settings may be overriden both by environment variables, or by overriding settings with the contents of ini-files in the directory defined by the \fB-c\fP option.
+
+The active configuration, with values assigned from environment and arguments, can be output using the \fB--dumpconfig\fP \fIformat\fP option. Note that entries having keys prefixed with underscore (e.g. _SEQ) are not actual configuration settings, and thus cannot be overridden with environment variables.
+
+To refer to a configuration setting by environment variables, the \fIsection\fP and \fIkey\fP are concatenated together with an underscore, and transformed to upper-case. For example, the configuration variable \fIFOO_BAZ_BAR\fP refers to an ini-file entry as follows:
+
+.EX
+[foo]
+bar_baz = xyzzy
+.EE
+
+In the \fBENVIRONMENT\fP section below, the relevant configuration settings for this tool is listed along with a short description of its meaning.
+
+Some configuration settings may also be overriden by command line options. Also note that the use of the \fB-n\fP and \fB--env-prefix\fP options affect how environment and configuration is read. The effects of options on how configuration settings are affective is described in the respective \fBOPTIONS\fP section.
+
+.SH ENVIRONMENT
+
+
+.TP
+\fICHAIN_SPEC\fP
+String specifying the type of chain connected to, in the format \fI<engine>:<fork>:<network_id>:<common_name>\fP. For EVM nodes the \fIengine\fP value will always be \fIevm\fP.
+
+.TP
+\fIRPC_BATCH_LIMIT\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together.
+
+.TP
+\fIRPC_DIALECT\fP
+Enables translations of EVM node specific formatting and response codes.
+
+.TP
+\fIRPC_PROVIDER\fP
+Fully-qualified URL to the RPC endpoint of the blockchain node.
+
+.SH LICENSE
+
+This documentation and its source is licensed under the Creative Commons Attribution-Sharealike 4.0 International license.
+
+The source code of the tool this documentation describes is licensed under the GNU General Public License 3.0.
+
+.SH COPYRIGHT
+
+Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
+PGP: 59A844A484AC11253D3A3E9DCDCBD24DD1D0E001
+
+
+
+.SH SOURCE CODE
+
+https://git.defalsify.org
+
+.SH SEE ALSO
+
+eth-monitor (1)

man/build/eth-monitor-list.1

@@ -0,0 +1,159 @@
+.TH eth-monitor-list 1
+
+
+.SH NAME
+eth-monitor-list \- Query transactions cache
+
+
+.SH SYNOPSIS
+.SY eth-monitor-list
+[ -i \fIchain_spec\fP ] [ p \fIeth_provider\fP ] [ -a \fIaddress\fP ... ] \fIcache_dir\fP
+.YS
+
+
+.SH DESCRIPTION
+List transactions stored in cache matching the given address.
+.P
+Any block data and/or transaction data matchin the relevant hashes returned by the query will be used to create the output. The \fB--fresh\fP option may be defined to force all block and transaction data from the RPC provider endpoint instead.
+.P
+For details on rendering and filtering, please refer to to \fBeth-monitor (1)\fP man page.
+
+
+.SS OPTIONS
+
+.TP
+\fB-0\fP
+Omit newline to output
+
+.TP
+\fB--address \fI\fIaddress
+\fP\fP
+Add an address of interest to match any role. Complements \fB--address-file\fP.
+
+.TP
+\fB-c \fI\fIconfig_dir\fP\fP, \fB--config \fI\fIconfig_dir\fP\fP
+Load configuration files from given directory. All files with an .ini extension will be loaded, of which all must contain valid ini file data.
+
+.TP
+\fB--dumpconfig \fI\fIformat\fP\fP
+Output configuration settings rendered from environment and inputs. Valid arguments are \fIini\fP for ini file output, and \fIenv\fP for environment variable output. See \fBCONFIGURATION\fP.
+
+.TP
+\fB--env-prefix\fP
+Environment prefix for variables to overwrite configuration. Example: If \fB--env-prefix\fP is set to \fBFOO\fP then configuration variable \fBBAR_BAZ\fP would be set by environment variable \fBFOO_BAZ_BAR\fP. Also see \fBENVIRONMENT\fP.
+
+.TP
+\fB--filter \fI\fImodule
+\fP\fP
+Add code execution filter to all matching transactions. The argument must be a python module path. Several filters may be added by supplying the option multiple times. Filters will be executed in the order the options are given. See \fBDEFINING FILTERS\fP section of \fBeth-monitor (1)\fP for more details.
+
+.TP
+\fB--fresh \fI\fI
+\fP\fP
+Only use hashes from cache, and retrieve all block and transaction data from RPC endpoint.
+
+.TP
+\fB--height\fP
+Block height at which to query state for. Does not apply to transactions.
+
+.TP
+\fB-i \fI\fIchain_spec\fP\fP, \fB--chain-spec \fI\fIchain_spec\fP\fP
+Chain specification string, in the format <engine>:<fork>:<chain_id>:<common_name>. Example: "evm:london:1:ethereum". Overrides the \fIRPC_CREDENTIALS\fP configuration setting.
+
+.TP
+\fB-n \fI\fInamespace\fP\fP, \fB--namespace \fI\fInamespace\fP\fP
+Load given configuration namespace. Configuration will be loaded from the immediate configuration subdirectory with the same name.
+
+.TP
+\fB--no-logs\fP
+Turn of logging completely. Negates \fB-v\fP and \fB-vv\fP
+
+.TP
+\fB-p\fP, \fB--rpc-provider\fP
+Fully-qualified URL of RPC provider. Overrides the \fIRPC_PROVIDER\fP configuration setting.
+
+.TP
+\fB--raw\fP
+Produce output most optimized for machines.
+
+.TP
+\fB--renderer \fI\fImodule
+\fP\fP
+Add output renderer filter to all matching transactions. The argument must be a python module path. Several renderers may be added by supplying the option multiple times. See \fBRENDERERS\fP section of \fBeth-monitor (1)\fP for more details.
+
+.TP
+\fB--rpc-batch-limit\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together. Overrides the \fIRPC_BATCH_LIMIT\fP configuration setting.
+
+.TP
+\fB--rpc-dialect\fP
+RPC backend dialect. If specified it \fImay\fP help with encoding and decoding issues. Overrides the \fIRPC_DIALECT\fP configuration setting.
+
+.TP
+\fB-u\fP, \fB--unsafe\fP
+Allow addresses that do not pass checksum.
+
+.TP
+\fB-v\fP
+Verbose. Show logs for important state changes.
+
+.TP
+\fB-vv\fP
+Very verbose. Show logs with debugging information.
+
+.SH CONFIGURATION
+
+All configuration settings may be overriden both by environment variables, or by overriding settings with the contents of ini-files in the directory defined by the \fB-c\fP option.
+
+The active configuration, with values assigned from environment and arguments, can be output using the \fB--dumpconfig\fP \fIformat\fP option. Note that entries having keys prefixed with underscore (e.g. _SEQ) are not actual configuration settings, and thus cannot be overridden with environment variables.
+
+To refer to a configuration setting by environment variables, the \fIsection\fP and \fIkey\fP are concatenated together with an underscore, and transformed to upper-case. For example, the configuration variable \fIFOO_BAZ_BAR\fP refers to an ini-file entry as follows:
+
+.EX
+[foo]
+bar_baz = xyzzy
+.EE
+
+In the \fBENVIRONMENT\fP section below, the relevant configuration settings for this tool is listed along with a short description of its meaning.
+
+Some configuration settings may also be overriden by command line options. Also note that the use of the \fB-n\fP and \fB--env-prefix\fP options affect how environment and configuration is read. The effects of options on how configuration settings are affective is described in the respective \fBOPTIONS\fP section.
+
+.SH ENVIRONMENT
+
+
+.TP
+\fICHAIN_SPEC\fP
+String specifying the type of chain connected to, in the format \fI<engine>:<fork>:<network_id>:<common_name>\fP. For EVM nodes the \fIengine\fP value will always be \fIevm\fP.
+
+.TP
+\fIRPC_BATCH_LIMIT\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together.
+
+.TP
+\fIRPC_DIALECT\fP
+Enables translations of EVM node specific formatting and response codes.
+
+.TP
+\fIRPC_PROVIDER\fP
+Fully-qualified URL to the RPC endpoint of the blockchain node.
+
+.SH LICENSE
+
+This documentation and its source is licensed under the Creative Commons Attribution-Sharealike 4.0 International license.
+
+The source code of the tool this documentation describes is licensed under the GNU General Public License 3.0.
+
+.SH COPYRIGHT
+
+Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
+PGP: 59A844A484AC11253D3A3E9DCDCBD24DD1D0E001
+
+
+
+.SH SOURCE CODE
+
+https://git.defalsify.org
+
+.SH SEE ALSO
+
+eth-monitor (1)

man/build/eth-monitor-sync.1

@@ -0,0 +1,276 @@
+.TH eth-monitor 1
+
+.SH NAME
+eth-monitor \- Cache, index and monitor transactions with an EVM node rpc
+
+.SH SYNOPSIS
+.SY eth-monitor
+[ --skip-history ] [ --single ] [ p \fIeth_provider\fP ] [ --includes-file \fIfile\fP ] [ -i chain_spec ] 
+.YS
+.SY eth-monitor
+[ --skip-history ] [ --single ] [ p \fIeth_provider\fP ] [ --excludes-file \fIfile\fP ] [ --include-default ] [ -i chain_spec ] 
+.YS
+
+
+.SH DESCRIPTION
+The \fBeth-monitor\fP has fulfills three distinct but related functions:
+.IP
+1. A customizable view of on transactions of interest.
+.IP
+2. A block and transaction cache.
+.IP
+3. Arbitrary code executions using a transaction (and its block) as input.
+
+.P
+Using an EVM RPC endpoint, the \fBeth-monitor\fP tool will retrieve blocks within a given range and provides arbitrary processing of each transaction.
+.P
+A collection of options is provided to control the behavior of which block ranges to sync, which criteria to use for display and cache, and what code to execute for matching transactions. Details on each topic can be found in the \fBSYNCING\fP, \fBMATCHING ADDRESSES\fP and \fBDEFINING FILTERS\fP sections below, respectively.
+.P
+Example executions of the tool can be found in the \fBEXAMPLES\fP section.
+.P
+
+
+.SS OPTIONS
+
+.TP
+\fB-0\fP
+Omit newline to output
+
+.TP
+\fB--address \fI\fIaddress
+\fP\fP
+Add an address of interest to match any role. Complements \fB--address-file\fP.
+
+.TP
+\fB-c \fI\fIconfig_dir\fP\fP, \fB--config \fI\fIconfig_dir\fP\fP
+Load configuration files from given directory. All files with an .ini extension will be loaded, of which all must contain valid ini file data.
+
+.TP
+\fB--context-key \fI\fIkey=value
+\fP\fP
+Add a key-value pair that gets passed to the syncer context. May be specified several times.
+
+.TP
+\fB--dumpconfig \fI\fIformat\fP\fP
+Output configuration settings rendered from environment and inputs. Valid arguments are \fIini\fP for ini file output, and \fIenv\fP for environment variable output. See \fBCONFIGURATION\fP.
+
+.TP
+\fB--env-prefix\fP
+Environment prefix for variables to overwrite configuration. Example: If \fB--env-prefix\fP is set to \fBFOO\fP then configuration variable \fBBAR_BAZ\fP would be set by environment variable \fBFOO_BAZ_BAR\fP. Also see \fBENVIRONMENT\fP.
+
+.TP
+\fB--excludes-file \fI\fIfile
+\fP\fP
+Load address exclude matching rules from file. See \fBMATCHING ADDRESSES\fP.
+
+.TP
+\fB--exec \fI\fIaddress
+\fP\fP
+Add an address of interest to executable address array. Complements \fB--address-file\fP.
+
+.TP
+\fB--filter \fI\fImodule
+\fP\fP
+Add code execution filter to all matched transactions. The argument must be a python module path. Several filters may be added by supplying the option multiple times. Filters will be executed in the order the options are given. See \fBDEFINING FILTERS\fP section of \fBeth-monitor (1)\fP for more details.
+
+.TP
+\fB--height\fP
+Block height at which to query state for. Does not apply to transactions.
+
+.TP
+\fB-i \fI\fIchain_spec\fP\fP, \fB--chain-spec \fI\fIchain_spec\fP\fP
+Chain specification string, in the format <engine>:<fork>:<chain_id>:<common_name>. Example: "evm:london:1:ethereum". Overrides the \fIRPC_CREDENTIALS\fP configuration setting.
+
+.TP
+\fB--include-default \fI\fI
+\fP\fP
+Match all addresses by default. Addresses may be excluded using --excludes-file. If this is set, --input, --output, --exec and --includes-file will have no effect.
+
+.TP
+\fB--includes-file \fI\fIfile
+\fP\fP
+Load address include matching rules from file. See \fBMATCHING ADDRESSES\fP.
+
+.TP
+\fB--input \fI\fIaddress
+\fP\fP
+Add an address of interest to inputs (recipients) array. Complements \fB--address-file\fP.
+
+.TP
+\fB-n \fI\fInamespace\fP\fP, \fB--namespace \fI\fInamespace\fP\fP
+Load given configuration namespace. Configuration will be loaded from the immediate configuration subdirectory with the same name.
+
+.TP
+\fB--no-logs\fP
+Turn of logging completely. Negates \fB-v\fP and \fB-vv\fP
+
+.TP
+\fB--output \fI\fIaddress
+\fP\fP
+Add an address of interest to outputs (sender) array. Complements \fB--address-file\fP.
+
+.TP
+\fB-p\fP, \fB--rpc-provider\fP
+Fully-qualified URL of RPC provider. Overrides the \fIRPC_PROVIDER\fP configuration setting.
+
+.TP
+\fB--raw\fP
+Produce output most optimized for machines.
+
+.TP
+\fB--renderer \fI\fImodule
+\fP\fP
+Add output renderer filter to all matched transactions. The argument must be a python module path. Several renderers may be added by supplying the option multiple times. See \fBRENDERERS\fP section of \fBeth-monitor (1)\fP for more details.
+
+.TP
+\fB--rpc-batch-limit\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together. Overrides the \fIRPC_BATCH_LIMIT\fP configuration setting.
+
+.TP
+\fB--rpc-dialect\fP
+RPC backend dialect. If specified it \fImay\fP help with encoding and decoding issues. Overrides the \fIRPC_DIALECT\fP configuration setting.
+
+.TP
+\fB--store-block-data \fI\fI
+\fP\fP
+Store block data in cache for matching transactions. Requires \fB--cache-dir\fP.
+
+.TP
+\fB--store-tx-data \fI\fI
+\fP\fP
+Store transaction data in cache for matching transactions. Requires \fB--cache-dir\fP.
+
+.TP
+\fB-u\fP, \fB--unsafe\fP
+Allow addresses that do not pass checksum.
+
+.TP
+\fB-v\fP
+Verbose. Show logs for important state changes.
+
+.TP
+\fB-vv\fP
+Very verbose. Show logs with debugging information.
+
+.TP
+\fB--x-address \fI\fIaddress
+\fP\fP
+Add an address of interest to match any role.
+
+.TP
+\fB--x-exec \fI\fIaddress
+\fP\fP
+Add an address of disinterest to executable address array.
+
+.TP
+\fB--x-input \fI\fIaddress
+\fP\fP
+Add an address of disinterest to inputs (recipients) array.
+
+.TP
+\fB--x-output \fI\fIaddress
+\fP\fP
+Add an address of disinterest to outputs (sender) array.
+
+.SH CONFIGURATION
+
+All configuration settings may be overriden both by environment variables, or by overriding settings with the contents of ini-files in the directory defined by the \fB-c\fP option.
+
+The active configuration, with values assigned from environment and arguments, can be output using the \fB--dumpconfig\fP \fIformat\fP option. Note that entries having keys prefixed with underscore (e.g. _SEQ) are not actual configuration settings, and thus cannot be overridden with environment variables.
+
+To refer to a configuration setting by environment variables, the \fIsection\fP and \fIkey\fP are concatenated together with an underscore, and transformed to upper-case. For example, the configuration variable \fIFOO_BAZ_BAR\fP refers to an ini-file entry as follows:
+
+.EX
+[foo]
+bar_baz = xyzzy
+.EE
+
+In the \fBENVIRONMENT\fP section below, the relevant configuration settings for this tool is listed along with a short description of its meaning.
+
+Some configuration settings may also be overriden by command line options. Also note that the use of the \fB-n\fP and \fB--env-prefix\fP options affect how environment and configuration is read. The effects of options on how configuration settings are affective is described in the respective \fBOPTIONS\fP section.
+
+.SH MATCHING ADDRESSES
+By default, addresses to match against transactions need to be explicitly specified. This behavior can be reversed with the \fB--include-default\fP option. Addresses to match are defined using the \fB--input\fP, \fB--output\fP and \fB--exec\fP options. Addresses specified multiple times will be deduplicated.
+.P
+Inclusion rules may also be loaded from file by specifying the \fB--includes-file\fP and \fB--excludes-file\fP options. Each file must specify the outputs, inputs and exec addresses as comma separated lists respectively, separated by tabs.
+.P
+In the current state of this tool, address matching will affect all parts of the processing; cache, code execution and rendering. 
+
+.SH SYNCING
+When a sync is initiated, the state of this sync is persisted. This way, previous syncs that did not complete for some reason will be resumed where they left off.
+.P
+A special sync type \fB--head\fP starts syncing at the current head of the chain, and continue to sync until interrupted. When resuming sync, a new sync range between the current block head and the block height at which the previous \fB--head\fP sync left off will automatically be created.
+.P
+Syncs can be forced to (re)run for ranges regardless of previous state by using the \fB--single\fP option. However, there is no protection in place from preventing code filters from being executed again on the same transaction when this is done. See \fBDEFINING FILTERS\fP below.
+
+
+.SH CACHE
+When syncing, the hash of a block and transaction matching the address criteria will be stored in the cache. The hashes can be used for future data lookups.
+.P
+If \fB--store-block-data\fP and/or \fB--store-tx-data\fP is set, a copy of the block and/or transaction data will also be stored, respectively.
+
+
+.SH RENDERING
+Rendering in the context of \fBeth-monitor\fP refers to a formatted output stream that occurs independently of caching and code execution.
+.P
+Filters for rendering may be specified by specifying python modules to the \fB--renderer\fP option. This option may be specified multiple times.
+.P
+Rendering filters will be executed in order, and the first filter to return \fIFalse\fP
+
+
+.SH DEFINING FILTERS
+
+Filters will strictly be executed in the order which they are defined on the command line.
+
+A python module used for filter must fulfill two conditions:
+
+.IP
+1. It must provide a class named \fIFilter\fP in the package base namespace.
+.IP
+2. The \fIFilter\fP class must extend the \fIchainsyncer.filter.SyncFilter\fP interface, and at least override the \fIfilter\fP method.
+
+
+.SS SYNCER AND FILTER CONTEXT
+
+Key-value pairs specified with `--context-key` will be passed to the filter's \fIprepare\fP method, aswell as the \fIctx\fP parameter of the \fIfilter\fP method.
+
+
+.SH FURTHER READING
+
+Refer to the \fBchainsyncer\fP chapter n \fIinfo chaintool\fP for in-depth information on the subjects of syncing and filtering.
+.SH ENVIRONMENT
+
+
+.TP
+\fICHAIN_SPEC\fP
+String specifying the type of chain connected to, in the format \fI<engine>:<fork>:<network_id>:<common_name>\fP. For EVM nodes the \fIengine\fP value will always be \fIevm\fP.
+
+.TP
+\fIRPC_BATCH_LIMIT\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together.
+
+.TP
+\fIRPC_DIALECT\fP
+Enables translations of EVM node specific formatting and response codes.
+
+.TP
+\fIRPC_PROVIDER\fP
+Fully-qualified URL to the RPC endpoint of the blockchain node.
+
+.SH LICENSE
+
+This documentation and its source is licensed under the Creative Commons Attribution-Sharealike 4.0 International license.
+
+The source code of the tool this documentation describes is licensed under the GNU General Public License 3.0.
+
+.SH COPYRIGHT
+
+Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
+PGP: 59A844A484AC11253D3A3E9DCDCBD24DD1D0E001
+
+
+
+.SH SOURCE CODE
+
+https://git.defalsify.org
+

man/build/eth-monitor.1

@@ -0,0 +1,276 @@
+.TH eth-monitor 1
+
+.SH NAME
+eth-monitor \- Cache, index and monitor transactions with an EVM node rpc
+
+.SH SYNOPSIS
+.SY eth-monitor
+[ --skip-history ] [ --single ] [ p \fIeth_provider\fP ] [ --includes-file \fIfile\fP ] [ -i chain_spec ] 
+.YS
+.SY eth-monitor
+[ --skip-history ] [ --single ] [ p \fIeth_provider\fP ] [ --excludes-file \fIfile\fP ] [ --include-default ] [ -i chain_spec ] 
+.YS
+
+
+.SH DESCRIPTION
+The \fBeth-monitor\fP has fulfills three distinct but related functions:
+.IP
+1. A customizable view of on transactions of interest.
+.IP
+2. A block and transaction cache.
+.IP
+3. Arbitrary code executions using a transaction (and its block) as input.
+
+.P
+Using an EVM RPC endpoint, the \fBeth-monitor\fP tool will retrieve blocks within a given range and provides arbitrary processing of each transaction.
+.P
+A collection of options is provided to control the behavior of which block ranges to sync, which criteria to use for display and cache, and what code to execute for matching transactions. Details on each topic can be found in the \fBSYNCING\fP, \fBMATCHING ADDRESSES\fP and \fBDEFINING FILTERS\fP sections below, respectively.
+.P
+Example executions of the tool can be found in the \fBEXAMPLES\fP section.
+.P
+
+
+.SS OPTIONS
+
+.TP
+\fB-0\fP
+Omit newline to output
+
+.TP
+\fB--address \fI\fIaddress
+\fP\fP
+Add an address of interest to match any role. Complements \fB--address-file\fP.
+
+.TP
+\fB-c \fI\fIconfig_dir\fP\fP, \fB--config \fI\fIconfig_dir\fP\fP
+Load configuration files from given directory. All files with an .ini extension will be loaded, of which all must contain valid ini file data.
+
+.TP
+\fB--context-key \fI\fIkey=value
+\fP\fP
+Add a key-value pair that gets passed to the syncer context. May be specified several times.
+
+.TP
+\fB--dumpconfig \fI\fIformat\fP\fP
+Output configuration settings rendered from environment and inputs. Valid arguments are \fIini\fP for ini file output, and \fIenv\fP for environment variable output. See \fBCONFIGURATION\fP.
+
+.TP
+\fB--env-prefix\fP
+Environment prefix for variables to overwrite configuration. Example: If \fB--env-prefix\fP is set to \fBFOO\fP then configuration variable \fBBAR_BAZ\fP would be set by environment variable \fBFOO_BAZ_BAR\fP. Also see \fBENVIRONMENT\fP.
+
+.TP
+\fB--excludes-file \fI\fIfile
+\fP\fP
+Load address exclude matching rules from file. See \fBMATCHING ADDRESSES\fP.
+
+.TP
+\fB--exec \fI\fIaddress
+\fP\fP
+Add an address of interest to executable address array. Complements \fB--address-file\fP.
+
+.TP
+\fB--filter \fI\fImodule
+\fP\fP
+Add code execution filter to all matched transactions. The argument must be a python module path. Several filters may be added by supplying the option multiple times. Filters will be executed in the order the options are given. See \fBDEFINING FILTERS\fP section of \fBeth-monitor (1)\fP for more details.
+
+.TP
+\fB--height\fP
+Block height at which to query state for. Does not apply to transactions.
+
+.TP
+\fB-i \fI\fIchain_spec\fP\fP, \fB--chain-spec \fI\fIchain_spec\fP\fP
+Chain specification string, in the format <engine>:<fork>:<chain_id>:<common_name>. Example: "evm:london:1:ethereum". Overrides the \fIRPC_CREDENTIALS\fP configuration setting.
+
+.TP
+\fB--include-default \fI\fI
+\fP\fP
+Match all addresses by default. Addresses may be excluded using --excludes-file. If this is set, --input, --output, --exec and --includes-file will have no effect.
+
+.TP
+\fB--includes-file \fI\fIfile
+\fP\fP
+Load address include matching rules from file. See \fBMATCHING ADDRESSES\fP.
+
+.TP
+\fB--input \fI\fIaddress
+\fP\fP
+Add an address of interest to inputs (recipients) array. Complements \fB--address-file\fP.
+
+.TP
+\fB-n \fI\fInamespace\fP\fP, \fB--namespace \fI\fInamespace\fP\fP
+Load given configuration namespace. Configuration will be loaded from the immediate configuration subdirectory with the same name.
+
+.TP
+\fB--no-logs\fP
+Turn of logging completely. Negates \fB-v\fP and \fB-vv\fP
+
+.TP
+\fB--output \fI\fIaddress
+\fP\fP
+Add an address of interest to outputs (sender) array. Complements \fB--address-file\fP.
+
+.TP
+\fB-p\fP, \fB--rpc-provider\fP
+Fully-qualified URL of RPC provider. Overrides the \fIRPC_PROVIDER\fP configuration setting.
+
+.TP
+\fB--raw\fP
+Produce output most optimized for machines.
+
+.TP
+\fB--renderer \fI\fImodule
+\fP\fP
+Add output renderer filter to all matched transactions. The argument must be a python module path. Several renderers may be added by supplying the option multiple times. See \fBRENDERERS\fP section of \fBeth-monitor (1)\fP for more details.
+
+.TP
+\fB--rpc-batch-limit\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together. Overrides the \fIRPC_BATCH_LIMIT\fP configuration setting.
+
+.TP
+\fB--rpc-dialect\fP
+RPC backend dialect. If specified it \fImay\fP help with encoding and decoding issues. Overrides the \fIRPC_DIALECT\fP configuration setting.
+
+.TP
+\fB--store-block-data \fI\fI
+\fP\fP
+Store block data in cache for matching transactions. Requires \fB--cache-dir\fP.
+
+.TP
+\fB--store-tx-data \fI\fI
+\fP\fP
+Store transaction data in cache for matching transactions. Requires \fB--cache-dir\fP.
+
+.TP
+\fB-u\fP, \fB--unsafe\fP
+Allow addresses that do not pass checksum.
+
+.TP
+\fB-v\fP
+Verbose. Show logs for important state changes.
+
+.TP
+\fB-vv\fP
+Very verbose. Show logs with debugging information.
+
+.TP
+\fB--x-address \fI\fIaddress
+\fP\fP
+Add an address of interest to match any role.
+
+.TP
+\fB--x-exec \fI\fIaddress
+\fP\fP
+Add an address of disinterest to executable address array.
+
+.TP
+\fB--x-input \fI\fIaddress
+\fP\fP
+Add an address of disinterest to inputs (recipients) array.
+
+.TP
+\fB--x-output \fI\fIaddress
+\fP\fP
+Add an address of disinterest to outputs (sender) array.
+
+.SH CONFIGURATION
+
+All configuration settings may be overriden both by environment variables, or by overriding settings with the contents of ini-files in the directory defined by the \fB-c\fP option.
+
+The active configuration, with values assigned from environment and arguments, can be output using the \fB--dumpconfig\fP \fIformat\fP option. Note that entries having keys prefixed with underscore (e.g. _SEQ) are not actual configuration settings, and thus cannot be overridden with environment variables.
+
+To refer to a configuration setting by environment variables, the \fIsection\fP and \fIkey\fP are concatenated together with an underscore, and transformed to upper-case. For example, the configuration variable \fIFOO_BAZ_BAR\fP refers to an ini-file entry as follows:
+
+.EX
+[foo]
+bar_baz = xyzzy
+.EE
+
+In the \fBENVIRONMENT\fP section below, the relevant configuration settings for this tool is listed along with a short description of its meaning.
+
+Some configuration settings may also be overriden by command line options. Also note that the use of the \fB-n\fP and \fB--env-prefix\fP options affect how environment and configuration is read. The effects of options on how configuration settings are affective is described in the respective \fBOPTIONS\fP section.
+
+.SH MATCHING ADDRESSES
+By default, addresses to match against transactions need to be explicitly specified. This behavior can be reversed with the \fB--include-default\fP option. Addresses to match are defined using the \fB--input\fP, \fB--output\fP and \fB--exec\fP options. Addresses specified multiple times will be deduplicated.
+.P
+Inclusion rules may also be loaded from file by specifying the \fB--includes-file\fP and \fB--excludes-file\fP options. Each file must specify the outputs, inputs and exec addresses as comma separated lists respectively, separated by tabs.
+.P
+In the current state of this tool, address matching will affect all parts of the processing; cache, code execution and rendering. 
+
+.SH SYNCING
+When a sync is initiated, the state of this sync is persisted. This way, previous syncs that did not complete for some reason will be resumed where they left off.
+.P
+A special sync type \fB--head\fP starts syncing at the current head of the chain, and continue to sync until interrupted. When resuming sync, a new sync range between the current block head and the block height at which the previous \fB--head\fP sync left off will automatically be created.
+.P
+Syncs can be forced to (re)run for ranges regardless of previous state by using the \fB--single\fP option. However, there is no protection in place from preventing code filters from being executed again on the same transaction when this is done. See \fBDEFINING FILTERS\fP below.
+
+
+.SH CACHE
+When syncing, the hash of a block and transaction matching the address criteria will be stored in the cache. The hashes can be used for future data lookups.
+.P
+If \fB--store-block-data\fP and/or \fB--store-tx-data\fP is set, a copy of the block and/or transaction data will also be stored, respectively.
+
+
+.SH RENDERING
+Rendering in the context of \fBeth-monitor\fP refers to a formatted output stream that occurs independently of caching and code execution.
+.P
+Filters for rendering may be specified by specifying python modules to the \fB--renderer\fP option. This option may be specified multiple times.
+.P
+Rendering filters will be executed in order, and the first filter to return \fIFalse\fP
+
+
+.SH DEFINING FILTERS
+
+Filters will strictly be executed in the order which they are defined on the command line.
+
+A python module used for filter must fulfill two conditions:
+
+.IP
+1. It must provide a class named \fIFilter\fP in the package base namespace.
+.IP
+2. The \fIFilter\fP class must extend the \fIchainsyncer.filter.SyncFilter\fP interface, and at least override the \fIfilter\fP method.
+
+
+.SS SYNCER AND FILTER CONTEXT
+
+Key-value pairs specified with `--context-key` will be passed to the filter's \fIprepare\fP method, aswell as the \fIctx\fP parameter of the \fIfilter\fP method.
+
+
+.SH FURTHER READING
+
+Refer to the \fBchainsyncer\fP chapter n \fIinfo chaintool\fP for in-depth information on the subjects of syncing and filtering.
+.SH ENVIRONMENT
+
+
+.TP
+\fICHAIN_SPEC\fP
+String specifying the type of chain connected to, in the format \fI<engine>:<fork>:<network_id>:<common_name>\fP. For EVM nodes the \fIengine\fP value will always be \fIevm\fP.
+
+.TP
+\fIRPC_BATCH_LIMIT\fP
+Set number of RPC requests that can be set to the RPC provider as a batch request. This is made available through settings to any request builder implementing batch requests. A value of 1 means no batch will be used. A value of 0 indicates that the limit is not relevant. Any other positive value signals the maximum number of requests to be batched together.
+
+.TP
+\fIRPC_DIALECT\fP
+Enables translations of EVM node specific formatting and response codes.
+
+.TP
+\fIRPC_PROVIDER\fP
+Fully-qualified URL to the RPC endpoint of the blockchain node.
+
+.SH LICENSE
+
+This documentation and its source is licensed under the Creative Commons Attribution-Sharealike 4.0 International license.
+
+The source code of the tool this documentation describes is licensed under the GNU General Public License 3.0.
+
+.SH COPYRIGHT
+
+Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
+PGP: 59A844A484AC11253D3A3E9DCDCBD24DD1D0E001
+
+
+
+.SH SOURCE CODE
+
+https://git.defalsify.org
+

man/eth-monitor.custom.groff

@@ -29,14 +29,19 @@ Rendering filters will be executed in order, and the first filter to return \fIF
 
 .SH DEFINING FILTERS
 
+Filters will strictly be executed in the order which they are defined on the command line.
+
 A python module used for filter must fulfill two conditions:
 
 .IP
 1. It must provide a class named \fIFilter\fP in the package base namespace.
 .IP
-2. The \fIFilter\fP class must include a method named \fIfilter\fP with the signature \fIdef filter(self, conn, block, tx, db_session=None)\fP. 
+2. The \fIFilter\fP class must extend the \fIchainsyncer.filter.SyncFilter\fP interface, and at least override the \fIfilter\fP method.
 
-Filters will strictly be executed in the order which they are defined on the command line.
+
+.SS SYNCER AND FILTER CONTEXT
+
+Key-value pairs specified with `--context-key` will be passed to the filter's \fIprepare\fP method, aswell as the \fIctx\fP parameter of the \fIfilter\fP method.
 
 
 .SH FURTHER READING

man/eth-monitor.overrides

@@ -13,3 +13,4 @@ storetx	Store transaction data in cache for matching transactions. Requires \fB-
 storeblock	Store block data in cache for matching transactions. Requires \fB--cache-dir\fP.	--store-block-data	
 renderer	Add output renderer filter to all matched transactions. The argument must be a python module path. Several renderers may be added by supplying the option multiple times. See \fBRENDERERS\fP section of \fBeth-monitor (1)\fP for more details.	--renderer	module
 filter	Add code execution filter to all matched transactions. The argument must be a python module path. Several filters may be added by supplying the option multiple times. Filters will be executed in the order the options are given. See \fBDEFINING FILTERS\fP section of \fBeth-monitor (1)\fP for more details.	--filter	module
+context_key	Add a key-value pair that gets passed to the syncer context. May be specified several times.	--context-key	key=value

requirements.txt

@@ -1,6 +1,6 @@
-chainlib-eth~=0.5.0
-chainlib~=0.5.0
-chainsyncer~=0.8.2
+chainlib-eth~=0.6.0
+chainlib~=0.5.2
+chainsyncer~=0.8.5
 leveldir~=0.3.0
-eth-cache~=0.3.0
+eth-cache~=0.4.0
 confini~=0.6.3

run_tests.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -a
+set -e
+set -x
+default_pythonpath=$PYTHONPATH:.
+export PYTHONPATH=${default_pythonpath:-.}
+>&2 echo using pythonpath $PYTHONPATH
+for f in `ls tests/*.py`; do
+	python $f
+done
+for f in `ls tests/rules/*.py`; do
+	python $f
+done
+set +x
+set +e
+set +a

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = eth-monitor
-version = 0.8.2
+version = 0.8.9
 description = Monitor and cache transactions using match filters
 author = Louis Holbrook
 author_email = dev@holbrook.no

test_requirements.txt

@@ -1,5 +1,5 @@
-eth_tester==0.5.0b3
-py-evm==0.3.0a20
-rlp==2.0.1
+eth_tester==0.10.0b4
+py-evm==0.10.0b4
+rlp==3.0.0
 pytest==6.0.1
 coverage==5.5

tests/rules/test_base.py

@@ -0,0 +1,160 @@
+# standard imports
+import logging
+import unittest
+import os
+
+# local imports
+from eth_monitor.rules import *
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class TestRule(unittest.TestCase):
+
+    def setUp(self):
+        self.alice = os.urandom(20).hex()
+        self.bob = os.urandom(20).hex()
+        self.carol = os.urandom(20).hex()
+        self.dave = os.urandom(20).hex()
+        self.x = os.urandom(20).hex()
+        self.y = os.urandom(20).hex()
+        self.hsh = os.urandom(32).hex()
+
+    
+    def test_address_include(self):
+        data = b''
+        outs = [self.alice]
+        ins = []
+        execs = []
+        rule = RuleSimple(outs, ins, execs)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertTrue(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertFalse(r)
+
+        outs = []
+        ins = [self.alice]
+        execs = []
+        rule = RuleSimple(outs, ins, execs)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertTrue(r)
+
+        outs = []
+        ins = []
+        execs = [self.x]
+        rule = RuleSimple(outs, ins, execs)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh)
+        self.assertTrue(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertFalse(r)
+
+        data = b'deadbeef0123456789'
+        data_match = [data[:8]]
+        rule = RuleMethod(data_match)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh)
+        self.assertTrue(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh)
+        self.assertFalse(r)
+
+        rule = RuleData(data_match)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh)
+        self.assertTrue(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh)
+        self.assertTrue(r)
+
+
+    def test_address_exclude(self):
+        data = b''
+        outs = [self.alice]
+        ins = []
+        execs = []
+        rule = RuleSimple(outs, ins, execs)
+
+        c = AddressRules()
+        c.exclude(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertFalse(r)
+
+        c = AddressRules(include_by_default=True)
+        c.exclude(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertTrue(r)
+
+        outs = []
+        ins = [self.alice]
+        execs = []
+        rule = RuleSimple(outs, ins, execs)
+        c = AddressRules(include_by_default=True)
+        c.exclude(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertTrue(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertFalse(r)
+
+        outs = []
+        ins = []
+        execs = [self.x]
+        rule = RuleSimple(outs, ins, execs)
+        c = AddressRules(include_by_default=True)
+        c.exclude(rule)
+        r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertTrue(r)
+
+        data = b'deadbeef0123456789'
+        data_match = [data[:8]]
+        rule = RuleMethod(data_match)
+        c = AddressRules(include_by_default=True)
+        c.exclude(rule)
+        r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh)
+        self.assertTrue(r)
+
+        rule = RuleData(data_match)
+        c = AddressRules(include_by_default=True)
+        c.exclude(rule)
+        r = c.apply_rules_addresses(self.alice, self.x, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, b'abcd' + data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, b'abcd', self.hsh)
+        self.assertTrue(r)
+
+    
+    def test_address_include_exclude(self):
+        data = b''
+        outs = [self.alice]
+        ins = []
+        execs = []
+        rule = RuleSimple(outs, ins, execs)
+        c = AddressRules()
+        c.include(rule)
+        c.exclude(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertFalse(r)
+
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/rules/test_greedy.py

@@ -0,0 +1,90 @@
+import logging
+import unittest
+import os
+
+# local imports
+from eth_monitor.rules import *
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class TestRule(unittest.TestCase):
+
+    def setUp(self):
+        self.alice = os.urandom(20).hex()
+        self.bob = os.urandom(20).hex()
+        self.carol = os.urandom(20).hex()
+        self.dave = os.urandom(20).hex()
+        self.x = os.urandom(20).hex()
+        self.y = os.urandom(20).hex()
+        self.hsh = os.urandom(32).hex()
+
+
+    def test_greedy_includes(self):
+        data = b''
+        outs = [self.alice]
+        ins = [self.carol]
+        execs = []
+        rule = RuleSimple(outs, ins, execs, match_all=True)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.carol, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.alice, self.carol, data, self.hsh)
+        self.assertTrue(r)
+
+        rule = RuleSimple(outs, ins, execs)
+        c = AddressRules(match_all=True)
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertTrue(r)
+        r = c.apply_rules_addresses(self.bob, self.alice, data, self.hsh)
+        self.assertFalse(r)
+        r = c.apply_rules_addresses(self.bob, self.carol, data, self.hsh)
+        self.assertTrue(r)
+        r = c.apply_rules_addresses(self.alice, self.carol, data, self.hsh)
+        self.assertTrue(r)
+
+
+    def test_greedy_data(self):
+        data = os.urandom(128).hex()
+        data_match_one = data[4:8]
+        data_match_two = data[32:42]
+        data_match_fail = os.urandom(64).hex()
+        data_match = [data_match_one]
+
+        rule = RuleData(data_match, match_all=True)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertTrue(r)
+
+        data_match = [data_match_two]
+        rule = RuleData(data_match, match_all=True)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertTrue(r)
+
+        data_match = [data_match_two, data_match_one]
+        rule = RuleData(data_match, match_all=True)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertTrue(r)
+
+        data_match = [data_match_two, data_match_fail, data_match_one]
+        rule = RuleData(data_match, match_all=True)
+        c = AddressRules()
+        c.include(rule)
+        r = c.apply_rules_addresses(self.alice, self.bob, data, self.hsh)
+        self.assertFalse(r)
+
+
+if __name__ == '__main__':
+    unittest.main()