2022-01-23 23:07:59 +01:00
|
|
|
|
# standard imports
|
|
|
|
|
import argparse
|
|
|
|
|
import logging
|
|
|
|
|
import sys
|
|
|
|
|
import os
|
2022-01-24 01:17:27 +01:00
|
|
|
|
import time
|
2022-01-23 23:07:59 +01:00
|
|
|
|
|
|
|
|
|
# external imports
|
|
|
|
|
from chainlib.encode import TxHexNormalizer
|
|
|
|
|
from chainlib.eth.connection import EthHTTPConnection
|
|
|
|
|
from chainlib.chain import ChainSpec
|
2022-03-01 13:10:35 +01:00
|
|
|
|
from eth_cache.store.file import FileStore
|
2022-01-23 23:07:59 +01:00
|
|
|
|
|
|
|
|
|
# local imports
|
|
|
|
|
from eth_monitor.filters.cache import Filter as CacheFilter
|
|
|
|
|
from eth_monitor.filters import RuledFilter
|
2022-02-28 22:56:56 +01:00
|
|
|
|
from eth_monitor.rules import (
|
|
|
|
|
AddressRules,
|
|
|
|
|
RuleSimple,
|
|
|
|
|
)
|
2022-01-23 23:07:59 +01:00
|
|
|
|
|
|
|
|
|
logging.basicConfig(level=logging.WARNING)
|
|
|
|
|
logg = logging.getLogger()
|
|
|
|
|
|
|
|
|
|
normalize_address = TxHexNormalizer().wallet_address
|
|
|
|
|
|
2022-03-03 18:27:19 +01:00
|
|
|
|
services = [
|
|
|
|
|
'etherscan',
|
|
|
|
|
]
|
2022-01-24 01:17:27 +01:00
|
|
|
|
|
2022-01-23 23:07:59 +01:00
|
|
|
|
argparser = argparse.ArgumentParser('master eth events monitor')
|
|
|
|
|
argparser.add_argument('--api-key-file', dest='api_key_file', type=str, help='File to read API key from')
|
|
|
|
|
argparser.add_argument('--cache-dir', dest='cache_dir', type=str, help='Directory to store tx data')
|
2022-01-24 01:17:27 +01:00
|
|
|
|
argparser.add_argument('--store-tx-data', dest='store_tx_data', action='store_true', help='Include all transaction data objects by default')
|
|
|
|
|
argparser.add_argument('--store-block-data', dest='store_block_data', action='store_true', help='Include all block data objects by default')
|
2022-01-23 23:07:59 +01:00
|
|
|
|
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='evm:ethereum:1', help='Chain specification string')
|
2022-03-03 18:27:19 +01:00
|
|
|
|
argparser.add_argument('--address-file', dest='address_file', default=[], type=str, action='append', help='Add addresses from file')
|
|
|
|
|
argparser.add_argument('--list-services', dest='list', action='store_true', help='List all supported services')
|
2022-01-23 23:07:59 +01:00
|
|
|
|
argparser.add_argument('-a', '--address', default=[], type=str, action='append', help='Add address')
|
2022-03-01 13:10:35 +01:00
|
|
|
|
argparser.add_argument('--socks-host', dest='socks_host', type=str, help='Conect through socks host')
|
|
|
|
|
argparser.add_argument('--socks-port', dest='socks_port', type=int, help='Conect through socks port')
|
2022-01-24 01:17:27 +01:00
|
|
|
|
argparser.add_argument('--delay', type=float, default=0.2, help='Seconds to wait between each retrieval from importer')
|
2022-01-23 23:07:59 +01:00
|
|
|
|
argparser.add_argument('-v', action='store_true', help='Be verbose')
|
|
|
|
|
argparser.add_argument('-vv', action='store_true', help='Be more verbose')
|
|
|
|
|
argparser.add_argument('-p', type=str, help='RPC provider')
|
2022-03-03 18:27:19 +01:00
|
|
|
|
argparser.add_argument('service', nargs='?', type=str, help='Index service to import from')
|
2022-01-23 23:07:59 +01:00
|
|
|
|
args = argparser.parse_args(sys.argv[1:])
|
|
|
|
|
|
2022-03-03 18:27:19 +01:00
|
|
|
|
if args.list:
|
|
|
|
|
for s in services:
|
|
|
|
|
sys.stdout.write('{}\n'.format(s))
|
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
|
|
|
|
if not args.service:
|
|
|
|
|
argparser.error('the following arguments are required: service')
|
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
2022-01-23 23:07:59 +01:00
|
|
|
|
if args.vv:
|
|
|
|
|
logg.setLevel(logging.DEBUG)
|
|
|
|
|
elif args.v:
|
|
|
|
|
logg.setLevel(logging.INFO)
|
|
|
|
|
|
|
|
|
|
api_key = os.environ.get('API_KEY')
|
|
|
|
|
if args.api_key_file != None:
|
|
|
|
|
f = open(args.api_key_file, 'r')
|
|
|
|
|
api_key = f.read()
|
|
|
|
|
f.close()
|
|
|
|
|
|
|
|
|
|
rpc = EthHTTPConnection(args.p)
|
|
|
|
|
|
|
|
|
|
chain_spec = ChainSpec.from_chain_str(args.i)
|
|
|
|
|
|
2022-03-01 13:10:35 +01:00
|
|
|
|
def conn_socks(host, port):
|
2022-01-23 23:07:59 +01:00
|
|
|
|
import socks
|
|
|
|
|
import socket
|
|
|
|
|
|
|
|
|
|
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4, host, port, True)
|
|
|
|
|
socket.socket = socks.socksocket
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def collect_addresses(addresses=[], address_files=[]):
|
|
|
|
|
address_collection = []
|
|
|
|
|
for a in addresses:
|
|
|
|
|
a = normalize_address(a)
|
|
|
|
|
if a in address_collection:
|
|
|
|
|
logg.debug('skipping duplicate address {}'.format(a))
|
|
|
|
|
address_collection.append(a)
|
|
|
|
|
logg.info('added address {}'.format(a))
|
|
|
|
|
|
|
|
|
|
for fp in address_files:
|
|
|
|
|
logg.debug('processing file ' + fp)
|
|
|
|
|
f = open(fp, 'r')
|
|
|
|
|
while True:
|
|
|
|
|
a = f.readline()
|
|
|
|
|
if a == '':
|
|
|
|
|
break
|
|
|
|
|
a = a.rstrip()
|
|
|
|
|
a = normalize_address(a)
|
|
|
|
|
if a in address_collection:
|
|
|
|
|
logg.debug('skipping duplicate address {}'.format(a))
|
|
|
|
|
address_collection.append(a)
|
|
|
|
|
logg.info('added address {}'.format(a))
|
|
|
|
|
f.close()
|
|
|
|
|
|
|
|
|
|
return address_collection
|
|
|
|
|
|
|
|
|
|
|
2022-01-24 01:17:27 +01:00
|
|
|
|
def setup_address_rules(addresses):
|
|
|
|
|
rules = AddressRules()
|
2022-02-28 22:56:56 +01:00
|
|
|
|
outputs = []
|
|
|
|
|
inputs = []
|
|
|
|
|
execs = []
|
2022-01-24 01:17:27 +01:00
|
|
|
|
for address in addresses:
|
2022-02-28 22:56:56 +01:00
|
|
|
|
outputs.append(address)
|
|
|
|
|
inputs.append(address)
|
|
|
|
|
execs.append(address)
|
|
|
|
|
rule = RuleSimple(outputs, inputs, execs, description='etherscan import')
|
|
|
|
|
rules.include(rule)
|
2022-01-24 01:17:27 +01:00
|
|
|
|
return rules
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def setup_filter(chain_spec, cache_dir, include_tx_data, include_block_data, address_rules):
|
|
|
|
|
store = FileStore(chain_spec, cache_dir, address_rules=address_rules)
|
2022-01-23 23:07:59 +01:00
|
|
|
|
cache_dir = os.path.realpath(cache_dir)
|
|
|
|
|
if cache_dir == None:
|
|
|
|
|
import tempfile
|
|
|
|
|
cache_dir = tempfile.mkdtemp()
|
|
|
|
|
logg.info('using chain spec {} and dir {}'.format(chain_spec, cache_dir))
|
2022-01-24 01:17:27 +01:00
|
|
|
|
RuledFilter.init(store, include_tx_data=include_tx_data, include_block_data=include_block_data)
|
2022-01-23 23:07:59 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
2022-03-01 13:10:35 +01:00
|
|
|
|
if args.socks_host != None:
|
|
|
|
|
conn_socks(args.socks_host, args.socks_port)
|
2022-01-23 23:07:59 +01:00
|
|
|
|
addresses = collect_addresses(args.address, args.address_file)
|
|
|
|
|
|
2022-02-27 14:52:05 +01:00
|
|
|
|
from eth_monitor.importers.etherscan import Importer as EtherscanImporter
|
2022-01-23 23:44:31 +01:00
|
|
|
|
|
2022-01-24 01:17:27 +01:00
|
|
|
|
address_rules = setup_address_rules(args.address)
|
|
|
|
|
|
|
|
|
|
setup_filter(
|
|
|
|
|
chain_spec,
|
|
|
|
|
args.cache_dir,
|
|
|
|
|
bool(args.store_tx_data),
|
|
|
|
|
bool(args.store_block_data),
|
|
|
|
|
address_rules,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
cache_filter = CacheFilter(
|
|
|
|
|
rules_filter=address_rules,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
filters = [
|
|
|
|
|
cache_filter,
|
|
|
|
|
]
|
|
|
|
|
|
2022-03-03 18:27:19 +01:00
|
|
|
|
importer = []
|
|
|
|
|
if args.service == 'etherscan':
|
|
|
|
|
importer = EtherscanImporter(rpc, api_key, filters=filters, block_callback=RuledFilter.block_callback)
|
|
|
|
|
else:
|
|
|
|
|
raise ValueError('invalid service: {}'.format(args.service))
|
2022-01-23 23:07:59 +01:00
|
|
|
|
for a in addresses:
|
|
|
|
|
importer.get(a)
|
2022-01-24 01:17:27 +01:00
|
|
|
|
time.sleep(args.delay)
|
2022-01-23 23:07:59 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
main()
|