Factor out cache rpc and store to eth-cache

eth_monitor/filters/gas/__init__.py

@@ -17,6 +17,7 @@ def apply(c, result, chain_spec, conn, block, tx, db_session=None):
     value = value.rstrip('0')
     if value[len(value)-1] == '.':
         value += '0'
-    s = '{} {}\t{} -> {} = {}'.format(timestamp, tx.hash, tx.outputs[0], tx.inputs[0], value)
+
+    s = '{} {} {}\t{} -> {} = {}'.format(timestamp, tx.hash, tx.status.name, tx.outputs[0], tx.inputs[0], value)
     result.set(s)
     return False

eth_monitor/index.py

@@ -43,7 +43,7 @@ class AddressIndex:
 
             k = '{}.{}'.format(block_number, tx_index)
 
-            txs[k] = tx
+            txs[float(k)] = tx
 
         ks = list(txs.keys())
         ks.sort()

eth_monitor/rpc.py

@@ -1,81 +0,0 @@
-# standard imports
-import json
-from jsonrpc_std.parse import jsonrpc_from_dict
-import logging
-
-# external imports
-from hexathon import strip_0x
-
-logg = logging.getLogger(__name__)
-
-class CacheRPC:
-
-    def __init__(self, rpc, store):
-        self.rpc = rpc
-        self.store = store
-
-
-    def do(self, o):
-        req = jsonrpc_from_dict(o)
-        r = None
-        if req['method'] == 'eth_getBlockByNumber':
-            block_number = req['params'][0]
-            v = int(strip_0x(block_number), 16)
-            try:
-                j = self.store.get_block_number(v)
-                r = json.loads(j)
-                logg.debug('using cached block {} -> {}'.format(v, r['hash']))
-            except FileNotFoundError:
-                pass
-        elif req['method'] == 'eth_getBlockByHash':
-            block_hash = req['params'][0]
-            v = strip_0x(block_hash)
-            try:
-                j = self.store.get_block(v)
-                r = json.loads(j)
-                logg.debug('using cached block {}'.format(r['hash']))
-            except FileNotFoundError as e:
-                logg.debug('not found {}'.format(e))
-                pass
-        elif req['method'] == 'eth_getTransactionReceipt':
-            tx_hash = req['params'][0]
-            j = None
-            try:
-                tx_hash = strip_0x(tx_hash)
-                j = self.store.get_rcpt(tx_hash)
-                r = json.loads(j)
-                logg.debug('using cached rcpt {}'.format(tx_hash))
-            except FileNotFoundError as e:
-                logg.debug('no file {}'.format(e))
-                pass
-                
-#        elif req['method'] == 'eth_getTransactionByHash':
-#            raise ValueError(o)
-#        elif req['method'] == 'eth_getTransactionByBlockHashAndIndex':
-#            logg.debug('trying tx index {}'.format(o))
-#            v = req['params'][0]
-#            j = None
-#            try:
-#                j = self.store.get_block(v)
-#            except FileNotFoundError:
-#                pass
-#               
-#            if j != None:
-#                o = json.loads(j)
-#                idx = int(req['params'][1], 16)
-#                v = r['transactions'][idx] 
-#                j = None
-#                try:
-#                    j = self.store.get_tx(v)
-#                except FileNotFoundError:
-#                    pass
-#
-#                if j != None:
-#                    r = json.loads(j)
-#                    logg.debug('using cached tx {} -> {}'.format(req['params'], r['hash']))
-
-        if r == None:
-            logg.debug('passthru {}'.format(o))
-            r = self.rpc.do(o)
-
-        return r

eth_monitor/runnable/import.py

@@ -9,11 +9,11 @@ import time
 from chainlib.encode import TxHexNormalizer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.chain import ChainSpec
+from eth_cache.store.file import FileStore
 
 # local imports
 from eth_monitor.filters.cache import Filter as CacheFilter
 from eth_monitor.filters import RuledFilter
-from eth_monitor.store.file import FileStore
 from eth_monitor.rules import (
         AddressRules,
         RuleSimple,
@@ -33,6 +33,8 @@ argparser.add_argument('--store-block-data', dest='store_block_data', action='st
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='evm:ethereum:1', help='Chain specification string')
 argparser.add_argument('-f', '--address-file', dest='address_file', default=[], type=str, action='append', help='Add addresses from file')
 argparser.add_argument('-a', '--address', default=[], type=str, action='append', help='Add address')
+argparser.add_argument('--socks-host', dest='socks_host', type=str, help='Conect through socks host')
+argparser.add_argument('--socks-port', dest='socks_port', type=int, help='Conect through socks port')
 argparser.add_argument('--delay', type=float, default=0.2, help='Seconds to wait between each retrieval from importer')
 argparser.add_argument('-v', action='store_true', help='Be verbose')
 argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@@ -54,7 +56,7 @@ rpc = EthHTTPConnection(args.p)
 
 chain_spec = ChainSpec.from_chain_str(args.i)
 
-def conn_socks_tor(host='127.0.0.1', port=9050):
+def conn_socks(host, port):
     import socks
     import socket
 
@@ -114,7 +116,8 @@ def setup_filter(chain_spec, cache_dir, include_tx_data, include_block_data, add
 
 
 def main():
-    conn_socks_tor()
+    if args.socks_host != None:
+        conn_socks(args.socks_host, args.socks_port)
     addresses = collect_addresses(args.address, args.address_file)
 
     from eth_monitor.importers.etherscan import Importer as EtherscanImporter

eth_monitor/runnable/list.py

@@ -17,11 +17,11 @@ from chainlib.eth.tx import (
         receipt,
         Tx,
         )
+from eth_cache.store.file import FileStore
+from eth_cache.rpc import CacheRPC
 
 # local imports
-from eth_monitor.store.file import FileStore
 from eth_monitor.index import AddressIndex
-from eth_monitor.rpc import CacheRPC
 from eth_monitor.filters.out import OutFilter
 from eth_monitor.rules import AddressRules
         

eth_monitor/runnable/sync.py

@@ -19,6 +19,8 @@ from chainsyncer.driver.head import HeadSyncer
 from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.backend.file import FileBackend
 from chainsyncer.filter import NoopFilter
+from eth_cache.rpc import CacheRPC
+from eth_cache.store.file import FileStore
 
 # local imports
 from eth_monitor.chain import EthChainInterface
@@ -29,8 +31,6 @@ from eth_monitor.rules import (
         )
 from eth_monitor.filters import RuledFilter
 from eth_monitor.filters.out import OutFilter
-from eth_monitor.store.file import FileStore
-from eth_monitor.rpc import CacheRPC
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()

eth_monitor/store/file.py

@@ -1,149 +0,0 @@
-# standard imports
-import os
-import json
-import logging
-
-# external imports
-from hexathon import strip_0x
-from chainlib.eth.tx import (
-        Tx,
-        pack,
-        )
-from chainsyncer.backend.file import chain_dir_for
-from leveldir.numeric import NumDir
-from leveldir.hex import HexDir
-
-logg = logging.getLogger(__name__)
-
-base_dir = '/var/lib'
-
-
-class FileStore:
-
-    def put_tx(self, tx, include_data=False):
-        raw = pack(tx.src(), self.chain_spec)
-        tx_hash_dirnormal = strip_0x(tx.hash).upper()
-        tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
-        self.tx_raw_dir.add(tx_hash_bytes, raw)
-        if self.address_rules != None:
-            for a in tx.outputs + tx.inputs:
-                if self.address_rules.apply_rules_addresses(a, a, tx.hash):
-                    a_hex = strip_0x(a).upper()
-                    a = bytes.fromhex(a_hex)
-                    self.address_dir.add_dir(tx_hash_dirnormal, a, b'')
-                    dirpath = self.address_dir.to_filepath(a_hex)
-                    fp = os.path.join(dirpath, '.start')
-                    num = tx.block.number
-                    num_compare = 0
-                    try:
-                        f = open(fp, 'rb')
-                        r = f.read(8)
-                        f.close()
-                        num_compare = int.from_bytes(r, 'big')
-                    except FileNotFoundError:
-                        pass
-
-                    if num_compare == 0 or num < num_compare:
-                        logg.debug('recoding new start block {} for {}'.format(num, a))
-                        num_bytes = num.to_bytes(8, 'big')
-                        f = open(fp, 'wb')
-                        f.write(num_bytes)
-                        f.close()
-
-        if include_data:
-            src = json.dumps(tx.src()).encode('utf-8')
-            self.tx_dir.add(bytes.fromhex(strip_0x(tx.hash)), src)
-
-            rcpt_src = tx.rcpt_src()
-            logg.debug('rcpt {}'.format(rcpt_src))
-            if rcpt_src != None:
-                rcpt_src = json.dumps(rcpt_src).encode('utf-8')
-                self.rcpt_dir.add(bytes.fromhex(strip_0x(tx.hash)), rcpt_src)
-
-
-
-    def put_block(self, block, include_data=False):
-        hash_bytes = bytes.fromhex(strip_0x(block.hash))
-        self.block_num_dir.add(block.number, hash_bytes)
-        num_bytes = block.number.to_bytes(8, 'big')
-        self.block_hash_dir.add(hash_bytes, num_bytes)
-        if include_data:
-            src = json.dumps(block.src()).encode('utf-8')
-            self.block_src_dir.add(hash_bytes, src)
-
-
-    def get_block_number(self, block_number):
-        fp = self.block_num_dir.to_filepath(block_number)
-        f = open(fp, 'rb')
-        r = f.read()
-        f.close()
-        return self.get_block(r.hex())
-
-
-    def get_block(self, block_hash):
-        fp = self.block_src_dir.to_filepath(block_hash)
-        f = open(fp, 'rb')
-        r = f.read()
-        f.close()
-        return r
-
-
-    def get_tx(self, tx_hash):
-        fp = self.tx_dir.to_filepath(tx_hash)
-        f = open(fp, 'rb')
-        r = f.read()
-        f.close()
-        return r
-
-
-    def get_rcpt(self, tx_hash):
-        fp = self.rcpt_dir.to_filepath(tx_hash)
-        f = open(fp, 'rb')
-        r = f.read()
-        f.close()
-        return r
-
-
-    def get_address_tx(self, address):
-        fp = self.address_dir.to_filepath(address)
-        tx_hashes = []
-        for tx_hash in os.listdir(fp):
-            if tx_hash[0] == '.':
-                continue
-            tx_hashes.append(tx_hash)
-        return tx_hashes
-
-
-    def __init__(self, chain_spec, cache_root=base_dir, address_rules=None):
-        self.cache_root = os.path.join(
-            cache_root,
-            'eth_monitor',
-            chain_spec.engine(),
-            chain_spec.fork(),
-            str(chain_spec.chain_id()),
-            )
-        self.cache_root = os.path.realpath(self.cache_root)
-        self.chain_dir = chain_dir_for(self.cache_root)
-        self.cache_dir = os.path.join(self.chain_dir, 'cache')
-        self.block_src_path = os.path.join(self.cache_dir, 'block', 'src')
-        self.block_src_dir = HexDir(self.block_src_path, 32, levels=2)
-        self.block_num_path = os.path.join(self.cache_dir, 'block', 'num')
-        self.block_num_dir = NumDir(self.block_num_path, [100000, 1000])
-        self.block_hash_path = os.path.join(self.cache_dir, 'block', 'hash')
-        self.block_hash_dir = HexDir(self.block_hash_path, 32, levels=2)
-        self.tx_path = os.path.join(self.cache_dir, 'tx', 'src')
-        self.tx_raw_path = os.path.join(self.cache_dir, 'tx', 'raw')
-        self.tx_dir = HexDir(self.tx_path, 32, levels=2)
-        self.tx_raw_dir = HexDir(self.tx_raw_path, 32, levels=2)
-        self.rcpt_path = os.path.join(self.cache_dir, 'rcpt', 'src')
-        self.rcpt_raw_path = os.path.join(self.cache_dir, 'rcpt', 'raw')
-        self.rcpt_dir = HexDir(self.rcpt_path, 32, levels=2)
-        self.rcpt_raw_dir = HexDir(self.rcpt_raw_path, 32, levels=2)
-        self.address_path = os.path.join(self.cache_dir, 'address')
-        self.address_dir = HexDir(self.address_path, 20, levels=2)
-        self.chain_spec = chain_spec
-        self.address_rules = address_rules
-
-
-    def __str__(self):
-        return 'FileStore: root {}'.format(self.cache_root)

eth_monitor/store/null.py

@@ -1,39 +0,0 @@
-# standard imports
-import os
-import logging
-import json
-
-logg = logging.getLogger(__name__)
-
-
-class NullStore:
-
-    def put_tx(self, tx, include_data=False):
-        pass
-
-
-    def put_block(self, block, include_data=False):
-        pass
-
-
-    def get_block_number(self, v):
-        raise FileNotFoundError(v)
-
-
-    def get_block(self, v):
-        raise FileNotFoundError(v)
-
-
-    def get_tx(self, v):
-        raise FileNotFoundError(v)
-
-
-    def get_rcpt(self, v):
-        raise FileNotFoundError(v)
-
-
-    def __init__(self):
-        self.chain_dir = '/dev/null'
-
-    def __str__(self):
-        return "Nullstore"

requirements.txt

@@ -3,3 +3,4 @@ chainlib~=0.0.23
 chainsyncer~=0.1.0
 eth-erc20~=0.1.11
 leveldir~=0.3.0
+eth-cache~=0.1.0