Add test for tx put, block put
This commit is contained in:
lash
@@ -1,45 +0,0 @@
|
||||
# external imports
|
||||
from hexathon import strip_0x
|
||||
|
||||
|
||||
class AccountRegistry:
|
||||
|
||||
def __init__(self):
|
||||
self.senders = {}
|
||||
self.recipients = {}
|
||||
|
||||
|
||||
def __normalize_address(self, address):
|
||||
return bytes.fromhex(strip_0x(address))
|
||||
|
||||
def add(self, address, label):
|
||||
self.add_sender(address, label)
|
||||
self.add_recipient(address, label)
|
||||
|
||||
|
||||
def add_sender(self, address, label):
|
||||
a = self.__normalize_address(address)
|
||||
self.senders[a] = label
|
||||
|
||||
|
||||
def add_recipient(self, address, label):
|
||||
a = self.__normalize_address(address)
|
||||
self.recipients[a] = label
|
||||
|
||||
|
||||
def have(self, address):
|
||||
if self.get_sender(address) != None:
|
||||
return True
|
||||
if self.get_recipient(address) != None:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_sender(self, address):
|
||||
a = self.__normalize_address(address)
|
||||
return self.senders.get(a)
|
||||
|
||||
|
||||
def get_recipient(self, address):
|
||||
a = self.__normalize_address(address)
|
||||
return self.recipients.get(a)
|
||||
@@ -1,7 +0,0 @@
|
||||
class GasFilter:
|
||||
|
||||
def __init__(self, store):
|
||||
|
||||
|
||||
def filter(self, conn, block, tx, session):
|
||||
self.store.put()
|
||||
81
eth_cache/rpc.py
Normal file
81
eth_cache/rpc.py
Normal file
@@ -0,0 +1,81 @@
|
||||
# standard imports
|
||||
import json
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from jsonrpc_std.parse import jsonrpc_from_dict
|
||||
from hexathon import strip_0x
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
class CacheRPC:
|
||||
|
||||
def __init__(self, rpc, store):
|
||||
self.rpc = rpc
|
||||
self.store = store
|
||||
|
||||
|
||||
def do(self, o):
|
||||
req = jsonrpc_from_dict(o)
|
||||
r = None
|
||||
if req['method'] == 'eth_getBlockByNumber':
|
||||
block_number = req['params'][0]
|
||||
v = int(strip_0x(block_number), 16)
|
||||
try:
|
||||
j = self.store.get_block_number(v)
|
||||
r = json.loads(j)
|
||||
logg.debug('using cached block {} -> {}'.format(v, r['hash']))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
elif req['method'] == 'eth_getBlockByHash':
|
||||
block_hash = req['params'][0]
|
||||
v = strip_0x(block_hash)
|
||||
try:
|
||||
j = self.store.get_block(v)
|
||||
r = json.loads(j)
|
||||
logg.debug('using cached block {}'.format(r['hash']))
|
||||
except FileNotFoundError as e:
|
||||
logg.debug('not found {}'.format(e))
|
||||
pass
|
||||
elif req['method'] == 'eth_getTransactionReceipt':
|
||||
tx_hash = req['params'][0]
|
||||
j = None
|
||||
try:
|
||||
tx_hash = strip_0x(tx_hash)
|
||||
j = self.store.get_rcpt(tx_hash)
|
||||
r = json.loads(j)
|
||||
logg.debug('using cached rcpt {}'.format(tx_hash))
|
||||
except FileNotFoundError as e:
|
||||
logg.debug('no file {}'.format(e))
|
||||
pass
|
||||
|
||||
# elif req['method'] == 'eth_getTransactionByHash':
|
||||
# raise ValueError(o)
|
||||
# elif req['method'] == 'eth_getTransactionByBlockHashAndIndex':
|
||||
# logg.debug('trying tx index {}'.format(o))
|
||||
# v = req['params'][0]
|
||||
# j = None
|
||||
# try:
|
||||
# j = self.store.get_block(v)
|
||||
# except FileNotFoundError:
|
||||
# pass
|
||||
#
|
||||
# if j != None:
|
||||
# o = json.loads(j)
|
||||
# idx = int(req['params'][1], 16)
|
||||
# v = r['transactions'][idx]
|
||||
# j = None
|
||||
# try:
|
||||
# j = self.store.get_tx(v)
|
||||
# except FileNotFoundError:
|
||||
# pass
|
||||
#
|
||||
# if j != None:
|
||||
# r = json.loads(j)
|
||||
# logg.debug('using cached tx {} -> {}'.format(req['params'], r['hash']))
|
||||
|
||||
if r == None:
|
||||
logg.debug('passthru {}'.format(o))
|
||||
r = self.rpc.do(o)
|
||||
|
||||
return r
|
||||
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# standard imports
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import select
|
||||
|
||||
# external imports
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.connection import EthHTTPConnection
|
||||
from chainlib.eth.tx import (
|
||||
pack,
|
||||
receipt,
|
||||
transaction,
|
||||
Tx,
|
||||
)
|
||||
from chainlib.eth.block import (
|
||||
Block,
|
||||
block_by_hash,
|
||||
)
|
||||
from hexathon import strip_0x
|
||||
|
||||
# local imports
|
||||
from eth_cache.store import (
|
||||
PointerHexDir,
|
||||
TxFileStore,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
default_eth_provider = os.environ.get('ETH_PROVIDER', 'http://localhost:8545')
|
||||
default_data_dir = os.path.realpath(os.path.join(os.environ.get('HOME', ''), '.eth_cache'))
|
||||
|
||||
def stdin_arg(t=0):
|
||||
h = select.select([sys.stdin], [], [], t)
|
||||
if len(h[0]) > 0:
|
||||
v = h[0][0].read()
|
||||
return v.rstrip()
|
||||
return None
|
||||
|
||||
|
||||
argparser = argparse.ArgumentParser('eth-get', description='display information about an Ethereum address or transaction', epilog='address/transaction can be provided as an argument or from standard input')
|
||||
argparser.add_argument('-p', '--provider', dest='p', default=default_eth_provider, type=str, help='Web3 provider url (http only)')
|
||||
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='evm:ethereum:1', help='Chain specification string')
|
||||
argparser.add_argument('-v', action='store_true', help='Be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='Be more verbose')
|
||||
argparser.add_argument('--data-dir', dest='data_dir', type=str, help='Be more verbose')
|
||||
argparser.add_argument('tx_hash', nargs='?', default=stdin_arg(), type=str, help='Item to get information for (address og transaction)')
|
||||
args = argparser.parse_args()
|
||||
|
||||
|
||||
if args.vv:
|
||||
logg.setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logg.setLevel(logging.INFO)
|
||||
|
||||
tx_hash = args.tx_hash
|
||||
if tx_hash == None:
|
||||
tx_hash = stdin_arg(t=None)
|
||||
if tx_hash == None:
|
||||
argparser.error('need first positional argument or value from stdin')
|
||||
|
||||
chain_spec = ChainSpec.from_chain_str(args.i)
|
||||
data_dir = args.data_dir
|
||||
if data_dir == None:
|
||||
data_dir = os.path.join(default_data_dir, str(chain_spec))
|
||||
rpc_provider = args.p
|
||||
|
||||
if __name__ == '__main__':
|
||||
rpc = EthHTTPConnection(rpc_provider)
|
||||
o = transaction(tx_hash)
|
||||
r = rpc.do(o)
|
||||
tx = Tx(r)
|
||||
tx_raw = pack(tx.src, chain_spec)
|
||||
|
||||
o = receipt(tx_hash)
|
||||
r = rpc.do(o)
|
||||
tx.apply_receipt(r)
|
||||
|
||||
rcpt = Tx.src_normalize(r)
|
||||
o = block_by_hash(rcpt['block_hash'])
|
||||
r = rpc.do(o)
|
||||
block = Block(r)
|
||||
tx.apply_block(block)
|
||||
|
||||
store_backend = PointerHexDir(data_dir, 32)
|
||||
store_backend.register_pointer('address')
|
||||
store = TxFileStore(chain_spec, store_backend)
|
||||
#store_backend.add(bytes.fromhex(strip_0x(tx_hash)), tx_raw)
|
||||
store.put(block, tx, [])
|
||||
@@ -1,70 +0,0 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import os
|
||||
|
||||
# external imports
|
||||
from hexdir import HexDir
|
||||
from chainlib.eth.tx import pack
|
||||
from hexathon import strip_0x
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PointerHexDir(HexDir):
|
||||
|
||||
def __init__(self, root_path, key_length, levels=2, prefix_length=0):
|
||||
super(PointerHexDir, self).__init__(root_path, key_length, levels, prefix_length)
|
||||
self.pointers = {}
|
||||
|
||||
|
||||
def register_pointer(self, label, dir_name=None):
|
||||
if dir_name == None:
|
||||
dir_name = label
|
||||
pointer_dir = os.path.join(self.path, dir_name)
|
||||
os.makedirs(pointer_dir, exist_ok=True)
|
||||
|
||||
label_file = os.path.join(pointer_dir, '.label')
|
||||
try:
|
||||
os.stat(label_file)
|
||||
except FileNotFoundError:
|
||||
f = open(label_file, 'w')
|
||||
f.write(label)
|
||||
f.close()
|
||||
|
||||
self.pointers[label] = pointer_dir
|
||||
|
||||
|
||||
def add_pointer(self, pointer, pointer_relpath, destination_path):
|
||||
if isinstance(pointer_relpath, list):
|
||||
link_path = os.path.join(self.pointers[pointer], *pointer_relpath)
|
||||
else:
|
||||
link_path = os.path.join(self.pointers[pointer], pointer_relpath)
|
||||
os.makedirs(os.path.dirname(link_path), exist_ok=True)
|
||||
os.symlink(destination_path, link_path)
|
||||
logg.debug('added link {} -> {}'.format(link_path, destination_path))
|
||||
|
||||
|
||||
def add(self, key, content, prefix=b'', pointers={}):
|
||||
(c, entry_path) = super(PointerHexDir, self).add(key, content, prefix=prefix)
|
||||
for k in pointers.keys():
|
||||
self.add_pointer(k, pointers[k], entry_path)
|
||||
|
||||
|
||||
class TxFileStore:
|
||||
|
||||
def __init__(self, chain_spec, backend):
|
||||
self.backend = backend
|
||||
self.chain_spec = chain_spec
|
||||
|
||||
|
||||
def put(self, block, tx, addresses=[], attrs={}):
|
||||
tx_src = tx.as_dict()
|
||||
tx_raw = pack(tx_src, self.chain_spec)
|
||||
|
||||
if len(addresses) == 0:
|
||||
self.backend.add(bytes.fromhex(tx.hash), tx_raw, pointers={})
|
||||
return
|
||||
|
||||
for address in addresses:
|
||||
filename = '{}_{}_{}'.format(block.number, tx.index, strip_0x(tx.hash))
|
||||
self.backend.add(bytes.fromhex(tx.hash), tx_raw, pointers={'address': [strip_0x(address),filename]})
|
||||
154
eth_cache/store/file.py
Normal file
154
eth_cache/store/file.py
Normal file
@@ -0,0 +1,154 @@
|
||||
# standard imports
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from hexathon import strip_0x
|
||||
from chainlib.eth.tx import (
|
||||
Tx,
|
||||
pack,
|
||||
)
|
||||
from leveldir.numeric import NumDir
|
||||
from leveldir.hex import HexDir
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
default_base_dir = '/var/lib'
|
||||
|
||||
|
||||
def chain_dir_for(chain_spec, base_dir=default_base_dir):
|
||||
chain_dir = os.path.join(base_dir, str(chain_spec).replace(':', '/'))
|
||||
return os.path.join(chain_dir, 'eth_cache')
|
||||
|
||||
|
||||
class FileStore:
|
||||
|
||||
def put_tx(self, tx, include_data=False):
|
||||
raw = pack(tx.src(), self.chain_spec)
|
||||
tx_hash_dirnormal = strip_0x(tx.hash).upper()
|
||||
tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
|
||||
self.tx_raw_dir.add(tx_hash_bytes, raw)
|
||||
if self.address_rules != None:
|
||||
for a in tx.outputs + tx.inputs:
|
||||
if self.address_rules.apply_rules_addresses(a, a, tx.hash):
|
||||
a_hex = strip_0x(a).upper()
|
||||
a = bytes.fromhex(a_hex)
|
||||
self.address_dir.add_dir(tx_hash_dirnormal, a, b'')
|
||||
dirpath = self.address_dir.to_filepath(a_hex)
|
||||
fp = os.path.join(dirpath, '.start')
|
||||
num = tx.block.number
|
||||
num_compare = 0
|
||||
try:
|
||||
f = open(fp, 'rb')
|
||||
r = f.read(8)
|
||||
f.close()
|
||||
num_compare = int.from_bytes(r, 'big')
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
if num_compare == 0 or num < num_compare:
|
||||
logg.debug('recoding new start block {} for {}'.format(num, a))
|
||||
num_bytes = num.to_bytes(8, 'big')
|
||||
f = open(fp, 'wb')
|
||||
f.write(num_bytes)
|
||||
f.close()
|
||||
|
||||
if include_data:
|
||||
src = json.dumps(tx.src()).encode('utf-8')
|
||||
self.tx_dir.add(bytes.fromhex(strip_0x(tx.hash)), src)
|
||||
|
||||
rcpt_src = tx.rcpt_src()
|
||||
logg.debug('rcpt {}'.format(rcpt_src))
|
||||
if rcpt_src != None:
|
||||
rcpt_src = json.dumps(rcpt_src).encode('utf-8')
|
||||
self.rcpt_dir.add(bytes.fromhex(strip_0x(tx.hash)), rcpt_src)
|
||||
|
||||
|
||||
|
||||
def put_block(self, block, include_data=False):
|
||||
hash_bytes = bytes.fromhex(strip_0x(block.hash))
|
||||
self.block_num_dir.add(block.number, hash_bytes)
|
||||
num_bytes = block.number.to_bytes(8, 'big')
|
||||
self.block_hash_dir.add(hash_bytes, num_bytes)
|
||||
if include_data:
|
||||
src = json.dumps(block.src()).encode('utf-8')
|
||||
self.block_src_dir.add(hash_bytes, src)
|
||||
|
||||
|
||||
def get_block_number(self, block_number):
|
||||
fp = self.block_num_dir.to_filepath(block_number)
|
||||
f = open(fp, 'rb')
|
||||
r = f.read()
|
||||
f.close()
|
||||
return self.get_block(r.hex())
|
||||
|
||||
|
||||
def get_block(self, block_hash):
|
||||
fp = self.block_src_dir.to_filepath(block_hash)
|
||||
f = open(fp, 'rb')
|
||||
r = f.read()
|
||||
f.close()
|
||||
return r
|
||||
|
||||
|
||||
def get_tx(self, tx_hash):
|
||||
fp = self.tx_dir.to_filepath(tx_hash)
|
||||
f = open(fp, 'rb')
|
||||
r = f.read()
|
||||
f.close()
|
||||
return r
|
||||
|
||||
|
||||
def get_rcpt(self, tx_hash):
|
||||
fp = self.rcpt_dir.to_filepath(tx_hash)
|
||||
f = open(fp, 'rb')
|
||||
r = f.read()
|
||||
f.close()
|
||||
return r
|
||||
|
||||
|
||||
def get_address_tx(self, address):
|
||||
fp = self.address_dir.to_filepath(address)
|
||||
tx_hashes = []
|
||||
for tx_hash in os.listdir(fp):
|
||||
if tx_hash[0] == '.':
|
||||
continue
|
||||
tx_hashes.append(tx_hash)
|
||||
return tx_hashes
|
||||
|
||||
|
||||
def __init__(self, chain_spec, cache_root=default_base_dir, address_rules=None):
|
||||
# self.cache_root = os.path.join(
|
||||
# cache_root,
|
||||
# 'eth_cache',
|
||||
# chain_spec.engine(),
|
||||
# chain_spec.fork(),
|
||||
# str(chain_spec.chain_id()),
|
||||
# )
|
||||
#self.cache_root = os.path.realpath(self.cache_root)
|
||||
#self.chain_dir = chain_dir_for(chain_spec, self.cache_root)
|
||||
self.chain_dir = chain_dir_for(chain_spec, cache_root)
|
||||
self.cache_dir = self.chain_dir
|
||||
self.block_src_path = os.path.join(self.cache_dir, 'block', 'src')
|
||||
self.block_src_dir = HexDir(self.block_src_path, 32, levels=2)
|
||||
self.block_num_path = os.path.join(self.cache_dir, 'block', 'num')
|
||||
self.block_num_dir = NumDir(self.block_num_path, [100000, 1000])
|
||||
self.block_hash_path = os.path.join(self.cache_dir, 'block', 'hash')
|
||||
self.block_hash_dir = HexDir(self.block_hash_path, 32, levels=2)
|
||||
self.tx_path = os.path.join(self.cache_dir, 'tx', 'src')
|
||||
self.tx_raw_path = os.path.join(self.cache_dir, 'tx', 'raw')
|
||||
self.tx_dir = HexDir(self.tx_path, 32, levels=2)
|
||||
self.tx_raw_dir = HexDir(self.tx_raw_path, 32, levels=2)
|
||||
self.rcpt_path = os.path.join(self.cache_dir, 'rcpt', 'src')
|
||||
self.rcpt_raw_path = os.path.join(self.cache_dir, 'rcpt', 'raw')
|
||||
self.rcpt_dir = HexDir(self.rcpt_path, 32, levels=2)
|
||||
self.rcpt_raw_dir = HexDir(self.rcpt_raw_path, 32, levels=2)
|
||||
self.address_path = os.path.join(self.cache_dir, 'address')
|
||||
self.address_dir = HexDir(self.address_path, 20, levels=2)
|
||||
self.chain_spec = chain_spec
|
||||
self.address_rules = address_rules
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return 'FileStore: root {}'.format(self.cache_root)
|
||||
40
eth_cache/store/null.py
Normal file
40
eth_cache/store/null.py
Normal file
@@ -0,0 +1,40 @@
|
||||
# standard imports
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NullStore:
|
||||
|
||||
def put_tx(self, tx, include_data=False):
|
||||
pass
|
||||
|
||||
|
||||
def put_block(self, block, include_data=False):
|
||||
pass
|
||||
|
||||
|
||||
def get_block_number(self, v):
|
||||
raise FileNotFoundError(v)
|
||||
|
||||
|
||||
def get_block(self, v):
|
||||
raise FileNotFoundError(v)
|
||||
|
||||
|
||||
def get_tx(self, v):
|
||||
raise FileNotFoundError(v)
|
||||
|
||||
|
||||
def get_rcpt(self, v):
|
||||
raise FileNotFoundError(v)
|
||||
|
||||
|
||||
def __init__(self):
|
||||
self.chain_dir = '/dev/null'
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return "Nullstore"
|
||||
Reference in New Issue
Block a user