134 lines
4.3 KiB
Python
134 lines
4.3 KiB
Python
|
# standard imports
|
|||
|
|
import logging
|
|||
|
|
#import threading
|
|||
|
|
import multiprocessing
|
|||
|
|
import queue
|
|||
|
|
|
|||
|
|
# external imports
|
|||
|
|
from chainlib.error import RPCException
|
|||
|
|
|
|||
|
|
# local imports
|
|||
|
|
from .history import HistorySyncer
|
|||
|
|
from chainsyncer.error import SyncDone
|
|||
|
|
|
|||
|
|
logg = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ThreadedHistorySyncer(HistorySyncer):
|
|||
|
|
|
|||
|
|
def __init__(self, conn_factory, thread_limit, backend, chain_interface, pre_callback=None, block_callback=None, post_callback=None, conn_limit=0):
|
|||
|
|
super(ThreadedHistorySyncer, self).__init__(backend, chain_interface, pre_callback, block_callback, post_callback)
|
|||
|
|
self.workers = []
|
|||
|
|
if conn_limit == 0:
|
|||
|
|
conn_limit = thread_limit
|
|||
|
|
#self.conn_pool = queue.Queue(conn_limit)
|
|||
|
|
#self.queue = queue.Queue(thread_limit)
|
|||
|
|
#self.quit_queue = queue.Queue(1)
|
|||
|
|
self.conn_pool = multiprocessing.Queue(conn_limit)
|
|||
|
|
self.queue = multiprocessing.Queue(thread_limit)
|
|||
|
|
self.quit_queue = multiprocessing.Queue(1)
|
|||
|
|
#self.lock = threading.Lock()
|
|||
|
|
self.lock = multiprocessing.Lock()
|
|||
|
|
for i in range(thread_limit):
|
|||
|
|
#w = threading.Thread(target=self.worker)
|
|||
|
|
w = multiprocessing.Process(target=self.worker)
|
|||
|
|
self.workers.append(w)
|
|||
|
|
|
|||
|
|
for i in range(conn_limit):
|
|||
|
|
self.conn_pool.put(conn_factory())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def terminate(self):
|
|||
|
|
self.quit_queue.put(())
|
|||
|
|
super(ThreadedHistorySyncer, self).terminate()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def worker(self):
|
|||
|
|
while True:
|
|||
|
|
block_number = None
|
|||
|
|
try:
|
|||
|
|
block_number = self.queue.get(timeout=0.01)
|
|||
|
|
except queue.Empty:
|
|||
|
|
if self.quit_queue.qsize() > 0:
|
|||
|
|
#logg.debug('{} received quit'.format(threading.current_thread().getName()))
|
|||
|
|
logg.debug('{} received quit'.format(multiprocessing.current_process().name))
|
|||
|
|
return
|
|||
|
|
continue
|
|||
|
|
conn = self.conn_pool.get()
|
|||
|
|
try:
|
|||
|
|
logg.debug('processing parent {} {}'.format(conn, block_number))
|
|||
|
|
self.process_parent(conn, block_number)
|
|||
|
|
except IndexError:
|
|||
|
|
pass
|
|||
|
|
except RPCException as e:
|
|||
|
|
logg.error('RPC failure for block {}, resubmitting to queue: {}'.format(block, e))
|
|||
|
|
self.queue.put(block_number)
|
|||
|
|
conn = self.conn_pool.put(conn)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def process_parent(self, conn, block_number):
|
|||
|
|
logg.debug('getting block {}'.format(block_number))
|
|||
|
|
o = self.chain_interface.block_by_number(block_number)
|
|||
|
|
r = conn.do(o)
|
|||
|
|
block = self.chain_interface.block_from_src(r)
|
|||
|
|
logg.debug('got block typ {}'.format(type(block)))
|
|||
|
|
super(ThreadedHistorySyncer, self).process(conn, block)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def process_single(self, conn, block, tx):
|
|||
|
|
self.filter.apply(conn, block, tx)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def process(self, conn, block):
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
|
|||
|
|
#def process(self, conn, block):
|
|||
|
|
def get(self, conn):
|
|||
|
|
if not self.running:
|
|||
|
|
raise SyncDone()
|
|||
|
|
|
|||
|
|
block_number = None
|
|||
|
|
tx_index = None
|
|||
|
|
flags = None
|
|||
|
|
((block_number, tx_index), flags) = self.backend.get()
|
|||
|
|
try:
|
|||
|
|
#logg.debug('putting {}'.format(block.number))
|
|||
|
|
#self.queue.put((conn, block_number,), timeout=0.1)
|
|||
|
|
self.queue.put(block_number, timeout=0.1)
|
|||
|
|
except queue.Full:
|
|||
|
|
#logg.debug('queue full, try again')
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
target, flags = self.backend.target()
|
|||
|
|
next_block = block_number + 1
|
|||
|
|
if next_block > target:
|
|||
|
|
self.quit_queue.put(())
|
|||
|
|
raise SyncDone()
|
|||
|
|
self.backend.set(self.backend.block_height + 1, 0)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# def get(self, conn):
|
|||
|
|
# try:
|
|||
|
|
# r = super(ThreadedHistorySyncer, self).get(conn)
|
|||
|
|
# return r
|
|||
|
|
# except SyncDone as e:
|
|||
|
|
# self.quit_queue.put(())
|
|||
|
|
# raise e
|
|||
|
|
|
|||
|
|
|
|||
|
|
def loop(self, interval, conn):
|
|||
|
|
for w in self.workers:
|
|||
|
|
w.start()
|
|||
|
|
r = super(ThreadedHistorySyncer, self).loop(interval, conn)
|
|||
|
|
for w in self.workers:
|
|||
|
|
w.join()
|
|||
|
|
while True:
|
|||
|
|
try:
|
|||
|
|
self.quit_queue.get_nowait()
|
|||
|
|
except queue.Empty:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
logg.info('workers done {}'.format(r))
|