Implement chainlib basedir override

chainsyncer/driver/__init__.py

@@ -0,0 +1 @@
+from .base import Syncer

chainsyncer/driver/base.py

@@ -0,0 +1,73 @@
+# standard imports
+import uuid
+import logging
+import time
+import signal
+import json
+
+# external imports
+from chainlib.error import JSONRPCException
+
+# local imports
+from chainsyncer.filter import SyncFilter
+from chainsyncer.error import (
+        SyncDone,
+        NoBlockForYou,
+    )
+
+logg = logging.getLogger(__name__)
+
+
+def noop_callback(block, tx):
+    logg.debug('noop callback ({},{})'.format(block, tx))
+
+
+class Syncer:
+
+    running_global = True
+    yield_delay=0.005
+    signal_request = [signal.SIGINT, signal.SIGTERM]
+    signal_set = False
+    name = 'base'
+
+    def __init__(self, backend, chain_interface, pre_callback=None, block_callback=None, post_callback=None):
+        self.chain_interface = chain_interface
+        self.cursor = None
+        self.running = True
+        self.backend = backend
+        self.filter = SyncFilter(backend)
+        self.block_callback = block_callback
+        self.pre_callback = pre_callback
+        self.post_callback = post_callback
+        if not Syncer.signal_set:
+            for sig in Syncer.signal_request:
+                signal.signal(sig, self.__sig_terminate)
+            Syncer.signal_set = True
+
+
+    def __sig_terminate(self, sig, frame):
+        logg.warning('got signal {}'.format(sig))
+        self.terminate()
+
+
+    def terminate(self):
+        logg.info('termination requested!')
+        Syncer.running_global = False
+        Syncer.running = False
+
+
+    def add_filter(self, f):
+        self.filter.add(f)
+        self.backend.register_filter(str(f))
+
+
+    def process_single(self, conn, block, tx):
+        self.backend.set(block.number, tx.index)
+        self.filter.apply(conn, block, tx)
+
+    
+    def __str__(self):
+        return 'syncer "{}" {}'.format(
+                self.name,
+                self.backend,
+                )

chainsyncer/driver/head.py

@@ -0,0 +1,52 @@
+# standard imports
+import logging
+
+# local imports
+from chainsyncer.error import NoBlockForYou
+from .poll import BlockPollSyncer
+
+logg = logging.getLogger(__name__)
+
+class HeadSyncer(BlockPollSyncer):
+
+    name = 'head'
+
+    def process(self, conn, block):
+        (pair, fltr) = self.backend.get()
+        logg.debug('process block {} (backend {}:{})'.format(block, pair, fltr))
+        i = pair[1] # set tx index from previous
+        tx = None
+        while True:
+            try:
+                tx = block.tx(i)
+            except AttributeError:
+                o = tx(block.txs[i])
+                r = conn.do(o)
+                tx = self.interface.tx_from_src(Tx.src_normalize(r), block=block)
+            #except IndexError as e:
+            #    logg.debug('index error syncer tx get {}'.format(e))
+            #    break
+
+            # TODO: Move specifics to eth subpackage, receipts are not a global concept
+            rcpt = conn.do(self.chain_interface.tx_receipt(tx.hash))
+            if rcpt != None:
+                tx.apply_receipt(self.chain_interface.src_normalize(rcpt))
+
+            self.process_single(conn, block, tx)
+            self.backend.reset_filter()
+                        
+            i += 1
+        
+
+    def get(self, conn):
+        (height, flags) = self.backend.get()
+        block_number = height[0]
+        block_hash = []
+        o = self.chain_interface.block_by_number(block_number)
+        r = conn.do(o)
+        if r == None:
+            raise NoBlockForYou()
+        b = self.chain_interface.block_from_src(r)
+        b.txs = b.txs[height[1]:]
+
+        return b

chainsyncer/driver/history.py

@@ -0,0 +1,45 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.error import RPCException
+
+# local imports
+from .head import HeadSyncer
+from chainsyncer.error import SyncDone
+from chainlib.error import RPCException
+
+logg = logging.getLogger(__name__)
+
+
+class HistorySyncer(HeadSyncer):
+
+    name = 'history'
+
+    def __init__(self, backend, chain_interface, pre_callback=None, block_callback=None, post_callback=None):
+        super(HeadSyncer, self).__init__(backend, chain_interface, pre_callback, block_callback, post_callback)
+        self.block_target = None
+        (block_number, flags) = self.backend.target()
+        if block_number == None:
+            raise AttributeError('backend has no future target. Use HeadSyner instead')
+        self.block_target = block_number
+        logg.debug('block target {}'.format(self.block_target))
+
+
+    def get(self, conn):
+        (height, flags) = self.backend.get()
+        if self.block_target < height[0]:
+            raise SyncDone(self.block_target)
+        block_number = height[0]
+        block_hash = []
+        o = self.chain_interface.block_by_number(block_number)
+        try:
+            r = conn.do(o)
+        # TODO: Disambiguate whether error is temporary or permanent, if permanent, SyncDone should be raised, because a historical sync is attempted into the future
+        except RPCException:
+            r = None
+        if r == None:
+            raise SyncDone() #NoBlockForYou()
+        b = self.chain_interface.block_from_src(r)
+
+        return b

chainsyncer/driver/poll.py

@@ -0,0 +1,59 @@
+# standard imports
+import logging
+import time
+
+# local imports
+from .base import Syncer
+from chainsyncer.error import (
+        SyncDone,
+        NoBlockForYou,
+        )
+
+logg = logging.getLogger(__name__)
+
+
+
+class BlockPollSyncer(Syncer):
+
+    name = 'blockpoll'
+
+    def __init__(self, backend, chain_interface, pre_callback=None, block_callback=None, post_callback=None):
+        super(BlockPollSyncer, self).__init__(backend, chain_interface, pre_callback, block_callback, post_callback)
+
+
+    def loop(self, interval, conn):
+        (pair, fltr) = self.backend.get()
+        start_tx = pair[1]
+
+        while self.running and Syncer.running_global:
+            if self.pre_callback != None:
+                self.pre_callback()
+            while True and Syncer.running_global:
+                if start_tx > 0:
+                    start_tx -= 1
+                    continue
+                try:
+                    block = self.get(conn)
+                except SyncDone as e:
+                    logg.info('all blocks sumitted for processing: {}'.format(e))
+                    return self.backend.get()
+                except NoBlockForYou as e:
+                    break
+# TODO: To properly handle this, ensure that previous request is rolled back
+#                except sqlalchemy.exc.OperationalError as e:
+#                    logg.error('database error: {}'.format(e))
+#                    break
+
+                if self.block_callback != None:
+                    self.block_callback(block, None)
+
+                last_block = block
+                try:
+                    self.process(conn, block)
+                except IndexError:
+                    self.backend.set(block.number + 1, 0)
+                start_tx = 0
+                time.sleep(self.yield_delay)
+            if self.post_callback != None:
+                self.post_callback()
+            time.sleep(interval)

chainsyncer/driver/thread.py

@@ -0,0 +1,133 @@
+# standard imports
+import logging
+#import threading
+import multiprocessing
+import queue
+
+# external imports
+from chainlib.error import RPCException
+
+# local imports
+from .history import HistorySyncer
+from chainsyncer.error import SyncDone
+
+logg = logging.getLogger(__name__)
+
+
+
+class ThreadedHistorySyncer(HistorySyncer):
+
+    def __init__(self, conn_factory, thread_limit, backend, chain_interface, pre_callback=None, block_callback=None, post_callback=None, conn_limit=0):
+        super(ThreadedHistorySyncer, self).__init__(backend, chain_interface, pre_callback, block_callback, post_callback)
+        self.workers = []
+        if conn_limit == 0:
+            conn_limit = thread_limit
+        #self.conn_pool = queue.Queue(conn_limit)
+        #self.queue = queue.Queue(thread_limit)
+        #self.quit_queue = queue.Queue(1)
+        self.conn_pool = multiprocessing.Queue(conn_limit)
+        self.queue = multiprocessing.Queue(thread_limit)
+        self.quit_queue = multiprocessing.Queue(1)
+        #self.lock = threading.Lock()
+        self.lock = multiprocessing.Lock()
+        for i in range(thread_limit):
+            #w = threading.Thread(target=self.worker)
+            w = multiprocessing.Process(target=self.worker)
+            self.workers.append(w)
+
+        for i in range(conn_limit):
+            self.conn_pool.put(conn_factory())
+
+
+    def terminate(self):
+        self.quit_queue.put(())
+        super(ThreadedHistorySyncer, self).terminate()
+
+
+    def worker(self):
+        while True:
+            block_number = None
+            try:
+                block_number = self.queue.get(timeout=0.01)
+            except queue.Empty:
+                if self.quit_queue.qsize() > 0:
+                    #logg.debug('{} received quit'.format(threading.current_thread().getName()))
+                    logg.debug('{} received quit'.format(multiprocessing.current_process().name))
+                    return
+                continue
+            conn = self.conn_pool.get()
+            try:
+                logg.debug('processing parent {} {}'.format(conn, block_number))
+                self.process_parent(conn, block_number)
+            except IndexError:
+                pass
+            except RPCException as e:
+                logg.error('RPC failure for block {}, resubmitting to queue: {}'.format(block, e))
+                self.queue.put(block_number)
+            conn = self.conn_pool.put(conn)
+
+
+    def process_parent(self, conn, block_number):
+            logg.debug('getting block {}'.format(block_number))
+            o = self.chain_interface.block_by_number(block_number)
+            r = conn.do(o)
+            block = self.chain_interface.block_from_src(r)
+            logg.debug('got block typ {}'.format(type(block)))
+            super(ThreadedHistorySyncer, self).process(conn, block)
+
+
+    def process_single(self, conn, block, tx):
+        self.filter.apply(conn, block, tx)
+
+
+    def process(self, conn, block):
+        pass
+
+
+    #def process(self, conn, block):
+    def get(self, conn):
+        if not self.running:
+            raise SyncDone()
+
+        block_number = None
+        tx_index = None
+        flags = None
+        ((block_number, tx_index), flags) = self.backend.get()
+        try:
+            #logg.debug('putting {}'.format(block.number))
+            #self.queue.put((conn, block_number,), timeout=0.1)
+            self.queue.put(block_number, timeout=0.1)
+        except queue.Full:
+            #logg.debug('queue full, try again')
+            return
+   
+        target, flags = self.backend.target()
+        next_block = block_number + 1
+        if next_block > target:
+            self.quit_queue.put(())
+            raise SyncDone()
+        self.backend.set(self.backend.block_height + 1, 0)
+
+
+#    def get(self, conn):
+#        try:
+#            r = super(ThreadedHistorySyncer, self).get(conn)
+#            return r
+#        except SyncDone as e:
+#            self.quit_queue.put(())
+#            raise e
+
+
+    def loop(self, interval, conn):
+        for w in self.workers:
+            w.start()
+        r = super(ThreadedHistorySyncer, self).loop(interval, conn)
+        for w in self.workers:
+            w.join()
+        while True:
+            try:
+                self.quit_queue.get_nowait()
+            except queue.Empty:
+                break
+
+        logg.info('workers done {}'.format(r))

chainsyncer/driver/threadpool.py

@@ -0,0 +1,171 @@
+# standard imports
+import logging
+#import threading
+import multiprocessing
+import queue
+import time
+
+# external imports
+from chainlib.error import RPCException
+
+# local imports
+from .history import HistorySyncer
+from chainsyncer.error import SyncDone
+
+logg = logging.getLogger(__name__)
+
+
+def foobarcb(v):
+    logg.debug('foooz {}'.format(v))
+
+
+class ThreadPoolTask:
+
+    process_func = None
+    chain_interface = None
+
+    def poolworker(self, block_number, conn):
+#            conn = args[1].get()
+            try:
+                logg.debug('processing parent {} {}'.format(conn, block_number))
+                #self.process_parent(self.conn, block_number)
+                self.process_parent(conn, block_number)
+            except IndexError:
+                pass
+            except RPCException as e:
+                logg.error('RPC failure for block {}, resubmitting to queue: {}'.format(block, e))
+                raise e
+                #self.queue.put(block_number)
+#            conn = self.conn_pool.put(conn)
+
+    def process_parent(self, conn, block_number):
+            logg.debug('getting block {}'.format(block_number))
+            o = self.chain_interface.block_by_number(block_number)
+            r = conn.do(o)
+            block = self.chain_interface.block_from_src(r)
+            logg.debug('got block typ {}'.format(type(block)))
+            #super(ThreadedHistorySyncer, self).process(conn, block)
+            self.process_func(conn, block)
+
+
+
+class ThreadPoolHistorySyncer(HistorySyncer):
+
+    def __init__(self, conn_factory, thread_limit, backend, chain_interface, pre_callback=None, block_callback=None, post_callback=None, conn_limit=0):
+        super(ThreadPoolHistorySyncer, self).__init__(backend, chain_interface, pre_callback, block_callback, post_callback)
+        self.workers = []
+        self.thread_limit = thread_limit
+        if conn_limit == 0:
+            self.conn_limit = self.thread_limit
+        #self.conn_pool = queue.Queue(conn_limit)
+        #self.queue = queue.Queue(thread_limit)
+        #self.quit_queue = queue.Queue(1)
+        #self.conn_pool = multiprocessing.Queue(conn_limit)
+        #self.queue = multiprocessing.Queue(thread_limit)
+        #self.quit_queue = multiprocessing.Queue(1)
+        #self.lock = threading.Lock()
+        #self.lock = multiprocessing.Lock()
+        ThreadPoolTask.process_func = super(ThreadPoolHistorySyncer, self).process
+        ThreadPoolTask.chain_interface = chain_interface
+        #for i in range(thread_limit):
+            #w = threading.Thread(target=self.worker)
+        #    w = multiprocessing.Process(target=self.worker)
+        #    self.workers.append(w)
+
+        #for i in range(conn_limit):
+        #    self.conn_pool.put(conn_factory())
+        self.conn_factory = conn_factory
+        self.worker_pool = None
+
+
+    def terminate(self):
+        #self.quit_queue.put(())
+        super(ThreadPoolHistorySyncer, self).terminate()
+
+
+#    def worker(self):
+#        while True:
+#            block_number = None
+#            try:
+#                block_number = self.queue.get(timeout=0.01)
+#            except queue.Empty:
+#                if self.quit_queue.qsize() > 0:
+#                    #logg.debug('{} received quit'.format(threading.current_thread().getName()))
+#                    logg.debug('{} received quit'.format(multiprocessing.current_process().name))
+#                    return
+#                continue
+#            conn = self.conn_pool.get()
+#            try:
+#                logg.debug('processing parent {} {}'.format(conn, block_number))
+#                self.process_parent(conn, block_number)
+#            except IndexError:
+#                pass
+#            except RPCException as e:
+#                logg.error('RPC failure for block {}, resubmitting to queue: {}'.format(block, e))
+#                self.queue.put(block_number)
+#            conn = self.conn_pool.put(conn)
+#
+
+
+    def process_single(self, conn, block, tx):
+        self.filter.apply(conn, block, tx)
+
+
+    def process(self, conn, block):
+        pass
+
+
+    #def process(self, conn, block):
+    def get(self, conn):
+        if not self.running:
+            raise SyncDone()
+
+        block_number = None
+        tx_index = None
+        flags = None
+        ((block_number, tx_index), flags) = self.backend.get()
+        #try:
+            #logg.debug('putting {}'.format(block.number))
+            #self.queue.put((conn, block_number,), timeout=0.1)
+            #self.queue.put(block_number, timeout=0.1)
+        #except queue.Full:
+            #logg.debug('queue full, try again')
+        #    return
+        task = ThreadPoolTask()
+        conn = self.conn_factory()
+        self.worker_pool.apply_async(task.poolworker, (block_number, conn,), {}, foobarcb)
+
+        target, flags = self.backend.target()
+        next_block = block_number + 1
+        if next_block > target:
+            #self.quit_queue.put(())
+            self.worker_pool.close()
+            raise SyncDone()
+        self.backend.set(self.backend.block_height + 1, 0)
+
+
+#    def get(self, conn):
+#        try:
+#            r = super(ThreadedHistorySyncer, self).get(conn)
+#            return r
+#        except SyncDone as e:
+#            self.quit_queue.put(())
+#            raise e
+
+
+    def loop(self, interval, conn):
+        self.worker_pool = multiprocessing.Pool(self.thread_limit)
+        #for w in self.workers:
+        #    w.start()
+        r = super(ThreadPoolHistorySyncer, self).loop(interval, conn)
+        #for w in self.workers:
+        #    w.join()
+        #while True:
+        #    try:
+        #     self.quit_queue.get_nowait()
+        #    except queue.Empty:
+        #        break
+        time.sleep(1)
+        self.worker_pool.join()
+
+        logg.info('workers done {}'.format(r))