Revert "Implement mocks on generic tx, block"

This reverts commit dd1879bb91.

CHANGELOG

@@ -1,3 +1,23 @@
+- 0.2.11
+	* Upgrade shep to handle exception in filestore list
+- 0.2.10
+	* Upgrade shep to guarantee state lock atomicity
+- 0.2.9
+	* Minimize instantiations of adapters in filter execution
+- 0.2.8
+	* Upgrade chainsyncer
+- 0.2.7
+	* Upgrade chainlib
+- 0.2.6
+	* Deps upgrade
+- 0.2.5
+	* Deps upgrade
+- 0.2.4
+	* Allow omission of state store sync in queue store backend
+- 0.2.2
+	* Fix missing symbol crashes related to race conditions
+- 0.2.1
+	* Receive removed race checks from chainqueue
 - 0.2.0
 	* primitive race condition handling between fs access of sync and queue
 	* re-enable throttling based on in-flight transaction count

chaind/adapters/base.py

@@ -1,28 +1,27 @@
+# standard imports
+import logging
+import time
+
 # external imports
 from chainqueue import Store as QueueStore
 
 # local imports
-from chaind.error import BackendIntegrityError
+from chaind.lock import StoreLock
+
+logg = logging.getLogger(__name__)
 
 
 class ChaindAdapter:
 
-    race_delay = 0.1
+    def __init__(self, chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, store_sync=True):
-
-    def __init__(self, chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0):
         self.cache_adapter = cache_adapter
         self.dispatcher = dispatcher
-        err = None
+        store_lock = StoreLock()
-        for i in range(3):
+        while True:
             try:
-                self.store = QueueStore(chain_spec, state_store, index_store, counter_store, cache=cache)
+                self.store = QueueStore(chain_spec, state_store, index_store, counter_store, cache=cache, sync=store_sync)
-                err = None
                 break
             except FileNotFoundError as e:
-                logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(tx_hash, e))
+                logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(e))
-                err = e
+                store_lock.again()
-                time.sleep(self.race_delay)
                 continue
-
-        if err != None:
-            raise BackendIntegrityError(err)

chaind/adapters/fs.py

@@ -1,6 +1,7 @@
 # standard imports
 import logging
 import os
+import time
 
 # external imports
 from chainlib.error import RPCException
@@ -11,24 +12,27 @@ from chainqueue.store.fs import (
         CounterStore,
         )
 from shep.store.file import SimpleFileStoreFactory
-from shep.error import StateInvalid
+from shep.error import (
+        StateInvalid,
+        StateLockedKey,
+        )
 
 # local imports
 from .base import ChaindAdapter
-from chaind.error import BackendIntegrityError
+from chaind.lock import StoreLock
 
 logg = logging.getLogger(__name__)
 
 
 class ChaindFsAdapter(ChaindAdapter):
 
-    def __init__(self, chain_spec, path, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, digest_bytes=32, event_callback=None):
+    def __init__(self, chain_spec, path, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, digest_bytes=32, event_callback=None, store_sync=True):
-        factory = SimpleFileStoreFactory(path).add
+        factory = SimpleFileStoreFactory(path, use_lock=True).add
         state_store = Status(factory, allow_invalid=True, event_callback=event_callback)
         index_path = os.path.join(path, 'tx')
         index_store = IndexStore(index_path, digest_bytes=digest_bytes)
         counter_store = CounterStore(path)
-        super(ChaindFsAdapter, self).__init__(chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=cache, pending_retry_threshold=pending_retry_threshold, error_retry_threshold=error_retry_threshold)
+        super(ChaindFsAdapter, self).__init__(chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=cache, pending_retry_threshold=pending_retry_threshold, error_retry_threshold=error_retry_threshold, store_sync=store_sync)
 
 
     def put(self, signed_tx):
@@ -38,21 +42,23 @@ class ChaindFsAdapter(ChaindAdapter):
 
     def get(self, tx_hash):
         v = None
-        err = None
+        store_lock = StoreLock()
-        for i in range(3):
+        while True:
             try:
                 v = self.store.get(tx_hash)
-                err = None
+                break
             except StateInvalid as e:
                 logg.error('I am just a simple syncer and do not know how to handle the state which the tx {} is in: {}'.format(tx_hash, e))
                 return None
             except FileNotFoundError as e:
-                err = e
+                logg.debug('queuestore get (file missing) {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
-                time.sleep(self.race_delay)
+                store_lock.again()
-                logg.debug('queuestore get {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
                 continue
-        if v ==None:
+            except StateLockedKey as e:
-            raise BackendIntegrityError(tx_hash)
+                logg.debug('queuestore get (statelock) {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
+                store_lock.again()
+                continue
+
         return v[1]
 
 
@@ -84,12 +90,19 @@ class ChaindFsAdapter(ChaindAdapter):
     def succeed(self, block, tx):
         if self.store.is_reserved(tx.hash):
             raise QueueLockError(tx.hash)
-
+        r = self.store.final(tx.hash, block, tx, error=False)
-        return self.store.final(tx.hash, block, tx, error=False)
+        (k, v) = self.store.get(tx.hash)
+        self.store.purge(k)
+        return r
 
 
     def fail(self, block, tx):
-        return self.store.final(tx.hash, block, tx, error=True)
+        if self.store.is_reserved(tx.hash):
+            raise QueueLockError(tx.hash)
+        r = self.store.final(tx.hash, block, tx, error=True)
+        (k, v) = self.store.get(tx.hash)
+        self.store.purge(k)
+        return r
 
 
     def sendfail(self):
@@ -101,15 +114,44 @@ class ChaindFsAdapter(ChaindAdapter):
 
 
     def dispatch(self, tx_hash):
-        entry = self.store.send_start(tx_hash)
+        entry = None
+
+        store_lock = StoreLock()
+        while True:
+            try:
+                entry = self.store.send_start(tx_hash)
+                break
+            except FileNotFoundError as e:
+                logg.debug('dispatch failed to find {} in backend, will try again: {}'.format(tx_hash, e))
+                store_lock.again()
+                continue
+            except StateLockedKey as e:
+                logg.debug('dispatch failed to find {} in backend, will try again: {}'.format(tx_hash, e))
+                store_lock.again()
+                continue
+
         tx_wire = entry.serialize()
 
         r = None
         try:
             r = self.dispatcher.send(tx_wire)
-        except RPCException:
+        except RPCException as e:
+            logg.error('dispatch send failed for {}: {}'.format(tx_hash, e))
             self.store.fail(tx_hash)
             return False
 
-        self.store.send_end(tx_hash)
+        store_lock = StoreLock()
+        while True:
+            try:
+                self.store.send_end(tx_hash)
+                break
+            except FileNotFoundError as e:
+                logg.debug('dispatch failed to find {} in backend, will try again: {}'.format(tx_hash, e))
+                store_lock.again(e)
+                continue
+            except StateLockedKey as e:
+                logg.debug('dispatch failed to find {} in backend, will try again: {}'.format(tx_hash, e))
+                store_lock.again(e)
+                continue
+
         return True

chaind/dispatch.py

@@ -0,0 +1,33 @@
+# standard imports
+import logging
+
+# local ipmorts
+from chaind.adapters.fs import ChaindFsAdapter
+from chaind.eth.cache import EthCacheTx
+
+logg = logging.getLogger(__name__)
+
+
+class DispatchProcessor:
+
+    def __init__(self, chain_spec, queue_dir, dispatcher):
+        self.dispatcher = dispatcher
+        self.chain_spec = chain_spec,
+        self.queue_dir = queue_dir
+
+
+    def process(self, rpc, limit=50):
+        adapter = ChaindFsAdapter(
+            self.chain_spec,
+            self.queue_dir, 
+            EthCacheTx,
+            self.dispatcher,
+            )
+        
+        upcoming = adapter.upcoming(limit=limit)
+        logg.info('processor has {} candidates for {}, processing with limit {}'.format(len(upcoming), self.chain_spec, limit))
+        i = 0
+        for tx_hash in upcoming:
+            if adapter.dispatch(tx_hash):
+                i += 1
+        return i

chaind/error.py

@@ -22,5 +22,5 @@ class QueueLockError(Exception):
     pass
 
 
-class BackendIntegrityError(Exception):
+class BackendError(Exception):
     pass

chaind/filter.py

@@ -7,63 +7,89 @@ from chainlib.status import Status as TxStatus
 from chainsyncer.filter import SyncFilter
 from chainqueue.error import NotLocalTxError
 from chaind.adapters.fs import ChaindFsAdapter
+from shep.error import StateLockedKey
 
 # local imports
 from .error import (
         QueueLockError,
-        BackendIntegrityError,
+        BackendError,
         )
+from chaind.lock import StoreLock
 
 logg = logging.getLogger(__name__)
 
 
 class StateFilter(SyncFilter):
 
-    delay_limit = 3.0
-    race_delay = 0.1
-
     def __init__(self, chain_spec, adapter_path, tx_adapter, throttler=None):
         self.chain_spec = chain_spec
         self.adapter_path = adapter_path
         self.tx_adapter = tx_adapter
         self.throttler = throttler
+        self.last_block_height = 0
+        self.adapter = None
+        self.store_lock = None
+
+
+    def __get_adapter(self, block, force_reload=False):
+        if self.store_lock == None:
+            self.store_lock = StoreLock()
+
+        reload = False
+        if block.number != self.last_block_height:
+            reload = True
+        elif self.adapter == None:
+            reload = True
+        elif force_reload:
+            reload = True
+        
+        self.last_block_height = block.number
+
+        if reload:
+            while True:
+                logg.info('reloading adapter')
+                try:
+                    self.adapter = ChaindFsAdapter(
+                        self.chain_spec,
+                        self.adapter_path,
+                        self.tx_adapter,
+                        None,
+                        )
+                    break
+                except BackendError as e:
+                    logg.error('adapter instantiation failed: {}, one more try'.format(e))
+                    self.store_lock.again()
+                    continue
+
+        return self.adapter
 
 
     def filter(self, conn, block, tx, session=None):
         cache_tx = None
-        for i in range(3):
+        queue_adapter = self.__get_adapter(block)
-            queue_adapter = None
+        
-            try:
+        self.store_lock.reset()
-                queue_adapter = ChaindFsAdapter(
+    
-                    self.chain_spec,
+        while True:
-                    self.adapter_path,
-                    self.tx_adapter,
-                    None,
-                    )
-            except BackendIntegrityError as e:
-                logg.error('adapter instantiation failed: {}, one more try'.format(e))
-                time.sleep(self.race_delay)
-                continue
-
             try:
                 cache_tx = queue_adapter.get(tx.hash)
+                break
             except NotLocalTxError:
                 logg.debug('skipping not local transaction {}'.format(tx.hash))
                 return False
-            except BackendIntegrityError as e:
+            except BackendError as e:
-                logg.error('adapter instantiation failed: {}, one more try'.format(e))
+                logg.error('adapter get failed: {}, one more try'.format(e))
-                time.sleep(self.race_delay)
+                self.store_lock.again()
+                queue_adapter = self.__get_adapter(block, force_reload=True)
                 continue
 
-            break
-
         if cache_tx == None:
             raise NotLocalTxError(tx.hash)
 
-        delay = 0.01
+        self.store_lock.reset()
+
+        queue_lock = StoreLock(error=QueueLockError)
         while True:
-            if delay > self.delay_limit:
-                raise QueueLockError('The queue lock for tx {} seems to be stuck. Human meddling needed.'.format(tx.hash))
             try:
                 if tx.status == TxStatus.SUCCESS:
                     queue_adapter.succeed(block, tx)
@@ -72,8 +98,24 @@ class StateFilter(SyncFilter):
                 break
             except QueueLockError as e:
                 logg.debug('queue item {} is blocked, will retry: {}'.format(tx.hash, e))
-                time.sleep(delay)
+                queue_lock.again()
-                delay *= 2
+            except FileNotFoundError as e:
+                logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
+                self.store_lock.again()
+                queue_adapter = self.__get_adapter(block, force_reload=True)
+                continue
+            except NotLocalTxError as e:
+                logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
+                self.store_lock.again()
+                queue_adapter = self.__get_adapter(block, force_reload=True)
+                continue
+            except StateLockedKey as e:
+                logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
+                self.store_lock.again()
+                queue_adapter = self.__get_adapter(block, force_reload=True)
+                continue
+
+        logg.info('filter registered {} for {} in {}'.format(tx.status.name, tx.hash, block))
 
         if self.throttler != None:
             self.throttler.dec(tx.hash)

chaind/lock.py

@@ -0,0 +1,34 @@
+# standard imports
+import time
+
+# local imports
+from .error import BackendError
+
+BASE_DELAY = 0.01
+BASE_DELAY_LIMIT = 10.0
+
+
+class StoreLock:
+
+    def __init__(self, delay=BASE_DELAY, delay_limit=BASE_DELAY_LIMIT, error=BackendError, description=None):
+        self.base_delay = delay
+        self.delay = delay
+        self.delay_limit = delay_limit
+        self.error = error
+        self.description = description
+
+    
+    def again(self, e=None):
+        if self.delay > self.delay_limit:
+            err = None
+            if e != None:
+                err = str(e)
+            else:
+                err = self.description
+            raise self.error(err)
+        time.sleep(self.delay)
+        self.delay *= 2
+
+
+    def reset(self):
+        self.delay = self.base_delay

chaind/session.py

@@ -8,19 +8,21 @@ import stat
 from hexathon import strip_0x
 
 # local imports
-from chaind.error import (
+from .error import (
         NothingToDoError,
         ClientGoneError,
         ClientBlockError,
         ClientInputError,
         )
+from .lock import StoreLock
+from .error import BackendError
 
 logg = logging.getLogger(__name__)
 
 
 class SessionController:
 
-    def __init__(self, config, adapter, processor):
+    def __init__(self, config, processor):
         self.dead = False
         os.makedirs(os.path.dirname(config.get('SESSION_SOCKET_PATH')), exist_ok=True)
         try:
@@ -35,7 +37,6 @@ class SessionController:
         self.srv.settimeout(float(config.get('SESSION_DISPATCH_DELAY')))
         self.processor = processor
         self.chain_spec = config.get('CHAIN_SPEC')
-        self.adapter = adapter
 
 
     def shutdown(self, signo, frame):
@@ -59,7 +60,16 @@ class SessionController:
 
 
     def process(self, conn):
-        r = self.processor(self.chain_spec, self.adapter, conn)
+        state_lock = StoreLock()
+        r = None
+        while True:
+            try:
+                r = self.processor(conn)
+                break
+            except BackendError as e:
+                state_lock.again(e)
+                continue
+                
         if r > 0:
             self.srv.settimeout(0.1)
         else:

chaind/unittest/common.py

@@ -29,8 +29,6 @@ class MockDispatcher:
 
 
     def send(self, v):
-        import sys
-        sys.stderr.write('susu v {} {}\n'.format(v, self.fails))
         if v in self.fails:
             raise RPCException('{} is in fails'.format(v))
         pass
@@ -41,3 +39,9 @@ class MockTx:
     def __init__(self, tx_hash, status=TxStatus.SUCCESS):
         self.hash = tx_hash
         self.status = status
+
+
+class MockBlock:
+
+    def __init__(self, number):
+        self.number = number

requirements.txt

@@ -1,6 +1,6 @@
-chainlib~=0.1.1
+chainlib~=0.1.2
-chainqueue~=0.1.6
+chainqueue~=0.1.15
-chainsyncer~=0.4.3
+chainsyncer~=0.4.7
 confini~=0.6.0
 funga~=0.5.2
 pyxdg~=0.26

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = chaind
-version = 0.2.1
+version = 0.2.11
 description = Base package for chain queue service
 author = Louis Holbrook
 author_email = dev@holbrook.no

tests/test_fs.py

@@ -14,6 +14,7 @@ from chaind.filter import StateFilter
 # test imports
 from chaind.unittest.common import (
     MockTx,
+    MockBlock,
     MockCacheAdapter,
     MockDispatcher,
     )
@@ -76,7 +77,8 @@ class TestChaindFs(TestChaindFsBase):
         
         fltr = StateFilter(self.chain_spec, self.path, MockCacheAdapter)
         tx = MockTx(hsh)
-        fltr.filter(None, None, tx)
+        block = MockBlock(42)
+        fltr.filter(None, block, tx)
 
 
     def test_fs_filter_fail(self):
@@ -87,7 +89,8 @@ class TestChaindFs(TestChaindFsBase):
         
         fltr = StateFilter(self.chain_spec, self.path, MockCacheAdapter)
         tx = MockTx(hsh, TxStatus.ERROR)
-        fltr.filter(None, None, tx)
+        block = MockBlock(42)
+        fltr.filter(None, block, tx)
 
 
     def test_upcoming(self):