Receive race handling from chainqueue, rehabilitate tests

CHANGELOG

@@ -1,3 +1,6 @@
+- 0.2.0
+	* primitive race condition handling between fs access of sync and queue
+	* re-enable throttling based on in-flight transaction count
 - 0.1.2
 	* add settings object
 - 0.1.0

chaind/adapters/base.py

@@ -1,10 +1,28 @@
 # external imports
 from chainqueue import Store as QueueStore
 
+# local imports
+from chaind.error import BackendIntegrityError
+
 
 class ChaindAdapter:
 
+    race_delay = 0.1
+
     def __init__(self, chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0):
         self.cache_adapter = cache_adapter
         self.dispatcher = dispatcher
-        self.store = QueueStore(chain_spec, state_store, index_store, counter_store, cache=cache)
+        err = None
+        for i in range(3):
+            try:
+                self.store = QueueStore(chain_spec, state_store, index_store, counter_store, cache=cache)
+                err = None
+                break
+            except FileNotFoundError as e:
+                logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(tx_hash, e))
+                err = e
+                time.sleep(self.race_delay)
+                continue
+
+        if err != None:
+            raise BackendIntegrityError(err)

chaind/adapters/fs.py

@@ -15,15 +15,16 @@ from shep.error import StateInvalid
 
 # local imports
 from .base import ChaindAdapter
+from chaind.error import BackendIntegrityError
 
 logg = logging.getLogger(__name__)
 
 
 class ChaindFsAdapter(ChaindAdapter):
 
-    def __init__(self, chain_spec, path, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, digest_bytes=32):
+    def __init__(self, chain_spec, path, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, digest_bytes=32, event_callback=None):
         factory = SimpleFileStoreFactory(path).add
-        state_store = Status(factory)
+        state_store = Status(factory, allow_invalid=True, event_callback=event_callback)
         index_path = os.path.join(path, 'tx')
         index_store = IndexStore(index_path, digest_bytes=digest_bytes)
         counter_store = CounterStore(path)
@@ -37,16 +38,35 @@ class ChaindFsAdapter(ChaindAdapter):
 
     def get(self, tx_hash):
         v = None
-        try:
-            v = self.store.get(tx_hash)
-        except StateInvalid as e:
-            logg.error('I am just a simple syncer and do not know how to handle the state which the tx {} is in: {}'.format(tx_hash, e))
-            return None
+        err = None
+        for i in range(3):
+            try:
+                v = self.store.get(tx_hash)
+                err = None
+            except StateInvalid as e:
+                logg.error('I am just a simple syncer and do not know how to handle the state which the tx {} is in: {}'.format(tx_hash, e))
+                return None
+            except FileNotFoundError as e:
+                err = e
+                time.sleep(self.race_delay)
+                logg.debug('queuestore get {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
+                continue
+        if v ==None:
+            raise BackendIntegrityError(tx_hash)
         return v[1]
 
 
-    def upcoming(self):
-        return self.store.upcoming()
+    def upcoming(self, limit=0):
+        real_limit = 0
+        in_flight = []
+        if limit > 0:
+            in_flight = self.store.by_state(state=self.store.IN_NETWORK, not_state=self.store.FINAL)
+            real_limit = limit - len(in_flight)
+            if real_limit <= 0:
+                return []
+        r = self.store.upcoming(limit=real_limit)
+        logg.info('upcoming returning {} upcoming from limit {} less {} active in-flight txs'.format(len(r), limit, len(in_flight)))
+        return r
 
 
     def pending(self):
@@ -57,6 +77,10 @@ class ChaindFsAdapter(ChaindAdapter):
         return self.store.deferred()
 
 
+    def failed(self):
+        return self.store.failed()
+
+
     def succeed(self, block, tx):
         if self.store.is_reserved(tx.hash):
             raise QueueLockError(tx.hash)
@@ -68,6 +92,10 @@ class ChaindFsAdapter(ChaindAdapter):
         return self.store.final(tx.hash, block, tx, error=True)
 
 
+    def sendfail(self):
+        return self.store.fail(tx.hash)
+
+
     def enqueue(self, tx_hash):
         return self.store.enqueue(tx_hash)
 

chaind/data/config/token.ini

@@ -0,0 +1,2 @@
+[token]
+module =

chaind/error.py

@@ -20,3 +20,7 @@ class ClientInputError(ValueError):
 
 class QueueLockError(Exception):
     pass
+
+
+class BackendIntegrityError(Exception):
+    pass

chaind/filter.py

@@ -6,27 +6,59 @@ import time
 from chainlib.status import Status as TxStatus
 from chainsyncer.filter import SyncFilter
 from chainqueue.error import NotLocalTxError
+from chaind.adapters.fs import ChaindFsAdapter
 
 # local imports
-from .error import QueueLockError
+from .error import (
+        QueueLockError,
+        BackendIntegrityError,
+        )
 
 logg = logging.getLogger(__name__)
 
+
 class StateFilter(SyncFilter):
 
     delay_limit = 3.0
+    race_delay = 0.1
 
-    def __init__(self, adapter, throttler=None):
-        self.adapter = adapter
+    def __init__(self, chain_spec, adapter_path, tx_adapter, throttler=None):
+        self.chain_spec = chain_spec
+        self.adapter_path = adapter_path
+        self.tx_adapter = tx_adapter
         self.throttler = throttler
 
 
     def filter(self, conn, block, tx, session=None):
-        try:
-            cache_tx = self.adapter.get(tx.hash)
-        except NotLocalTxError:
-            logg.debug('skipping not local transaction {}'.format(tx.hash))
-            return False
+        cache_tx = None
+        for i in range(3):
+            queue_adapter = None
+            try:
+                queue_adapter = ChaindFsAdapter(
+                    self.chain_spec,
+                    self.adapter_path,
+                    self.tx_adapter,
+                    None,
+                    )
+            except BackendIntegrityError as e:
+                logg.error('adapter instantiation failed: {}, one more try'.format(e))
+                time.sleep(self.race_delay)
+                continue
+
+            try:
+                cache_tx = queue_adapter.get(tx.hash)
+            except NotLocalTxError:
+                logg.debug('skipping not local transaction {}'.format(tx.hash))
+                return False
+            except BackendIntegrityError as e:
+                logg.error('adapter instantiation failed: {}, one more try'.format(e))
+                time.sleep(self.race_delay)
+                continue
+
+            break
+
+        if cache_tx == None:
+            raise NotLocalTxError(tx.hash)
 
         delay = 0.01
         while True:
@@ -34,9 +66,9 @@ class StateFilter(SyncFilter):
                 raise QueueLockError('The queue lock for tx {} seems to be stuck. Human meddling needed.'.format(tx.hash))
             try:
                 if tx.status == TxStatus.SUCCESS:
-                    self.adapter.succeed(block, tx)
+                    queue_adapter.succeed(block, tx)
                 else:
-                    self.adapter.fail(block, tx)
+                    queue_adapter.fail(block, tx)
                 break
             except QueueLockError as e:
                 logg.debug('queue item {} is blocked, will retry: {}'.format(tx.hash, e))

chaind/session.py

@@ -104,7 +104,6 @@ class SessionController:
             logg.error('invalid input "{}"'.format(data_in_str))
             raise ClientInputError()
 
-        logg.info('recv {} bytes'.format(len(data)))
         return (srvs, data,)
 
 

chaind/unittest/common.py

@@ -1,5 +1,4 @@
 # standard imports
-import unittest
 import hashlib
 import tempfile
 
@@ -9,9 +8,6 @@ from chainlib.status import Status as TxStatus
 from chainlib.chain import ChainSpec
 from chainlib.error import RPCException
 
-# local imports
-from chaind.adapters.fs import ChaindFsAdapter
-
 
 class MockCacheAdapter(CacheTokenTx):
 
@@ -33,7 +29,9 @@ class MockDispatcher:
 
 
     def send(self, v):
-        if v not in self.fails:
+        import sys
+        sys.stderr.write('susu v {} {}\n'.format(v, self.fails))
+        if v in self.fails:
             raise RPCException('{} is in fails'.format(v))
         pass
 
@@ -43,12 +41,3 @@ class MockTx:
     def __init__(self, tx_hash, status=TxStatus.SUCCESS):
         self.hash = tx_hash
         self.status = status
-
-
-class TestChaindFsBase(unittest.TestCase):
-
-    def setUp(self):
-        self.chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-        self.path = tempfile.mkdtemp()
-        self.adapter = ChaindFsAdapter(self.chain_spec, self.path, self.cache_adapter, self.dispatcher)
-

chaind/unittest/fs.py

@@ -0,0 +1,31 @@
+# standard imports
+import unittest
+import tempfile
+import logging
+
+# external imports
+from chainlib.chain import ChainSpec
+
+# local imports
+from chaind.adapters.fs import ChaindFsAdapter
+
+logging.STATETRACE = 5
+logg = logging.getLogger(__name__)
+logg.setLevel(logging.STATETRACE)
+
+
+class TestChaindFsBase(unittest.TestCase):
+
+    def setUp(self):
+        self.chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+        self.path = tempfile.mkdtemp()
+        self.adapter = ChaindFsAdapter(self.chain_spec, self.path, self.cache_adapter, self.dispatcher, event_callback=self.log_state)
+
+
+    def log_state(self, k, from_state, to_state):
+        logg.log(logging.STATETRACE, 'state change {}: {} -> {}'.format(
+            k,
+            from_state,
+            to_state,
+            )
+            )

requirements.txt

@@ -1,6 +1,6 @@
 chainlib~=0.1.1
-chainqueue~=0.1.5
-chainsyncer~=0.4.2
+chainqueue~=0.1.6
+chainsyncer~=0.4.3
 confini~=0.6.0
 funga~=0.5.2
 pyxdg~=0.26

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = chaind
-version = 0.1.3
+version = 0.2.1
 description = Base package for chain queue service
 author = Louis Holbrook
 author_email = dev@holbrook.no

tests/test_fs.py

@@ -15,15 +15,14 @@ from chaind.filter import StateFilter
 from chaind.unittest.common import (
     MockTx,
     MockCacheAdapter,
-    TestChaindFsBase,
+    MockDispatcher,
     )
-
+from chaind.unittest.fs import TestChaindFsBase
 
 logging.basicConfig(level=logging.DEBUG)
 logg = logging.getLogger()
 
 
-
 class TestChaindFs(TestChaindFsBase):
 
     def setUp(self):
@@ -43,12 +42,15 @@ class TestChaindFs(TestChaindFsBase):
         self.assertEqual(data, v)
 
 
-    def test_fs_defer(self):
+    def test_fs_fail(self):
         data = os.urandom(128).hex()
         hsh = self.adapter.put(data)
-        self.dispatcher.add_fail(hsh)
-        self.adapter.dispatch(hsh)
-        txs = self.adapter.deferred()
+        self.dispatcher.add_fail(data)
+
+        r = self.adapter.dispatch(hsh)
+        self.assertFalse(r)
+
+        txs = self.adapter.failed()
         self.assertEqual(len(txs), 1)
 
 
@@ -72,7 +74,7 @@ class TestChaindFs(TestChaindFsBase):
         data = os.urandom(128).hex()
         hsh = self.adapter.put(data)
         
-        fltr = StateFilter(self.adapter)
+        fltr = StateFilter(self.chain_spec, self.path, MockCacheAdapter)
         tx = MockTx(hsh)
         fltr.filter(None, None, tx)
 
@@ -83,10 +85,30 @@ class TestChaindFs(TestChaindFsBase):
         data = os.urandom(128).hex()
         hsh = self.adapter.put(data)
         
-        fltr = StateFilter(self.adapter)
+        fltr = StateFilter(self.chain_spec, self.path, MockCacheAdapter)
         tx = MockTx(hsh, TxStatus.ERROR)
         fltr.filter(None, None, tx)
 
 
+    def test_upcoming(self):
+        drv = QueueDriver(self.adapter)
+
+        txs = []
+        for i in range(10):
+            data = os.urandom(128).hex()
+            hsh = self.adapter.put(data)
+            txs.append(hsh)
+            self.adapter.enqueue(hsh)
+
+        r = self.adapter.upcoming(limit=5)
+        self.assertEqual(len(r), 5)
+
+        r = self.adapter.dispatch(txs[0])
+        self.assertTrue(r)
+
+        r = self.adapter.upcoming(limit=5)
+        self.assertEqual(len(r), 4)
+
+
 if __name__ == '__main__':
     unittest.main()