WIP whackamole race condition problems

chaind/adapters/base.py

@@ -1,5 +1,6 @@
 # standard imports
 import logging
+import time
 
 # external imports
 from chainqueue import Store as QueueStore
@@ -24,7 +25,7 @@ class ChaindAdapter:
                 err = None
                 break
             except FileNotFoundError as e:
-                logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(tx_hash, e))
+                logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(e))
                 err = e
                 time.sleep(self.race_delay)
                 continue

chaind/adapters/fs.py

@@ -1,6 +1,7 @@
 # standard imports
 import logging
 import os
+import time
 
 # external imports
 from chainlib.error import RPCException
@@ -43,6 +44,7 @@ class ChaindFsAdapter(ChaindAdapter):
             try:
                 v = self.store.get(tx_hash)
                 err = None
+                break
             except StateInvalid as e:
                 logg.error('I am just a simple syncer and do not know how to handle the state which the tx {} is in: {}'.format(tx_hash, e))
                 return None
@@ -51,7 +53,7 @@ class ChaindFsAdapter(ChaindAdapter):
                 time.sleep(self.race_delay)
                 logg.debug('queuestore get {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
                 continue
-        if v ==None:
+        if err != None:
             raise BackendIntegrityError(tx_hash)
         return v[1]
 
@@ -101,7 +103,22 @@ class ChaindFsAdapter(ChaindAdapter):
 
 
     def dispatch(self, tx_hash):
-        entry = self.store.send_start(tx_hash)
+        entry = None
+        err = None
+        for i in range(3):
+            try:
+                entry = self.store.send_start(tx_hash)
+                err = None
+                break
+            except FileNotFoundError as e:
+                logg.debug('dispatch failed to find {} in backend, will try again: {}'.format(tx_hash, err))
+                err = e
+                time.sleep(self.race_delay)
+                continue
+
+        if err != None:
+            raise BackendIntegrityError('dispatch failed to find {} in backend: {}'.format(tx_hash, err))
+
         tx_wire = entry.serialize()
 
         r = None

chaind/filter.py

@@ -73,22 +73,25 @@ class StateFilter(SyncFilter):
                     queue_adapter.succeed(block, tx)
                 else:
                     queue_adapter.fail(block, tx)
-                break
                 err = None
+                break
             except QueueLockError as e:
                 logg.debug('queue item {} is blocked, will retry: {}'.format(tx.hash, e))
                 time.sleep(delay)
                 delay *= 2
+                race_attempts = 0
                 err = None
             except FileNotFoundError as e:
                 err = e
                 logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
                 race_attempts += 1
+                time.sleep(self.race_delay)
                 continue
             except NotLocalTxError as e:
                 err = e
                 logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
                 race_attempts += 1
+                time.sleep(self.race_delay)
                 continue
 
         if err != None:

requirements.txt

@@ -1,5 +1,5 @@
 chainlib~=0.1.1
-chainqueue~=0.1.6
+chainqueue~=0.1.8
 chainsyncer~=0.4.3
 confini~=0.6.0
 funga~=0.5.2