WIP whackamole race condition problems
This commit is contained in:
parent
5102b4ac6e
commit
5d61506133
@ -1,5 +1,6 @@
|
|||||||
# standard imports
|
# standard imports
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
from chainqueue import Store as QueueStore
|
from chainqueue import Store as QueueStore
|
||||||
@ -24,7 +25,7 @@ class ChaindAdapter:
|
|||||||
err = None
|
err = None
|
||||||
break
|
break
|
||||||
except FileNotFoundError as e:
|
except FileNotFoundError as e:
|
||||||
logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(tx_hash, e))
|
logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(e))
|
||||||
err = e
|
err = e
|
||||||
time.sleep(self.race_delay)
|
time.sleep(self.race_delay)
|
||||||
continue
|
continue
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
# standard imports
|
# standard imports
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
from chainlib.error import RPCException
|
from chainlib.error import RPCException
|
||||||
@ -43,6 +44,7 @@ class ChaindFsAdapter(ChaindAdapter):
|
|||||||
try:
|
try:
|
||||||
v = self.store.get(tx_hash)
|
v = self.store.get(tx_hash)
|
||||||
err = None
|
err = None
|
||||||
|
break
|
||||||
except StateInvalid as e:
|
except StateInvalid as e:
|
||||||
logg.error('I am just a simple syncer and do not know how to handle the state which the tx {} is in: {}'.format(tx_hash, e))
|
logg.error('I am just a simple syncer and do not know how to handle the state which the tx {} is in: {}'.format(tx_hash, e))
|
||||||
return None
|
return None
|
||||||
@ -51,7 +53,7 @@ class ChaindFsAdapter(ChaindAdapter):
|
|||||||
time.sleep(self.race_delay)
|
time.sleep(self.race_delay)
|
||||||
logg.debug('queuestore get {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
|
logg.debug('queuestore get {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
|
||||||
continue
|
continue
|
||||||
if v ==None:
|
if err != None:
|
||||||
raise BackendIntegrityError(tx_hash)
|
raise BackendIntegrityError(tx_hash)
|
||||||
return v[1]
|
return v[1]
|
||||||
|
|
||||||
@ -101,7 +103,22 @@ class ChaindFsAdapter(ChaindAdapter):
|
|||||||
|
|
||||||
|
|
||||||
def dispatch(self, tx_hash):
|
def dispatch(self, tx_hash):
|
||||||
|
entry = None
|
||||||
|
err = None
|
||||||
|
for i in range(3):
|
||||||
|
try:
|
||||||
entry = self.store.send_start(tx_hash)
|
entry = self.store.send_start(tx_hash)
|
||||||
|
err = None
|
||||||
|
break
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
logg.debug('dispatch failed to find {} in backend, will try again: {}'.format(tx_hash, err))
|
||||||
|
err = e
|
||||||
|
time.sleep(self.race_delay)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if err != None:
|
||||||
|
raise BackendIntegrityError('dispatch failed to find {} in backend: {}'.format(tx_hash, err))
|
||||||
|
|
||||||
tx_wire = entry.serialize()
|
tx_wire = entry.serialize()
|
||||||
|
|
||||||
r = None
|
r = None
|
||||||
|
@ -73,22 +73,25 @@ class StateFilter(SyncFilter):
|
|||||||
queue_adapter.succeed(block, tx)
|
queue_adapter.succeed(block, tx)
|
||||||
else:
|
else:
|
||||||
queue_adapter.fail(block, tx)
|
queue_adapter.fail(block, tx)
|
||||||
break
|
|
||||||
err = None
|
err = None
|
||||||
|
break
|
||||||
except QueueLockError as e:
|
except QueueLockError as e:
|
||||||
logg.debug('queue item {} is blocked, will retry: {}'.format(tx.hash, e))
|
logg.debug('queue item {} is blocked, will retry: {}'.format(tx.hash, e))
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
delay *= 2
|
delay *= 2
|
||||||
|
race_attempts = 0
|
||||||
err = None
|
err = None
|
||||||
except FileNotFoundError as e:
|
except FileNotFoundError as e:
|
||||||
err = e
|
err = e
|
||||||
logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
|
logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
|
||||||
race_attempts += 1
|
race_attempts += 1
|
||||||
|
time.sleep(self.race_delay)
|
||||||
continue
|
continue
|
||||||
except NotLocalTxError as e:
|
except NotLocalTxError as e:
|
||||||
err = e
|
err = e
|
||||||
logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
|
logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
|
||||||
race_attempts += 1
|
race_attempts += 1
|
||||||
|
time.sleep(self.race_delay)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if err != None:
|
if err != None:
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
chainlib~=0.1.1
|
chainlib~=0.1.1
|
||||||
chainqueue~=0.1.6
|
chainqueue~=0.1.8
|
||||||
chainsyncer~=0.4.3
|
chainsyncer~=0.4.3
|
||||||
confini~=0.6.0
|
confini~=0.6.0
|
||||||
funga~=0.5.2
|
funga~=0.5.2
|
||||||
|
Loading…
Reference in New Issue
Block a user