Fix crashes related to race condition hits

This commit is contained in:
lash 2022-05-01 07:31:18 +00:00
parent 9b98703f24
commit 5102b4ac6e
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
4 changed files with 31 additions and 1 deletions

View File

@ -1,3 +1,7 @@
- 0.2.2
* Fix missing symbol crashes related to race conditions
- 0.2.1
* Receive removed race checks from chainqueue
- 0.2.0 - 0.2.0
* primitive race condition handling between fs access of sync and queue * primitive race condition handling between fs access of sync and queue
* re-enable throttling based on in-flight transaction count * re-enable throttling based on in-flight transaction count

View File

@ -1,9 +1,14 @@
# standard imports
import logging
# external imports # external imports
from chainqueue import Store as QueueStore from chainqueue import Store as QueueStore
# local imports # local imports
from chaind.error import BackendIntegrityError from chaind.error import BackendIntegrityError
logg = logging.getLogger(__name__)
class ChaindAdapter: class ChaindAdapter:

View File

@ -61,19 +61,40 @@ class StateFilter(SyncFilter):
raise NotLocalTxError(tx.hash) raise NotLocalTxError(tx.hash)
delay = 0.01 delay = 0.01
race_attempts = 0
err = None
while True: while True:
if delay > self.delay_limit: if delay > self.delay_limit:
raise QueueLockError('The queue lock for tx {} seems to be stuck. Human meddling needed.'.format(tx.hash)) raise QueueLockError('The queue lock for tx {} seems to be stuck. Human meddling needed.'.format(tx.hash))
elif race_attempts >= 3:
break
try: try:
if tx.status == TxStatus.SUCCESS: if tx.status == TxStatus.SUCCESS:
queue_adapter.succeed(block, tx) queue_adapter.succeed(block, tx)
else: else:
queue_adapter.fail(block, tx) queue_adapter.fail(block, tx)
break break
err = None
except QueueLockError as e: except QueueLockError as e:
logg.debug('queue item {} is blocked, will retry: {}'.format(tx.hash, e)) logg.debug('queue item {} is blocked, will retry: {}'.format(tx.hash, e))
time.sleep(delay) time.sleep(delay)
delay *= 2 delay *= 2
err = None
except FileNotFoundError as e:
err = e
logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
race_attempts += 1
continue
except NotLocalTxError as e:
err = e
logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
race_attempts += 1
continue
if err != None:
raise BackendIntegrityError('cannot find queue item {} in backend: {}'.format(tx.hash, err))
logg.info('filter registered {} for {} in {}'.format(tx.status.name, tx.hash, block))
if self.throttler != None: if self.throttler != None:
self.throttler.dec(tx.hash) self.throttler.dec(tx.hash)

View File

@ -1,6 +1,6 @@
[metadata] [metadata]
name = chaind name = chaind
version = 0.2.1 version = 0.2.2
description = Base package for chain queue service description = Base package for chain queue service
author = Louis Holbrook author = Louis Holbrook
author_email = dev@holbrook.no author_email = dev@holbrook.no