Add fs queue backend add file
This commit is contained in:
nolash
parent
f8c1deb752
commit
565a58252a
GPG Key ID:
21D2E7BB88C2A746
|
|
@ -1,153 +1 @@
|
|||
# standard imports
|
||||
import enum
|
||||
|
||||
|
||||
@enum.unique
|
||||
class StatusBits(enum.IntEnum):
|
||||
"""Individual bit flags that are combined to define the state and legacy of a queued transaction
|
||||
|
||||
"""
|
||||
QUEUED = 0x01 # transaction should be sent to network
|
||||
RESERVED = 0x02 # transaction is currently being handled by a thread
|
||||
IN_NETWORK = 0x08 # transaction is in network
|
||||
|
||||
DEFERRED = 0x10 # an attempt to send the transaction to network has failed
|
||||
GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
|
||||
|
||||
LOCAL_ERROR = 0x100 # errors that originate internally from the component
|
||||
NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
|
||||
NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
|
||||
UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
|
||||
|
||||
FINAL = 0x1000 # transaction processing has completed
|
||||
OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
|
||||
MANUAL = 0x8000 # transaction processing has been manually overridden
|
||||
|
||||
|
||||
@enum.unique
|
||||
class StatusEnum(enum.IntEnum):
|
||||
"""
|
||||
|
||||
- Inactive, not finalized. (<0)
|
||||
* PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
|
||||
* SENDFAIL: The transaction was not received by the node.
|
||||
* RETRY: The transaction is queued for a new send attempt after previously failing.
|
||||
* READYSEND: The transaction is queued for its first send attempt
|
||||
* OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
|
||||
* WAITFORGAS: The transaction is on hold pending gas funding.
|
||||
- Active state: (==0)
|
||||
* SENT: The transaction has been sent to the mempool.
|
||||
- Inactive, finalized. (>0)
|
||||
* FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
|
||||
* CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
|
||||
* OVERRIDDEN: Transaction has been manually overriden.
|
||||
* REJECTED: The transaction was rejected by the node.
|
||||
* REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
|
||||
* SUCCESS: THe transaction was successfully mined. (Block number will be set)
|
||||
|
||||
"""
|
||||
PENDING = 0
|
||||
|
||||
SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
|
||||
RETRY = StatusBits.QUEUED | StatusBits.DEFERRED
|
||||
READYSEND = StatusBits.QUEUED
|
||||
RESERVED = StatusBits.RESERVED
|
||||
|
||||
OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
|
||||
|
||||
WAITFORGAS = StatusBits.GAS_ISSUES
|
||||
|
||||
SENT = StatusBits.IN_NETWORK
|
||||
FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
|
||||
CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
|
||||
OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
|
||||
|
||||
REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
|
||||
REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
|
||||
SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL
|
||||
|
||||
|
||||
def status_str(v, bits_only=False):
|
||||
"""Render a human-readable string describing the status
|
||||
|
||||
If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
|
||||
|
||||
If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
|
||||
|
||||
:param v: Status bit field
|
||||
:type v: number
|
||||
:param bits_only: Only render individual bit labels.
|
||||
:type bits_only: bool
|
||||
:returns: Status string
|
||||
:rtype: str
|
||||
"""
|
||||
s = ''
|
||||
if not bits_only:
|
||||
try:
|
||||
s = StatusEnum(v).name
|
||||
return s
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if v == 0:
|
||||
return 'NONE'
|
||||
|
||||
for i in range(16):
|
||||
b = (1 << i)
|
||||
if (b & 0xffff) & v:
|
||||
n = StatusBits(b).name
|
||||
if len(s) > 0:
|
||||
s += ','
|
||||
s += n
|
||||
if not bits_only:
|
||||
s += '*'
|
||||
return s
|
||||
|
||||
|
||||
|
||||
def all_errors():
|
||||
"""Bit mask of all error states
|
||||
|
||||
:returns: Error flags
|
||||
:rtype: number
|
||||
"""
|
||||
return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
|
||||
|
||||
|
||||
def is_error_status(v):
|
||||
"""Check if value is an error state
|
||||
|
||||
:param v: Status bit field
|
||||
:type v: number
|
||||
:returns: True if error
|
||||
:rtype: bool
|
||||
"""
|
||||
return bool(v & all_errors())
|
||||
|
||||
|
||||
__ignore_manual_value = ~StatusBits.MANUAL
|
||||
def ignore_manual(v):
|
||||
return v & __ignore_manual_value
|
||||
|
||||
|
||||
def is_nascent(v):
|
||||
return ignore_manual(v) == StatusEnum.PENDING
|
||||
|
||||
|
||||
def dead():
|
||||
"""Bit mask defining whether a transaction is still likely to be processed on the network.
|
||||
|
||||
:returns: Bit mask
|
||||
:rtype: number
|
||||
"""
|
||||
return StatusBits.FINAL | StatusBits.OBSOLETE
|
||||
|
||||
|
||||
def is_alive(v):
|
||||
"""Check if transaction is still likely to be processed on the network.
|
||||
|
||||
The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
|
||||
|
||||
:returns:
|
||||
"""
|
||||
return bool(v & dead() == 0)
|
||||
from chainqueue.enum import *
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
# standard imports
|
||||
import enum
|
||||
|
||||
|
||||
@enum.unique
|
||||
class StatusBits(enum.IntEnum):
|
||||
"""Individual bit flags that are combined to define the state and legacy of a queued transaction
|
||||
|
||||
"""
|
||||
QUEUED = 0x01 # transaction should be sent to network
|
||||
RESERVED = 0x02 # transaction is currently being handled by a thread
|
||||
IN_NETWORK = 0x08 # transaction is in network
|
||||
|
||||
DEFERRED = 0x10 # an attempt to send the transaction to network has failed
|
||||
GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
|
||||
|
||||
LOCAL_ERROR = 0x100 # errors that originate internally from the component
|
||||
NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
|
||||
NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
|
||||
UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
|
||||
|
||||
FINAL = 0x1000 # transaction processing has completed
|
||||
OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
|
||||
MANUAL = 0x8000 # transaction processing has been manually overridden
|
||||
|
||||
|
||||
@enum.unique
|
||||
class StatusEnum(enum.IntEnum):
|
||||
"""
|
||||
|
||||
- Inactive, not finalized. (<0)
|
||||
* PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
|
||||
* SENDFAIL: The transaction was not received by the node.
|
||||
* RETRY: The transaction is queued for a new send attempt after previously failing.
|
||||
* READYSEND: The transaction is queued for its first send attempt
|
||||
* OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
|
||||
* WAITFORGAS: The transaction is on hold pending gas funding.
|
||||
- Active state: (==0)
|
||||
* SENT: The transaction has been sent to the mempool.
|
||||
- Inactive, finalized. (>0)
|
||||
* FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
|
||||
* CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
|
||||
* OVERRIDDEN: Transaction has been manually overriden.
|
||||
* REJECTED: The transaction was rejected by the node.
|
||||
* REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
|
||||
* SUCCESS: THe transaction was successfully mined. (Block number will be set)
|
||||
|
||||
"""
|
||||
PENDING = 0
|
||||
|
||||
SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
|
||||
RETRY = StatusBits.QUEUED | StatusBits.DEFERRED
|
||||
READYSEND = StatusBits.QUEUED
|
||||
RESERVED = StatusBits.RESERVED
|
||||
|
||||
OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
|
||||
|
||||
WAITFORGAS = StatusBits.GAS_ISSUES
|
||||
|
||||
SENT = StatusBits.IN_NETWORK
|
||||
FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
|
||||
CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
|
||||
OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
|
||||
|
||||
REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
|
||||
REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
|
||||
SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL
|
||||
|
||||
|
||||
def status_str(v, bits_only=False):
|
||||
"""Render a human-readable string describing the status
|
||||
|
||||
If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
|
||||
|
||||
If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
|
||||
|
||||
:param v: Status bit field
|
||||
:type v: number
|
||||
:param bits_only: Only render individual bit labels.
|
||||
:type bits_only: bool
|
||||
:returns: Status string
|
||||
:rtype: str
|
||||
"""
|
||||
s = ''
|
||||
if not bits_only:
|
||||
try:
|
||||
s = StatusEnum(v).name
|
||||
return s
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if v == 0:
|
||||
return 'NONE'
|
||||
|
||||
for i in range(16):
|
||||
b = (1 << i)
|
||||
if (b & 0xffff) & v:
|
||||
n = StatusBits(b).name
|
||||
if len(s) > 0:
|
||||
s += ','
|
||||
s += n
|
||||
if not bits_only:
|
||||
s += '*'
|
||||
return s
|
||||
|
||||
|
||||
def status_bytes(status=0):
|
||||
return status.to_bytes(8, byteorder='big')
|
||||
|
||||
|
||||
def all_errors():
|
||||
"""Bit mask of all error states
|
||||
|
||||
:returns: Error flags
|
||||
:rtype: number
|
||||
"""
|
||||
return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
|
||||
|
||||
|
||||
def is_error_status(v):
|
||||
"""Check if value is an error state
|
||||
|
||||
:param v: Status bit field
|
||||
:type v: number
|
||||
:returns: True if error
|
||||
:rtype: bool
|
||||
"""
|
||||
return bool(v & all_errors())
|
||||
|
||||
|
||||
__ignore_manual_value = ~StatusBits.MANUAL
|
||||
def ignore_manual(v):
|
||||
return v & __ignore_manual_value
|
||||
|
||||
|
||||
def is_nascent(v):
|
||||
return ignore_manual(v) == StatusEnum.PENDING
|
||||
|
||||
|
||||
def dead():
|
||||
"""Bit mask defining whether a transaction is still likely to be processed on the network.
|
||||
|
||||
:returns: Bit mask
|
||||
:rtype: number
|
||||
"""
|
||||
return StatusBits.FINAL | StatusBits.OBSOLETE
|
||||
|
||||
|
||||
def is_alive(v):
|
||||
"""Check if transaction is still likely to be processed on the network.
|
||||
|
||||
The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
|
||||
|
||||
:returns:
|
||||
"""
|
||||
return bool(v & dead() == 0)
|
||||
|
|
@ -21,3 +21,10 @@ class CacheIntegrityError(ChainQueueException):
|
|||
"""
|
||||
pass
|
||||
|
||||
|
||||
class BackendIntegrityError(ChainQueueException):
|
||||
"""Raised when queue backend has invalid state
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,89 @@
|
|||
# standard imports
|
||||
import stat
|
||||
import logging
|
||||
import os
|
||||
|
||||
# local imports
|
||||
from chainqueue.enum import (
|
||||
StatusBits,
|
||||
status_bytes,
|
||||
)
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FsQueueBackend:
|
||||
|
||||
|
||||
def add(self, label, content, prefix):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def get_index(self, idx):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def set_prefix(self, idx, prefix):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class FsQueue:
|
||||
|
||||
def __init__(self, root_path, backend=FsQueueBackend()):
|
||||
self.backend = backend
|
||||
self.path = root_path
|
||||
self.path_state = {}
|
||||
|
||||
try:
|
||||
fi = os.stat(self.path)
|
||||
self.__verify_directory()
|
||||
except FileNotFoundError:
|
||||
FsQueue.__prepare_directory(self.path)
|
||||
|
||||
for r in FsQueue.__state_dirs(self.path):
|
||||
self.path_state[r[0]] = r[1]
|
||||
|
||||
self.index_path = os.path.join(self.path, '.active')
|
||||
os.makedirs(self.index_path, exist_ok=True)
|
||||
|
||||
|
||||
def add(self, key, value):
|
||||
prefix = status_bytes()
|
||||
c = self.backend.add(key, value, prefix=prefix)
|
||||
|
||||
key_hex = key.hex()
|
||||
entry_path = os.path.join(self.index_path, key_hex)
|
||||
f = open(entry_path, 'xb')
|
||||
f.write(c.to_bytes(8, byteorder='big'))
|
||||
f.close()
|
||||
|
||||
ptr_path = os.path.join(self.path_state['new'], key_hex)
|
||||
os.link(entry_path, ptr_path)
|
||||
|
||||
logg.debug('added new queue entry {} -> {} index {}'.format(ptr_path, entry_path, c))
|
||||
|
||||
|
||||
@staticmethod
|
||||
def __state_dirs(path):
|
||||
r = []
|
||||
for s in [
|
||||
'new',
|
||||
'reserved',
|
||||
'ready',
|
||||
'error',
|
||||
'defer',
|
||||
]:
|
||||
r.append((s, os.path.join(path, 'spool', s)))
|
||||
return r
|
||||
|
||||
|
||||
def __verify_directory(self):
|
||||
return True
|
||||
|
||||
|
||||
@staticmethod
|
||||
def __prepare_directory(path):
|
||||
os.makedirs(path, exist_ok=True)
|
||||
os.makedirs(os.path.join(path, '.cache'))
|
||||
for r in FsQueue.__state_dirs(path):
|
||||
os.makedirs(r[1])
|
||||
|
|
@ -15,6 +15,7 @@ class HexDir:
|
|||
self.path = root_path
|
||||
self.key_length = key_length
|
||||
self.prefix_length = prefix_length
|
||||
self.entry_length = key_length + prefix_length
|
||||
self.__levels = levels + 2
|
||||
fi = None
|
||||
try:
|
||||
|
|
@ -44,6 +45,8 @@ class HexDir:
|
|||
key_hex = key.hex()
|
||||
entry_path = self.to_filepath(key_hex)
|
||||
|
||||
c = self.count()
|
||||
|
||||
os.makedirs(os.path.dirname(entry_path), exist_ok=True)
|
||||
f = open(entry_path, 'wb')
|
||||
f.write(content)
|
||||
|
|
@ -55,7 +58,18 @@ class HexDir:
|
|||
f.write(key)
|
||||
f.close()
|
||||
|
||||
logg.info('created new entry {} in {}'.format(key_hex, entry_path))
|
||||
logg.info('created new entry {} idx {} in {}'.format(key_hex, c, entry_path))
|
||||
|
||||
return c
|
||||
|
||||
|
||||
def count(self):
|
||||
fi = os.stat(self.master_file)
|
||||
c = fi.st_size / self.entry_length
|
||||
r = int(c)
|
||||
if r != c: # TODO: verify valid for check if evenly divided
|
||||
raise IndexError('master file not aligned')
|
||||
return r
|
||||
|
||||
|
||||
def set_prefix(self, idx, prefix):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
# standard imports
|
||||
import unittest
|
||||
import tempfile
|
||||
import shutil
|
||||
import logging
|
||||
import os
|
||||
|
||||
# local imports
|
||||
from chainqueue.fs.cache import FsQueue
|
||||
from chainqueue.fs.dir import HexDir
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
class HexDirTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp()
|
||||
self.hexdir = HexDir(os.path.join(self.dir, 'q'), 32, 2, 8)
|
||||
self.q = FsQueue(os.path.join(self.dir, 'spool'), backend=self.hexdir)
|
||||
logg.debug('setup fsqueue root {}'.format(self.dir))
|
||||
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.dir)
|
||||
logg.debug('cleaned fsqueue root {}'.format(self.dir))
|
||||
|
||||
|
||||
def test_new(self):
|
||||
tx_hash = os.urandom(32)
|
||||
tx_content = os.urandom(128)
|
||||
self.q.add(tx_hash, tx_content)
|
||||
|
||||
f = open(os.path.join(self.q.path_state['new'], tx_hash.hex()), 'rb')
|
||||
r = f.read()
|
||||
f.close()
|
||||
self.assertEqual(r, b'\x00' * 8)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -51,6 +51,13 @@ class HexDirTest(unittest.TestCase):
|
|||
self.hexdir.add(label, content, prefix=b'a')
|
||||
|
||||
|
||||
def test_index(self):
|
||||
self.hexdir.add(b'\xde\xad\xbe\xef', b'foo', b'ab')
|
||||
self.hexdir.add(b'\xbe\xef\xfe\xed', b'bar', b'cd')
|
||||
c = self.hexdir.add(b'\x01\x02\x03\x04', b'baz', b'ef')
|
||||
self.assertEqual(c, 2)
|
||||
|
||||
|
||||
def test_edit(self):
|
||||
self.hexdir.add(b'\xde\xad\xbe\xef', b'foo', b'ab')
|
||||
self.hexdir.add(b'\xbe\xef\xfe\xed', b'bar', b'cd')
|
||||
|
|
|
|||
Loading…
Reference in New Issue