Compare commits
No commits in common. "master" and "dev-0.2.3" have entirely different histories.
16
CHANGELOG
16
CHANGELOG
|
|
@ -1,19 +1,3 @@
|
|||
- 0.2.10
|
||||
* Add count active states method
|
||||
* Enable complete replace of NEW state on state instantiation
|
||||
- 0.2.9
|
||||
* Enable access to is_pure method
|
||||
* Numeric option for elements return value
|
||||
- 0.2.8
|
||||
* Remove sync on persist set
|
||||
- 0.2.7
|
||||
* Handle missing files in fs readdir list
|
||||
- 0.2.6
|
||||
* Ensure atomic state lock in fs
|
||||
- 0.2.5
|
||||
* Correct handling of persistent sync when no not-state filter has been set
|
||||
- 0.2.4
|
||||
* Add optional concurrency lock for persistence store, implemented for file store
|
||||
- 0.2.3
|
||||
* Add noop-store, for convenience for code using persist constructor but will only use memory state
|
||||
- 0.2.2
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[metadata]
|
||||
name = shep
|
||||
version = 0.2.10
|
||||
version = 0.2.3
|
||||
description = Multi-state key stores using bit masks
|
||||
author = Louis Holbrook
|
||||
author_email = dev@holbrook.no
|
||||
|
|
@ -22,7 +22,7 @@ licence_files =
|
|||
|
||||
[options]
|
||||
include_package_data = True
|
||||
python_requires = >= 3.7
|
||||
python_requires = >= 3.6
|
||||
packages =
|
||||
shep
|
||||
shep.store
|
||||
|
|
|
|||
|
|
@ -32,9 +32,3 @@ class StateTransitionInvalid(Exception):
|
|||
"""Raised if state transition verification fails
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class StateLockedKey(Exception):
|
||||
"""Attempt to write to a state key that is being written to by another client
|
||||
"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -3,10 +3,7 @@ import datetime
|
|||
|
||||
# local imports
|
||||
from .state import State
|
||||
from .error import (
|
||||
StateItemExists,
|
||||
StateLockedKey,
|
||||
)
|
||||
from .error import StateItemExists
|
||||
|
||||
|
||||
class PersistedState(State):
|
||||
|
|
@ -20,8 +17,8 @@ class PersistedState(State):
|
|||
:type logger: object
|
||||
"""
|
||||
|
||||
def __init__(self, factory, bits, logger=None, verifier=None, check_alias=True, event_callback=None, default_state=None):
|
||||
super(PersistedState, self).__init__(bits, logger=logger, verifier=verifier, check_alias=check_alias, event_callback=event_callback, default_state=default_state)
|
||||
def __init__(self, factory, bits, logger=None, verifier=None, check_alias=True, event_callback=None):
|
||||
super(PersistedState, self).__init__(bits, logger=logger, verifier=verifier, check_alias=check_alias, event_callback=event_callback)
|
||||
self.__store_factory = factory
|
||||
self.__stores = {}
|
||||
|
||||
|
|
@ -37,14 +34,13 @@ class PersistedState(State):
|
|||
|
||||
See shep.state.State.put
|
||||
"""
|
||||
k = self.to_name(state)
|
||||
to_state = super(PersistedState, self).put(key, state=state, contents=contents)
|
||||
|
||||
k = self.name(to_state)
|
||||
|
||||
self.__ensure_store(k)
|
||||
|
||||
self.__stores[k].put(key, contents)
|
||||
|
||||
super(PersistedState, self).put(key, state=state, contents=contents)
|
||||
|
||||
self.register_modify(key)
|
||||
|
||||
|
||||
|
|
@ -60,16 +56,11 @@ class PersistedState(State):
|
|||
k_to = self.name(to_state)
|
||||
self.__ensure_store(k_to)
|
||||
|
||||
contents = None
|
||||
try:
|
||||
contents = self.__stores[k_from].get(key)
|
||||
self.__stores[k_to].put(key, contents)
|
||||
self.__stores[k_from].remove(key)
|
||||
except StateLockedKey as e:
|
||||
super(PersistedState, self).unset(key, or_state, allow_base=True)
|
||||
raise e
|
||||
|
||||
#self.sync(to_state)
|
||||
contents = self.__stores[k_from].get(key)
|
||||
self.__stores[k_to].put(key, contents)
|
||||
self.__stores[k_from].remove(key)
|
||||
|
||||
self.sync(to_state)
|
||||
|
||||
return to_state
|
||||
|
||||
|
|
@ -144,7 +135,7 @@ class PersistedState(State):
|
|||
return to_state
|
||||
|
||||
|
||||
def sync(self, state=None, not_state=None):
|
||||
def sync(self, state=None):
|
||||
"""Reload resources for a single state in memory from the persisted state store.
|
||||
|
||||
:param state: State to load
|
||||
|
|
@ -152,20 +143,11 @@ class PersistedState(State):
|
|||
:raises StateItemExists: A content key is already recorded with a different state in memory than in persisted store.
|
||||
# :todo: if sync state is none, sync all
|
||||
"""
|
||||
|
||||
states_numeric = []
|
||||
if state == None:
|
||||
states_numeric = list(self.all(numeric=True))
|
||||
else:
|
||||
states_numeric = [state]
|
||||
|
||||
states = []
|
||||
for state in states_numeric:
|
||||
if not_state != None:
|
||||
if state & not_state == 0:
|
||||
states.append(self.name(state))
|
||||
else:
|
||||
states.append(self.name(state))
|
||||
if state == None:
|
||||
states = list(self.all())
|
||||
else:
|
||||
states = [self.name(state)]
|
||||
|
||||
ks = []
|
||||
for k in states:
|
||||
|
|
@ -226,11 +208,10 @@ class PersistedState(State):
|
|||
|
||||
See shep.state.State.replace
|
||||
"""
|
||||
super(PersistedState, self).replace(key, contents)
|
||||
state = self.state(key)
|
||||
k = self.name(state)
|
||||
r = self.__stores[k].replace(key, contents)
|
||||
super(PersistedState, self).replace(key, contents)
|
||||
return r
|
||||
return self.__stores[k].replace(key, contents)
|
||||
|
||||
|
||||
def modified(self, key):
|
||||
|
|
|
|||
|
|
@ -30,22 +30,16 @@ class State:
|
|||
|
||||
base_state_name = 'NEW'
|
||||
|
||||
def __init__(self, bits, logger=None, verifier=None, check_alias=True, event_callback=None, default_state=None):
|
||||
def __init__(self, bits, logger=None, verifier=None, check_alias=True, event_callback=None):
|
||||
self.__initial_bits = bits
|
||||
self.__bits = bits
|
||||
self.__limit = (1 << bits) - 1
|
||||
self.__c = 0
|
||||
setattr(self, self.base_state_name, 0)
|
||||
|
||||
if default_state == None:
|
||||
default_state = self.base_state_name
|
||||
|
||||
setattr(self, default_state, 0)
|
||||
|
||||
self.__reverse = {0: getattr(self, default_state)}
|
||||
self.__keys = {getattr(self, default_state): []}
|
||||
self.__reverse = {0: getattr(self, self.base_state_name)}
|
||||
self.__keys = {getattr(self, self.base_state_name): []}
|
||||
self.__keys_reverse = {}
|
||||
if default_state != self.base_state_name:
|
||||
self.__keys_reverse[default_state] = 0
|
||||
self.__contents = {}
|
||||
self.modified_last = {}
|
||||
self.verifier = verifier
|
||||
|
|
@ -59,7 +53,7 @@ class State:
|
|||
|
||||
|
||||
# return true if v is a single-bit state
|
||||
def is_pure(self, v):
|
||||
def __is_pure(self, v):
|
||||
if v == 0:
|
||||
return True
|
||||
c = 1
|
||||
|
|
@ -145,7 +139,7 @@ class State:
|
|||
def __add_state_list(self, state, item):
|
||||
if self.__keys.get(state) == None:
|
||||
self.__keys[state] = []
|
||||
if not self.is_pure(state) or state == 0:
|
||||
if not self.__is_pure(state) or state == 0:
|
||||
self.__keys[state].append(item)
|
||||
c = 1
|
||||
for i in range(self.__bits):
|
||||
|
|
@ -191,18 +185,12 @@ class State:
|
|||
self.__set(k, v)
|
||||
|
||||
|
||||
def to_name(self, k):
|
||||
if k == None:
|
||||
k = 0
|
||||
return self.name(k)
|
||||
|
||||
|
||||
def __alias(self, k, *args):
|
||||
v = 0
|
||||
for a in args:
|
||||
a = self.__check_value_cursor(a)
|
||||
v = self.__check_limit(v | a, pure=False)
|
||||
if self.is_pure(v):
|
||||
if self.__is_pure(v):
|
||||
raise ValueError('use add to add pure values')
|
||||
return self.__set(k, v)
|
||||
|
||||
|
|
@ -223,49 +211,36 @@ class State:
|
|||
return self.__alias(k, *args)
|
||||
|
||||
|
||||
def all(self, pure=False, numeric=False):
|
||||
"""Return list of all unique atomic and alias state strings.
|
||||
def all(self, pure=False):
|
||||
"""Return list of all unique atomic and alias states.
|
||||
|
||||
:rtype: list of ints
|
||||
:return: states
|
||||
"""
|
||||
l = []
|
||||
for k in dir(self):
|
||||
state = None
|
||||
if k[0] == '_':
|
||||
continue
|
||||
if k.upper() != k:
|
||||
continue
|
||||
if pure:
|
||||
state = self.from_name(k)
|
||||
if not self.is_pure(state):
|
||||
if not self.__is_pure(state):
|
||||
continue
|
||||
if numeric:
|
||||
if state == None:
|
||||
state = self.from_name(k)
|
||||
l.append(state)
|
||||
else:
|
||||
l.append(k)
|
||||
l.append(k)
|
||||
l.sort()
|
||||
return l
|
||||
|
||||
|
||||
def elements(self, v, numeric=False, as_string=True):
|
||||
def elements(self, v):
|
||||
r = []
|
||||
if v == None or v == 0:
|
||||
return self.base_state_name
|
||||
c = 1
|
||||
for i in range(self.__bits):
|
||||
if v & c > 0:
|
||||
if numeric:
|
||||
r.append(c)
|
||||
else:
|
||||
r.append(self.name(c))
|
||||
r.append(self.name(c))
|
||||
c <<= 1
|
||||
|
||||
if numeric or not as_string:
|
||||
return r
|
||||
|
||||
return '_' + '.'.join(r)
|
||||
|
||||
|
||||
|
|
@ -446,7 +421,7 @@ class State:
|
|||
:rtype: int
|
||||
:returns: Resulting state
|
||||
"""
|
||||
if not self.is_pure(or_state):
|
||||
if not self.__is_pure(or_state):
|
||||
raise ValueError('can only apply using single bit states')
|
||||
|
||||
current_state = self.__keys_reverse.get(key)
|
||||
|
|
@ -461,7 +436,7 @@ class State:
|
|||
return self.__move(key, current_state, to_state)
|
||||
|
||||
|
||||
def unset(self, key, not_state, allow_base=False):
|
||||
def unset(self, key, not_state):
|
||||
"""Unset a single bit, moving to a pure or alias state.
|
||||
|
||||
The resulting state cannot be State.base_state_name (0).
|
||||
|
|
@ -476,7 +451,7 @@ class State:
|
|||
:rtype: int
|
||||
:returns: Resulting state
|
||||
"""
|
||||
if not self.is_pure(not_state):
|
||||
if not self.__is_pure(not_state):
|
||||
raise ValueError('can only apply using single bit states')
|
||||
|
||||
current_state = self.__keys_reverse.get(key)
|
||||
|
|
@ -487,7 +462,7 @@ class State:
|
|||
if to_state == current_state:
|
||||
raise ValueError('invalid change for state {}: {}'.format(key, not_state))
|
||||
|
||||
if to_state == getattr(self, self.base_state_name) and not allow_base:
|
||||
if to_state == getattr(self, self.base_state_name):
|
||||
raise ValueError('State {} for {} cannot be reverted to {}'.format(current_state, key, self.base_state_name))
|
||||
|
||||
new_state = self.__reverse.get(to_state)
|
||||
|
|
@ -592,7 +567,7 @@ class State:
|
|||
state = self.__keys_reverse.get(key)
|
||||
if state == None:
|
||||
raise StateItemNotFound(key)
|
||||
if not self.is_pure(state):
|
||||
if not self.__is_pure(state):
|
||||
raise StateInvalid('cannot run next on an alias state')
|
||||
|
||||
if state == 0:
|
||||
|
|
@ -647,27 +622,3 @@ class State:
|
|||
statemask = ~statemask
|
||||
statemask &= self.__limit
|
||||
return statemask
|
||||
|
||||
|
||||
def purge(self, key):
|
||||
state = self.state(key)
|
||||
state_name = self.name(state)
|
||||
|
||||
v = self.__keys.get(state)
|
||||
v.remove(key)
|
||||
|
||||
del self.__keys_reverse[key]
|
||||
|
||||
try:
|
||||
del self.__contents[key]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
try:
|
||||
del self.modified_last[key]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
def count(self):
|
||||
return self.__c
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from .base import (
|
|||
re_processedname,
|
||||
StoreFactory,
|
||||
)
|
||||
from shep.error import StateLockedKey
|
||||
|
||||
|
||||
class SimpleFileStore:
|
||||
|
|
@ -16,42 +15,15 @@ class SimpleFileStore:
|
|||
:param path: Filesystem base path for all state directory
|
||||
:type path: str
|
||||
"""
|
||||
def __init__(self, path, binary=False, lock_path=None):
|
||||
def __init__(self, path, binary=False):
|
||||
self.__path = path
|
||||
os.makedirs(self.__path, exist_ok=True)
|
||||
if binary:
|
||||
self.__m = ['rb', 'wb']
|
||||
else:
|
||||
self.__m = ['r', 'w']
|
||||
self.__lock_path = lock_path
|
||||
if self.__lock_path != None:
|
||||
os.makedirs(lock_path, exist_ok=True)
|
||||
|
||||
|
||||
def __lock(self, k):
|
||||
if self.__lock_path == None:
|
||||
return
|
||||
fp = os.path.join(self.__lock_path, k)
|
||||
f = None
|
||||
try:
|
||||
f = open(fp, 'x')
|
||||
except FileExistsError:
|
||||
pass
|
||||
if f == None:
|
||||
raise StateLockedKey(k)
|
||||
f.close()
|
||||
|
||||
|
||||
def __unlock(self, k):
|
||||
if self.__lock_path == None:
|
||||
return
|
||||
fp = os.path.join(self.__lock_path, k)
|
||||
try:
|
||||
os.unlink(fp)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def put(self, k, contents=None):
|
||||
"""Add a new key and optional contents
|
||||
|
||||
|
|
@ -60,7 +32,6 @@ class SimpleFileStore:
|
|||
:param contents: Optional contents to assign for content key
|
||||
:type contents: any
|
||||
"""
|
||||
self.__lock(k)
|
||||
fp = os.path.join(self.__path, k)
|
||||
if contents == None:
|
||||
if self.__m[1] == 'wb':
|
||||
|
|
@ -71,7 +42,6 @@ class SimpleFileStore:
|
|||
f = open(fp, self.__m[1])
|
||||
f.write(contents)
|
||||
f.close()
|
||||
self.__unlock(k)
|
||||
|
||||
|
||||
def remove(self, k):
|
||||
|
|
@ -81,10 +51,8 @@ class SimpleFileStore:
|
|||
:type k: str
|
||||
:raises FileNotFoundError: Content key does not exist in the state
|
||||
"""
|
||||
self.__lock(k)
|
||||
fp = os.path.join(self.__path, k)
|
||||
os.unlink(fp)
|
||||
self.__unlock(k)
|
||||
|
||||
|
||||
def get(self, k):
|
||||
|
|
@ -96,12 +64,10 @@ class SimpleFileStore:
|
|||
:rtype: any
|
||||
:return: Contents
|
||||
"""
|
||||
self.__lock(k)
|
||||
fp = os.path.join(self.__path, k)
|
||||
f = open(fp, self.__m[0])
|
||||
r = f.read()
|
||||
f.close()
|
||||
self.__unlock(k)
|
||||
return r
|
||||
|
||||
|
||||
|
|
@ -111,21 +77,15 @@ class SimpleFileStore:
|
|||
:rtype: list of str
|
||||
:return: Content keys in state
|
||||
"""
|
||||
self.__lock('.list')
|
||||
files = []
|
||||
for p in os.listdir(self.__path):
|
||||
fp = os.path.join(self.__path, p)
|
||||
f = None
|
||||
try:
|
||||
f = open(fp, self.__m[0])
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
f = open(fp, self.__m[0])
|
||||
r = f.read()
|
||||
f.close()
|
||||
if len(r) == 0:
|
||||
r = None
|
||||
files.append((p, r,))
|
||||
self.__unlock('.list')
|
||||
return files
|
||||
|
||||
|
||||
|
|
@ -150,20 +110,16 @@ class SimpleFileStore:
|
|||
:param contents: Contents
|
||||
:type contents: any
|
||||
"""
|
||||
self.__lock(k)
|
||||
fp = os.path.join(self.__path, k)
|
||||
os.stat(fp)
|
||||
f = open(fp, self.__m[1])
|
||||
r = f.write(contents)
|
||||
f.close()
|
||||
self.__unlock(k)
|
||||
|
||||
|
||||
def modified(self, k):
|
||||
self.__lock(k)
|
||||
path = self.path(k)
|
||||
st = os.stat(path)
|
||||
self.__unlock(k)
|
||||
return st.st_ctime
|
||||
|
||||
|
||||
|
|
@ -177,10 +133,9 @@ class SimpleFileStoreFactory(StoreFactory):
|
|||
:param path: Filesystem path as base path for states
|
||||
:type path: str
|
||||
"""
|
||||
def __init__(self, path, binary=False, use_lock=False):
|
||||
def __init__(self, path, binary=False):
|
||||
self.__path = path
|
||||
self.__binary = binary
|
||||
self.__use_lock = use_lock
|
||||
|
||||
|
||||
def add(self, k):
|
||||
|
|
@ -191,13 +146,9 @@ class SimpleFileStoreFactory(StoreFactory):
|
|||
:rtype: SimpleFileStore
|
||||
:return: A filesystem persistence instance with the given identifier as subdirectory
|
||||
"""
|
||||
lock_path = None
|
||||
if self.__use_lock:
|
||||
lock_path = os.path.join(self.__path, '.lock')
|
||||
|
||||
k = str(k)
|
||||
store_path = os.path.join(self.__path, k)
|
||||
return SimpleFileStore(store_path, binary=self.__binary, lock_path=lock_path)
|
||||
return SimpleFileStore(store_path, binary=self.__binary)
|
||||
|
||||
|
||||
def ls(self):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from shep.error import (
|
|||
StateExists,
|
||||
StateInvalid,
|
||||
StateItemExists,
|
||||
StateLockedKey,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -258,52 +257,5 @@ class TestFileStore(unittest.TestCase):
|
|||
self.assertEqual(len(r), 3)
|
||||
|
||||
|
||||
def test_lock(self):
|
||||
factory = SimpleFileStoreFactory(self.d, use_lock=True)
|
||||
states = PersistedState(factory.add, 3)
|
||||
states.add('foo')
|
||||
states.add('bar')
|
||||
states.add('baz')
|
||||
states.alias('xyzzy', states.FOO | states.BAR)
|
||||
states.alias('plugh', states.FOO | states.BAR | states.BAZ)
|
||||
states.put('abcd')
|
||||
|
||||
lock_path = os.path.join(self.d, '.lock')
|
||||
os.stat(lock_path)
|
||||
|
||||
fp = os.path.join(self.d, '.lock', 'xxxx')
|
||||
f = open(fp, 'w')
|
||||
f.close()
|
||||
|
||||
with self.assertRaises(StateLockedKey):
|
||||
states.put('xxxx')
|
||||
|
||||
os.unlink(fp)
|
||||
states.put('xxxx')
|
||||
|
||||
states.set('xxxx', states.FOO)
|
||||
states.set('xxxx', states.BAR)
|
||||
states.replace('xxxx', contents='zzzz')
|
||||
|
||||
fp = os.path.join(self.d, '.lock', 'xxxx')
|
||||
f = open(fp, 'w')
|
||||
f.close()
|
||||
|
||||
with self.assertRaises(StateLockedKey):
|
||||
states.set('xxxx', states.BAZ)
|
||||
|
||||
v = states.state('xxxx')
|
||||
self.assertEqual(v, states.XYZZY)
|
||||
|
||||
with self.assertRaises(StateLockedKey):
|
||||
states.unset('xxxx', states.FOO)
|
||||
|
||||
with self.assertRaises(StateLockedKey):
|
||||
states.replace('xxxx', contents='yyyy')
|
||||
|
||||
v = states.get('xxxx')
|
||||
self.assertEqual(v, 'zzzz')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from shep import State
|
|||
from shep.error import (
|
||||
StateExists,
|
||||
StateInvalid,
|
||||
StateItemNotFound,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
|
@ -251,76 +250,5 @@ class TestState(unittest.TestCase):
|
|||
self.assertEqual(mask, states.ALL)
|
||||
|
||||
|
||||
def test_remove(self):
|
||||
states = State(1)
|
||||
states.add('foo')
|
||||
|
||||
states.put('xyzzy', contents='plugh')
|
||||
v = states.get('xyzzy')
|
||||
self.assertEqual(v, 'plugh')
|
||||
|
||||
states.next('xyzzy')
|
||||
|
||||
v = states.state('xyzzy')
|
||||
self.assertEqual(states.FOO, v)
|
||||
|
||||
states.purge('xyzzy')
|
||||
|
||||
with self.assertRaises(StateItemNotFound):
|
||||
states.state('xyzzy')
|
||||
|
||||
|
||||
def test_elements(self):
|
||||
states = State(2)
|
||||
states.add('foo')
|
||||
states.add('bar')
|
||||
states.alias('baz', states.FOO, states.BAR)
|
||||
|
||||
v = states.elements(states.BAZ)
|
||||
self.assertIn('FOO', v)
|
||||
self.assertIn('BAR', v)
|
||||
self.assertIsInstance(v, str)
|
||||
|
||||
v = states.elements(states.BAZ, numeric=True)
|
||||
self.assertIn(states.FOO, v)
|
||||
self.assertIn(states.BAR, v)
|
||||
|
||||
v = states.elements(states.BAZ, as_string=False)
|
||||
self.assertIn('FOO', v)
|
||||
self.assertIn('BAR', v)
|
||||
self.assertNotIsInstance(v, str)
|
||||
self.assertIsInstance(v, list)
|
||||
|
||||
|
||||
def test_count(self):
|
||||
states = State(3)
|
||||
states.add('foo')
|
||||
states.add('bar')
|
||||
self.assertEqual(states.count(), 2)
|
||||
states.add('baz')
|
||||
self.assertEqual(states.count(), 3)
|
||||
|
||||
|
||||
def test_pure(self):
|
||||
states = State(2)
|
||||
states.add('foo')
|
||||
states.add('bar')
|
||||
states.alias('baz', states.FOO, states.BAR)
|
||||
|
||||
v = states.is_pure(states.BAZ)
|
||||
self.assertFalse(v)
|
||||
|
||||
v = states.is_pure(states.FOO)
|
||||
self.assertTrue(v)
|
||||
|
||||
|
||||
def test_default(self):
|
||||
states = State(2, default_state='FOO')
|
||||
with self.assertRaises(StateItemNotFound):
|
||||
states.state('NEW')
|
||||
getattr(states, 'FOO')
|
||||
states.state('FOO')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue