Compare commits
9 Commits
Author | SHA1 | Date |
---|---|---|
lash | 98387157f5 | |
lash | 90f494a1f6 | |
lash | 497b615fe3 | |
lash | 6349ce2519 | |
lash | 40a73be7eb | |
lash | 2f95167895 | |
lash | 1d0e31d10d | |
lash | 52dffb8041 | |
lash | d915f17e2b |
|
@ -1,3 +1,10 @@
|
|||
- 0.3.0
|
||||
* Clean up lingering keys in lists when moving from alias state
|
||||
* Properly set default state when set through instantiation
|
||||
* Pass key to verifier (breaking change)
|
||||
* Ensure all states and aliases persisted even if empty
|
||||
* Allow auto-generated aliases when listing all states
|
||||
* Reformat auto-generated aliases, double underscore instead of dot (breaking change)
|
||||
- 0.2.10
|
||||
* Add count active states method
|
||||
* Enable complete replace of NEW state on state instantiation
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
Rocksdb extension requires python 3.9, as it uses lbry-rocksdb which at the time of writing (13.10.2022) does not have a package available for python 3.10.
|
|
@ -1,6 +1,6 @@
|
|||
[metadata]
|
||||
name = shep
|
||||
version = 0.2.10
|
||||
version = 0.3.0
|
||||
description = Multi-state key stores using bit masks
|
||||
author = Louis Holbrook
|
||||
author_email = dev@holbrook.no
|
||||
|
|
|
@ -2,7 +2,9 @@
|
|||
import datetime
|
||||
|
||||
# local imports
|
||||
from .state import State
|
||||
from .state import (
|
||||
State,
|
||||
)
|
||||
from .error import (
|
||||
StateItemExists,
|
||||
StateLockedKey,
|
||||
|
@ -24,10 +26,12 @@ class PersistedState(State):
|
|||
super(PersistedState, self).__init__(bits, logger=logger, verifier=verifier, check_alias=check_alias, event_callback=event_callback, default_state=default_state)
|
||||
self.__store_factory = factory
|
||||
self.__stores = {}
|
||||
self.__ensure_store(self.base_state_name)
|
||||
|
||||
|
||||
# Create state store container if missing.
|
||||
def __ensure_store(self, k):
|
||||
k = k.upper()
|
||||
if self.__stores.get(k) == None:
|
||||
self.__stores[k] = self.__store_factory(k)
|
||||
|
||||
|
@ -74,7 +78,7 @@ class PersistedState(State):
|
|||
return to_state
|
||||
|
||||
|
||||
def unset(self, key, not_state):
|
||||
def unset(self, key, not_state, allow_base=False):
|
||||
"""Persist a new state for a key or key/content.
|
||||
|
||||
See shep.state.State.unset
|
||||
|
@ -82,7 +86,7 @@ class PersistedState(State):
|
|||
from_state = self.state(key)
|
||||
k_from = self.name(from_state)
|
||||
|
||||
to_state = super(PersistedState, self).unset(key, not_state)
|
||||
to_state = super(PersistedState, self).unset(key, not_state, allow_base=allow_base)
|
||||
|
||||
k_to = self.name(to_state)
|
||||
self.__ensure_store(k_to)
|
||||
|
@ -144,7 +148,7 @@ class PersistedState(State):
|
|||
return to_state
|
||||
|
||||
|
||||
def sync(self, state=None, not_state=None):
|
||||
def sync(self, state=None, not_state=None, ignore_auto=True):
|
||||
"""Reload resources for a single state in memory from the persisted state store.
|
||||
|
||||
:param state: State to load
|
||||
|
@ -155,7 +159,7 @@ class PersistedState(State):
|
|||
|
||||
states_numeric = []
|
||||
if state == None:
|
||||
states_numeric = list(self.all(numeric=True))
|
||||
states_numeric = list(self.all(numeric=True, ignore_auto=ignore_auto))
|
||||
else:
|
||||
states_numeric = [state]
|
||||
|
||||
|
@ -237,3 +241,13 @@ class PersistedState(State):
|
|||
state = self.state(key)
|
||||
k = self.name(state)
|
||||
return self.__stores[k].modified(key)
|
||||
|
||||
|
||||
def add(self, key):
|
||||
self.__ensure_store(key)
|
||||
super(PersistedState, self).add(key)
|
||||
|
||||
|
||||
def alias(self, key, *args):
|
||||
self.__ensure_store(key)
|
||||
super(PersistedState, self).alias(key, *args)
|
||||
|
|
|
@ -15,6 +15,18 @@ from shep.error import (
|
|||
|
||||
re_name = r'^[a-zA-Z_\.]+$'
|
||||
|
||||
def join_elements(states):
|
||||
return '_' + '__'.join(states)
|
||||
|
||||
|
||||
def split_elements(s):
|
||||
if len(s) == 0:
|
||||
return []
|
||||
if s[0] == '_':
|
||||
s = s[1:]
|
||||
return s.split('__')
|
||||
|
||||
|
||||
class State:
|
||||
"""State is an in-memory bitmasked state store for key-value pairs, or even just keys alone.
|
||||
|
||||
|
@ -36,16 +48,20 @@ class State:
|
|||
self.__limit = (1 << bits) - 1
|
||||
self.__c = 0
|
||||
|
||||
self.__keys_reverse = {}
|
||||
|
||||
if default_state == None:
|
||||
default_state = self.base_state_name
|
||||
else:
|
||||
default_state = self.__check_name_valid(default_state)
|
||||
self.base_state_name = default_state
|
||||
self.__keys_reverse[default_state] = 0
|
||||
|
||||
setattr(self, default_state, 0)
|
||||
|
||||
self.__reverse = {0: getattr(self, default_state)}
|
||||
self.__keys = {getattr(self, default_state): []}
|
||||
self.__keys_reverse = {}
|
||||
if default_state != self.base_state_name:
|
||||
self.__keys_reverse[default_state] = 0
|
||||
self.__reverse = {0: default_state}
|
||||
self.__keys = {0: []}
|
||||
|
||||
self.__contents = {}
|
||||
self.modified_last = {}
|
||||
self.verifier = verifier
|
||||
|
@ -204,6 +220,7 @@ class State:
|
|||
v = self.__check_limit(v | a, pure=False)
|
||||
if self.is_pure(v):
|
||||
raise ValueError('use add to add pure values')
|
||||
k = k.replace('.', '__')
|
||||
return self.__set(k, v)
|
||||
|
||||
|
||||
|
@ -223,16 +240,31 @@ class State:
|
|||
return self.__alias(k, *args)
|
||||
|
||||
|
||||
def all(self, pure=False, numeric=False):
|
||||
def __all_bit(self):
|
||||
r = []
|
||||
r.append(self.name(0))
|
||||
v = 1
|
||||
while v < self.__limit:
|
||||
r.append(self.name(v))
|
||||
v <<= 1
|
||||
return r
|
||||
|
||||
|
||||
def all(self, pure=False, numeric=False, ignore_auto=True, bit_order=False):
|
||||
"""Return list of all unique atomic and alias state strings.
|
||||
|
||||
:rtype: list of ints
|
||||
:return: states
|
||||
"""
|
||||
l = []
|
||||
for k in dir(self):
|
||||
v = None
|
||||
if bit_order:
|
||||
v = self.__all_bit()
|
||||
else:
|
||||
v = dir(self)
|
||||
for k in v:
|
||||
state = None
|
||||
if k[0] == '_':
|
||||
if k[0] == '_' and ignore_auto:
|
||||
continue
|
||||
if k.upper() != k:
|
||||
continue
|
||||
|
@ -246,7 +278,8 @@ class State:
|
|||
l.append(state)
|
||||
else:
|
||||
l.append(k)
|
||||
l.sort()
|
||||
if not bit_order:
|
||||
l.sort()
|
||||
return l
|
||||
|
||||
|
||||
|
@ -256,7 +289,7 @@ class State:
|
|||
return self.base_state_name
|
||||
c = 1
|
||||
for i in range(self.__bits):
|
||||
if v & c > 0:
|
||||
if (v & c) > 0:
|
||||
if numeric:
|
||||
r.append(c)
|
||||
else:
|
||||
|
@ -266,14 +299,17 @@ class State:
|
|||
if numeric or not as_string:
|
||||
return r
|
||||
|
||||
return '_' + '.'.join(r)
|
||||
if len(r) == 1:
|
||||
return self.name(v)
|
||||
|
||||
return join_elements(r) #'_' + '.'.join(r)
|
||||
|
||||
|
||||
def from_elements(self, k):
|
||||
r = 0
|
||||
if k[0] != '_':
|
||||
raise ValueError('elements string must start with underscore (_), got {}'.format(k))
|
||||
for v in k[1:].split('.'):
|
||||
for v in k[1:].split('__'):
|
||||
r |= self.from_name(v)
|
||||
return r
|
||||
|
||||
|
@ -309,6 +345,8 @@ class State:
|
|||
:return: Numeric state value
|
||||
"""
|
||||
k = self.__check_name_valid(k)
|
||||
if k == self.base_state_name:
|
||||
return 0
|
||||
return getattr(self, k)
|
||||
|
||||
|
||||
|
@ -416,16 +454,19 @@ class State:
|
|||
raise StateCorruptionError(to_state)
|
||||
|
||||
if self.verifier != None:
|
||||
r = self.verifier(self, from_state, to_state)
|
||||
r = self.verifier(self, key, from_state, to_state)
|
||||
if r != None:
|
||||
raise StateTransitionInvalid(r)
|
||||
|
||||
current_state_list.pop(idx)
|
||||
|
||||
old_state = self.__keys_reverse.get(key)
|
||||
if self.event_callback != None:
|
||||
old_state = self.__keys_reverse.get(key)
|
||||
self.event_callback(key, self.name(old_state), self.name(to_state))
|
||||
|
||||
if old_state == 0:
|
||||
current_state_list.pop(idx)
|
||||
else:
|
||||
for k in self.elements(from_state, numeric=True):
|
||||
self.__keys[k].remove(key)
|
||||
self.__add_state_list(to_state, key)
|
||||
|
||||
self.register_modify(key)
|
||||
|
@ -464,12 +505,14 @@ class State:
|
|||
def unset(self, key, not_state, allow_base=False):
|
||||
"""Unset a single bit, moving to a pure or alias state.
|
||||
|
||||
The resulting state cannot be State.base_state_name (0).
|
||||
If allow_base is set to False, The resulting state cannot be State.base_state_name (0).
|
||||
|
||||
:param key: Content key to modify state for
|
||||
:type key: str
|
||||
:param or_state: Atomic stat to add
|
||||
:type or_state: int
|
||||
:paran allow_base: Allow state to be reset to 0
|
||||
:type allow_base: bool
|
||||
:raises ValueError: State is not a single bit state, or attempts to revert to State.base_state_name
|
||||
:raises StateItemNotFound: Content key is not registered
|
||||
:raises StateInvalid: Resulting state after addition of atomic state is unknown
|
||||
|
@ -559,7 +602,7 @@ class State:
|
|||
return []
|
||||
|
||||
|
||||
def sync(self, state=None):
|
||||
def sync(self, state=None, not_state=None, ignore_auto=True):
|
||||
"""Noop method for interface implementation providing sync to backend.
|
||||
|
||||
:param state: State to sync.
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
# standard imports
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
|
||||
# local imports
|
||||
from .base import (
|
||||
|
@ -204,9 +205,26 @@ class SimpleFileStoreFactory(StoreFactory):
|
|||
r = []
|
||||
for v in os.listdir(self.__path):
|
||||
if re.match(re_processedname, v):
|
||||
r.append(v)
|
||||
fp = os.path.join(self.__path, v)
|
||||
st = os.stat(fp)
|
||||
if stat.S_ISDIR(st.st_mode):
|
||||
r.append(v)
|
||||
return r
|
||||
|
||||
|
||||
def have(self, k):
|
||||
lock_path = None
|
||||
if self.__use_lock:
|
||||
lock_path = os.path.join(self.__path, '.lock')
|
||||
for d in self.ls():
|
||||
p = os.path.join(self.__path, d)
|
||||
s = SimpleFileStore(p, binary=self.__binary, lock_path=lock_path)
|
||||
try:
|
||||
s.get(k)
|
||||
except:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
|
|
@ -244,18 +244,21 @@ class TestFileStore(unittest.TestCase):
|
|||
|
||||
|
||||
def test_factory_ls(self):
|
||||
r = self.factory.ls()
|
||||
self.assertEqual(len(r), 4)
|
||||
|
||||
self.states.put('abcd')
|
||||
self.states.put('xxxx', state=self.states.BAZ)
|
||||
r = self.factory.ls()
|
||||
self.assertEqual(len(r), 2)
|
||||
self.assertEqual(len(r), 4)
|
||||
|
||||
self.states.put('yyyy', state=self.states.BAZ)
|
||||
r = self.factory.ls()
|
||||
self.assertEqual(len(r), 2)
|
||||
self.assertEqual(len(r), 4)
|
||||
|
||||
self.states.put('zzzz', state=self.states.BAR)
|
||||
r = self.factory.ls()
|
||||
self.assertEqual(len(r), 3)
|
||||
self.assertEqual(len(r), 4)
|
||||
|
||||
|
||||
def test_lock(self):
|
||||
|
|
|
@ -124,7 +124,7 @@ class TestState(unittest.TestCase):
|
|||
states.set('abcd', states.BAZ)
|
||||
v = states.state('abcd')
|
||||
s = states.name(v)
|
||||
self.assertEqual(s, '_FOO.BAZ')
|
||||
self.assertEqual(s, '_FOO__BAZ')
|
||||
|
||||
|
||||
def test_peek(self):
|
||||
|
@ -318,8 +318,41 @@ class TestState(unittest.TestCase):
|
|||
states = State(2, default_state='FOO')
|
||||
with self.assertRaises(StateItemNotFound):
|
||||
states.state('NEW')
|
||||
getattr(states, 'FOO')
|
||||
r = getattr(states, 'FOO')
|
||||
self.assertEqual(r, 0)
|
||||
states.state('FOO')
|
||||
states.put('bar')
|
||||
r = states.list(states.FOO)
|
||||
self.assertEqual(len(r), 1)
|
||||
|
||||
|
||||
def test_unset(self):
|
||||
states = State(2)
|
||||
states.add('one')
|
||||
states.add('two')
|
||||
states.alias('three', states.ONE, states.TWO)
|
||||
states.put('foo', state=states.ONE)
|
||||
states.set('foo', states.TWO)
|
||||
r = states.list(states.ONE)
|
||||
self.assertEqual(len(r), 1)
|
||||
r = states.list(states.TWO)
|
||||
self.assertEqual(len(r), 1)
|
||||
r = states.unset('foo', states.ONE)
|
||||
r = states.list(states.ONE)
|
||||
self.assertEqual(len(r), 0)
|
||||
r = states.list(states.TWO)
|
||||
self.assertEqual(len(r), 1)
|
||||
|
||||
|
||||
def test_move(self):
|
||||
states = State(1)
|
||||
states.add('one')
|
||||
states.put('foo')
|
||||
r = states.list(states.NEW)
|
||||
self.assertEqual(len(r), 1)
|
||||
states.move('foo', states.ONE)
|
||||
r = states.list(states.NEW)
|
||||
self.assertEqual(len(r), 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
@ -43,7 +43,7 @@ class TestStateItems(unittest.TestCase):
|
|||
self.mockstore = MockStore()
|
||||
|
||||
def mockstore_factory(v):
|
||||
self.mockstore.for_state = v
|
||||
#self.mockstore.for_state = v
|
||||
return self.mockstore
|
||||
|
||||
self.states = PersistedState(mockstore_factory, 4)
|
||||
|
@ -71,7 +71,7 @@ class TestStateItems(unittest.TestCase):
|
|||
item = b'foo'
|
||||
self.states.put(item, self.states.FOO, True)
|
||||
self.states.move(item, self.states.XYZZY)
|
||||
self.assertEqual(self.mockstore.for_state, self.states.name(self.states.XYZZY))
|
||||
#self.assertEqual(self.mockstore.for_state, self.states.name(self.states.XYZZY))
|
||||
# TODO: cant check the add because remove happens after remove, need better mock
|
||||
self.assertIsNone(self.mockstore.v.get(item))
|
||||
|
||||
|
@ -80,7 +80,7 @@ class TestStateItems(unittest.TestCase):
|
|||
item = b'foo'
|
||||
self.states.put(item)
|
||||
self.states.move(item, self.states.BAZ)
|
||||
self.assertEqual(self.mockstore.for_state, self.states.name(self.states.BAZ))
|
||||
#self.assertEqual(self.mockstore.for_state, self.states.name(self.states.BAZ))
|
||||
self.assertIsNone(self.mockstore.v.get(item))
|
||||
|
||||
|
||||
|
|
|
@ -8,7 +8,7 @@ from shep.error import (
|
|||
)
|
||||
|
||||
|
||||
def mock_verify(state, from_state, to_state):
|
||||
def mock_verify(state, key, from_state, to_state):
|
||||
if from_state == state.FOO:
|
||||
if to_state == state.BAR:
|
||||
return 'bar cannot follow foo'
|
||||
|
|
Loading…
Reference in New Issue