Correct regex for state recovery from persistnet store

CHANGELOG

@@ -1,3 +1,7 @@
+- 0.2.2
+	* Fix composite state factory load regex
+- 0.2.1
+	* Add rocksdb backend
 - 0.2.0
 	* Add redis backend
 	* UTC timestamp for modification time in core state

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = shep
-version = 0.2.0rc1
+version = 0.2.2
 description = Multi-state key stores using bit masks
 author = Louis Holbrook
 author_email = dev@holbrook.no

setup.py

@@ -1,3 +1,8 @@
 from setuptools import setup
 
-setup()
+setup(
+    extras_require={
+        'redis': 'redis==3.5.3',
+        'rocksdb': 'lbry-rocksdb==0.8.2',
+        },
+        )

shep/persist.py

@@ -39,7 +39,7 @@ class PersistedState(State):
         k = self.name(to_state)
 
         self.__ensure_store(k)
-        self.__stores[k].add(key, contents)
+        self.__stores[k].put(key, contents)
 
         self.register_modify(key)
 
@@ -57,7 +57,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         self.sync(to_state)
@@ -79,7 +79,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         return to_state
@@ -99,7 +99,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         self.register_modify(key)
@@ -125,7 +125,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         self.register_modify(key)

shep/store/base.py

@@ -0,0 +1,15 @@
+re_processedname = r'^_?[A-Z\._]*$'
+
+
+class StoreFactory:
+
+    def __del__(self):
+        self.close()
+
+
+    def add(self, k):
+        raise NotImplementedError()
+
+
+    def close(self):
+        pass

shep/store/file.py

@@ -1,5 +1,12 @@
 # standard imports
 import os
+import re
+
+# local imports
+from .base import (
+        re_processedname,
+        StoreFactory,
+        )
 
 
 class SimpleFileStore:
@@ -17,7 +24,7 @@ class SimpleFileStore:
             self.__m = ['r', 'w']
 
 
-    def add(self, k, contents=None):
+    def put(self, k, contents=None):
         """Add a new key and optional contents 
 
         :param k: Content key to add 
@@ -120,7 +127,7 @@ class SimpleFileStore:
         pass
 
 
-class SimpleFileStoreFactory:
+class SimpleFileStoreFactory(StoreFactory):
     """Provide a method to instantiate SimpleFileStore instances that provide persistence for individual states.
 
     :param path: Filesystem path as base path for states
@@ -142,3 +149,15 @@ class SimpleFileStoreFactory:
         k = str(k)
         store_path = os.path.join(self.__path, k)
         return SimpleFileStore(store_path, binary=self.__binary)
+
+
+    def ls(self):
+        r = []
+        for v in os.listdir(self.__path):
+            if re.match(re_processedname, v):
+                r.append(v)
+        return r
+
+
+    def close(self):
+        pass

shep/store/redis.py

@@ -1,6 +1,12 @@
+# standard imports
+import datetime
+
 # external imports
 import redis
 
+# local imports
+from .base import StoreFactory
+
 
 class RedisStore:
 
@@ -9,12 +15,13 @@ class RedisStore:
         self.__path = path
         self.__binary = binary
 
+
     def __to_path(self, k):
         return '.'.join([self.__path, k])
 
 
     def __from_path(self, s):
-        (left, right) = s.split('.', maxsplit=1)
+        (left, right) = s.split(b'.', maxsplit=1)
         return right
 
 
@@ -24,7 +31,7 @@ class RedisStore:
         return v.decode('utf-8')
 
 
-    def add(self, k, contents=b''):
+    def put(self, k, contents=b''):
         if contents == None:
             contents = b''
         k = self.__to_path(k)
@@ -48,7 +55,7 @@ class RedisStore:
         r = []
         for s in matches:
             k = self.__from_path(s)
-            v = self.redis.get(v)
+            v = self.redis.get(k)
             r.append((k, v,))
 
         return r
@@ -82,9 +89,9 @@ class RedisStore:
         self.redis.set(k)
 
 
-class RedisStoreFactory:
+class RedisStoreFactory(StoreFactory):
 
-    def __init__(self, host='localhost', port=6379, db=0, binary=False):
+    def __init__(self, host='localhost', port=6379, db=2, binary=False):
         self.redis = redis.Redis(host=host, port=port, db=db)
         self.__binary = binary
 
@@ -92,3 +99,19 @@ class RedisStoreFactory:
     def add(self, k):
         k = str(k)
         return RedisStore(k, self.redis, binary=self.__binary)
+
+
+    def close(self):
+        self.redis.close()
+
+
+    def ls(self):
+        r = []
+        (c, ks) = self.redis.scan(match='*')
+        for k in ks:
+            v = k.rsplit(b'.', maxsplit=1)
+            if v != k:
+                v = v[0].decode('utf-8')
+                if v not in r:
+                    r.append(v)
+        return r

shep/store/rocksdb.py

@@ -0,0 +1,147 @@
+# standard imports
+import datetime
+import os
+
+# external imports
+import rocksdb
+
+# local imports
+from .base import StoreFactory
+
+
+class RocksDbStore:
+
+    def __init__(self, path, db, binary=False):
+        self.db = db
+        self.__path = path
+        self.__binary = binary
+
+
+    def __to_key(self, k):
+        return k.encode('utf-8')
+
+
+    def __to_contents(self, v):
+        if isinstance(v, bytes):
+            return v
+        return v.encode('utf-8')
+
+
+    def __to_path(self, k):
+        return '.'.join([self.__path, k])
+
+
+    def __from_path(self, s):
+        (left, right) = s.split('.', maxsplit=1)
+        return right
+
+
+    def __to_result(self, v):
+        if self.__binary:
+            return v
+        return v.decode('utf-8')
+
+
+    def put(self, k, contents=b''):
+        if contents == None:
+            contents = b''
+        else:
+            contents = self.__to_contents(contents)
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        self.db.put(k, contents)
+
+
+    def remove(self, k):
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        self.db.delete(k)
+
+
+    def get(self, k):
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        v = self.db.get(k)
+        return self.__to_result(v)
+
+ 
+    def list(self):
+        it = self.db.iteritems()
+        kb_start = self.__to_key(self.__path)
+        it.seek(kb_start)
+
+        r = []
+        l = len(self.__path)
+        for (kb, v) in it:
+            k = kb.decode('utf-8') 
+            if len(k) < l or k[:l] != self.__path:
+                break
+            k = self.__from_path(k)
+            v = self.db.get(kb)
+            r.append((k, v,))
+
+        return r
+
+
+    def path(self):
+        return None
+
+
+    def replace(self, k, contents):
+        if contents == None:
+            contents = b''
+        else:
+            contents = self.__to_contents(contents)
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        v = self.db.get(k)
+        if v == None:
+            raise FileNotFoundError(k)
+        self.db.put(k, contents)
+
+
+    def modified(self, k):
+        k = self.__to_path(k)
+        k = '_mod' + k
+        v = self.db.get(k)
+        return int(v)
+
+
+    def register_modify(self, k):
+        k = self.__to_path(k)
+        k = '_mod' + k
+        ts = datetime.datetime.utcnow().timestamp()
+        self.db.set(k)
+
+
+class RocksDbStoreFactory(StoreFactory):
+
+    def __init__(self, path, binary=False):
+        try:
+            os.stat(path)
+        except FileNotFoundError:
+            os.makedirs(path)
+        self.db = rocksdb.DB(path, rocksdb.Options(create_if_missing=True))
+        self.__binary = binary
+
+
+    def add(self, k):
+        k = str(k)
+        return RocksDbStore(k, self.db, binary=self.__binary)
+
+
+    def close(self):
+        self.db.close()
+
+
+    def ls(self):
+        it = self.db.iterkeys()
+        it.seek_to_first()
+        r = []
+        for k in it:
+            v = k.rsplit(b'.', maxsplit=1)
+            if v != k:
+                v = v[0].decode('utf-8')
+                if v not in r:
+                    r.append(v)
+        return r

tests/test_file.py

@@ -2,6 +2,7 @@
 import unittest
 import tempfile
 import os
+import shutil
 
 # local imports
 from shep.persist import PersistedState
@@ -24,6 +25,10 @@ class TestFileStore(unittest.TestCase):
         self.states.add('baz') 
 
 
+    def tearDown(self):
+        shutil.rmtree(self.d)
+
+
     def test_add(self):
         self.states.put('abcd', state=self.states.FOO, contents='baz')
         fp = os.path.join(self.d, 'FOO', 'abcd')
@@ -176,9 +181,17 @@ class TestFileStore(unittest.TestCase):
         f.write('barbar')
         f.close()
 
+        fp = os.path.join(self.d, 'BAR', 'yyyy')
+        f = open(fp, 'w')
+        f.close()
+
         self.states.sync()
         self.assertEqual(self.states.get('abcd'), None)
+        self.assertEqual(self.states.state('abcd'), self.states.FOO)
         self.assertEqual(self.states.get('zzzz'), 'barbar')
+        self.assertEqual(self.states.state('zzzz'), self.states.BAR)
+        self.assertEqual(self.states.get('yyyy'), None)
+        self.assertEqual(self.states.state('yyyy'), self.states.BAR)
 
 
     def test_path(self):
@@ -229,5 +242,20 @@ class TestFileStore(unittest.TestCase):
         self.assertEqual(r, 'foo')
 
 
+    def test_factory_ls(self):
+        self.states.put('abcd')
+        self.states.put('xxxx', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('yyyy', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('zzzz', state=self.states.BAR)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 3)
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_redis.py

@@ -23,6 +23,7 @@ class TestRedisStore(unittest.TestCase):
     def setUp(self):
         from shep.store.redis import RedisStoreFactory
         self.factory = RedisStoreFactory()
+        self.factory.redis.flushall()
         self.states = PersistedState(self.factory.add, 3)
         self.states.add('foo') 
         self.states.add('bar') 
@@ -66,6 +67,24 @@ class TestRedisStore(unittest.TestCase):
         self.assertEqual(v, 'bar')
 
 
+    def test_factory_ls(self):
+        r = self.factory.ls()
+        self.assertEqual(len(r), 0)
+
+        self.states.put('abcd')
+        self.states.put('xxxx', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('yyyy', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('zzzz', state=self.states.BAR)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 3)
+
+
 if __name__ == '__main__':
     noredis = False
     redis = None

tests/test_rocksdb.py

@@ -0,0 +1,100 @@
+# standard imports
+import unittest
+import os
+import logging
+import sys
+import importlib
+import tempfile
+import shutil
+
+# local imports
+from shep.persist import PersistedState
+from shep.error import (
+        StateExists,
+        StateInvalid,
+        StateItemExists,
+        StateItemNotFound,
+        )
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class TestRedisStore(unittest.TestCase):
+        
+    def setUp(self):
+        from shep.store.rocksdb import RocksDbStoreFactory
+        self.d = tempfile.mkdtemp()
+        self.factory = RocksDbStoreFactory(self.d)
+        self.states = PersistedState(self.factory.add, 3)
+        self.states.add('foo') 
+        self.states.add('bar') 
+        self.states.add('baz') 
+
+
+    def tearDown(self):
+        shutil.rmtree(self.d)
+
+
+    def test_add(self):
+        self.states.put('abcd', state=self.states.FOO, contents='baz')
+        v = self.states.get('abcd')
+        self.assertEqual(v, 'baz')
+        v = self.states.state('abcd')
+        self.assertEqual(v, self.states.FOO)
+
+
+    def test_next(self):
+        self.states.put('abcd')
+
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.FOO)
+        
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.BAR)
+
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.BAZ)
+
+        with self.assertRaises(StateInvalid):
+            self.states.next('abcd')
+
+        v = self.states.state('abcd')
+        self.assertEqual(v, self.states.BAZ)
+
+
+    def test_replace(self):
+        with self.assertRaises(StateItemNotFound):
+            self.states.replace('abcd', contents='foo')
+
+        self.states.put('abcd', state=self.states.FOO, contents='baz')
+        self.states.replace('abcd', contents='bar')
+        v = self.states.get('abcd')
+        self.assertEqual(v, 'bar')
+
+
+    def test_factory_ls(self):
+        self.states.put('abcd')
+        self.states.put('xxxx', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('yyyy', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('zzzz', state=self.states.BAR)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 3)
+
+
+if __name__ == '__main__':
+    norocksdb = False
+    rocksdb = None
+    try:
+        importlib.import_module('rocksdb')
+    except ModuleNotFoundError:
+        logg.critical('rocksdb module not available, skipping tests.')
+        sys.exit(0)
+
+    unittest.main()

tests/test_store.py

@@ -21,7 +21,7 @@ class MockStore:
         self.for_state = 0
 
 
-    def add(self, k, contents=None):
+    def put(self, k, contents=None):
         self.v[k] = contents
 
 
@@ -84,6 +84,5 @@ class TestStateItems(unittest.TestCase):
         self.assertIsNone(self.mockstore.v.get(item))
 
 
-
 if __name__ == '__main__':
     unittest.main()