Complete redis and rocksdb tests for factory ls

CHANGELOG

@@ -1,3 +1,8 @@
+- 0.2.1
+	* Add rocksdb backend
+- 0.2.0
+	* Add redis backend
+	* UTC timestamp for modification time in core state
 - 0.1.1
 	* Optional, pluggable verifier to protect state transition
 	* Change method for atomic simultaneous set and unset

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = shep
-version = 0.1.1rc1
+version = 0.2.1rc1
 description = Multi-state key stores using bit masks
 author = Louis Holbrook
 author_email = dev@holbrook.no

setup.py

@@ -1,3 +1,8 @@
 from setuptools import setup
 
-setup()
+setup(
+    extras_require={
+        'redis': 'redis==3.5.3',
+        'rocksdb': 'lbry-rocksdb==0.8.2',
+        },
+        )

shep/persist.py

@@ -39,7 +39,9 @@ class PersistedState(State):
         k = self.name(to_state)
 
         self.__ensure_store(k)
-        self.__stores[k].add(key, contents)
+        self.__stores[k].put(key, contents)
+
+        self.register_modify(key)
 
 
     def set(self, key, or_state):
@@ -55,7 +57,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         self.sync(to_state)
@@ -77,7 +79,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         return to_state
@@ -97,7 +99,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         self.register_modify(key)
@@ -123,7 +125,7 @@ class PersistedState(State):
         self.__ensure_store(k_to)
 
         contents = self.__stores[k_from].get(key)
-        self.__stores[k_to].add(key, contents)
+        self.__stores[k_to].put(key, contents)
         self.__stores[k_from].remove(key)
 
         self.register_modify(key)

shep/state.py

@@ -613,7 +613,7 @@ class State:
 
 
     def register_modify(self, key):
-        self.modified_last[key] = datetime.datetime.now().timestamp()
+        self.modified_last[key] = datetime.datetime.utcnow().timestamp()
 
 
     def mask(self, key, states=0):

shep/store/base.py

@@ -0,0 +1,15 @@
+re_processedname = r'^_?[A-Z,\.]*$'
+
+
+class StoreFactory:
+
+    def __del__(self):
+        self.close()
+
+
+    def add(self, k):
+        raise NotImplementedError()
+
+
+    def close(self):
+        pass

shep/store/file.py

@@ -1,5 +1,12 @@
 # standard imports
 import os
+import re
+
+# local imports
+from .base import (
+        re_processedname,
+        StoreFactory,
+        )
 
 
 class SimpleFileStore:
@@ -17,7 +24,7 @@ class SimpleFileStore:
             self.__m = ['r', 'w']
 
 
-    def add(self, k, contents=None):
+    def put(self, k, contents=None):
         """Add a new key and optional contents 
 
         :param k: Content key to add 
@@ -120,7 +127,7 @@ class SimpleFileStore:
         pass
 
 
-class SimpleFileStoreFactory:
+class SimpleFileStoreFactory(StoreFactory):
     """Provide a method to instantiate SimpleFileStore instances that provide persistence for individual states.
 
     :param path: Filesystem path as base path for states
@@ -142,3 +149,16 @@ class SimpleFileStoreFactory:
         k = str(k)
         store_path = os.path.join(self.__path, k)
         return SimpleFileStore(store_path, binary=self.__binary)
+
+
+    def ls(self):
+        r = []
+        import sys
+        for v in os.listdir(self.__path):
+            if re.match(re_processedname, v):
+                r.append(v)
+        return r
+
+
+    def close(self):
+        pass

shep/store/redis.py

@@ -0,0 +1,117 @@
+# standard imports
+import datetime
+
+# external imports
+import redis
+
+# local imports
+from .base import StoreFactory
+
+
+class RedisStore:
+
+    def __init__(self, path, redis, binary=False):
+        self.redis = redis
+        self.__path = path
+        self.__binary = binary
+
+
+    def __to_path(self, k):
+        return '.'.join([self.__path, k])
+
+
+    def __from_path(self, s):
+        (left, right) = s.split(b'.', maxsplit=1)
+        return right
+
+
+    def __to_result(self, v):
+        if self.__binary:
+            return v
+        return v.decode('utf-8')
+
+
+    def put(self, k, contents=b''):
+        if contents == None:
+            contents = b''
+        k = self.__to_path(k)
+        self.redis.set(k, contents)
+
+
+    def remove(self, k):
+        k = self.__to_path(k)
+        self.redis.delete(k)
+
+
+    def get(self, k):
+        k = self.__to_path(k)
+        v = self.redis.get(k)
+        return self.__to_result(v)
+
+    
+    def list(self):
+        (cursor, matches) = self.redis.scan(match=self.__path + '.*')
+
+        r = []
+        for s in matches:
+            k = self.__from_path(s)
+            v = self.redis.get(k)
+            r.append((k, v,))
+
+        return r
+
+
+    def path(self):
+        return None
+
+
+    def replace(self, k, contents):
+        if contents == None:
+            contents = b''
+        k = self.__to_path(k)
+        v = self.redis.get(k)
+        if v == None:
+            raise FileNotFoundError(k)
+        self.redis.set(k, contents)
+
+
+    def modified(self, k):
+        k = self.__to_path(k)
+        k = '_mod' + k
+        v = self.redis.get(k)
+        return int(v)
+
+
+    def register_modify(self, k):
+        k = self.__to_path(k)
+        k = '_mod' + k
+        ts = datetime.datetime.utcnow().timestamp()
+        self.redis.set(k)
+
+
+class RedisStoreFactory(StoreFactory):
+
+    def __init__(self, host='localhost', port=6379, db=2, binary=False):
+        self.redis = redis.Redis(host=host, port=port, db=db)
+        self.__binary = binary
+
+
+    def add(self, k):
+        k = str(k)
+        return RedisStore(k, self.redis, binary=self.__binary)
+
+
+    def close(self):
+        self.redis.close()
+
+
+    def ls(self):
+        r = []
+        (c, ks) = self.redis.scan(match='*')
+        for k in ks:
+            v = k.rsplit(b'.', maxsplit=1)
+            if v != k:
+                v = v[0].decode('utf-8')
+                if v not in r:
+                    r.append(v)
+        return r

shep/store/rocksdb.py

@@ -0,0 +1,147 @@
+# standard imports
+import datetime
+import os
+
+# external imports
+import rocksdb
+
+# local imports
+from .base import StoreFactory
+
+
+class RocksDbStore:
+
+    def __init__(self, path, db, binary=False):
+        self.db = db
+        self.__path = path
+        self.__binary = binary
+
+
+    def __to_key(self, k):
+        return k.encode('utf-8')
+
+
+    def __to_contents(self, v):
+        if isinstance(v, bytes):
+            return v
+        return v.encode('utf-8')
+
+
+    def __to_path(self, k):
+        return '.'.join([self.__path, k])
+
+
+    def __from_path(self, s):
+        (left, right) = s.split('.', maxsplit=1)
+        return right
+
+
+    def __to_result(self, v):
+        if self.__binary:
+            return v
+        return v.decode('utf-8')
+
+
+    def put(self, k, contents=b''):
+        if contents == None:
+            contents = b''
+        else:
+            contents = self.__to_contents(contents)
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        self.db.put(k, contents)
+
+
+    def remove(self, k):
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        self.db.delete(k)
+
+
+    def get(self, k):
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        v = self.db.get(k)
+        return self.__to_result(v)
+
+ 
+    def list(self):
+        it = self.db.iteritems()
+        kb_start = self.__to_key(self.__path)
+        it.seek(kb_start)
+
+        r = []
+        l = len(self.__path)
+        for (kb, v) in it:
+            k = kb.decode('utf-8') 
+            if len(k) < l or k[:l] != self.__path:
+                break
+            k = self.__from_path(k)
+            v = self.db.get(kb)
+            r.append((k, v,))
+
+        return r
+
+
+    def path(self):
+        return None
+
+
+    def replace(self, k, contents):
+        if contents == None:
+            contents = b''
+        else:
+            contents = self.__to_contents(contents)
+        k = self.__to_path(k)
+        k = self.__to_key(k)
+        v = self.db.get(k)
+        if v == None:
+            raise FileNotFoundError(k)
+        self.db.put(k, contents)
+
+
+    def modified(self, k):
+        k = self.__to_path(k)
+        k = '_mod' + k
+        v = self.db.get(k)
+        return int(v)
+
+
+    def register_modify(self, k):
+        k = self.__to_path(k)
+        k = '_mod' + k
+        ts = datetime.datetime.utcnow().timestamp()
+        self.db.set(k)
+
+
+class RocksDbStoreFactory(StoreFactory):
+
+    def __init__(self, path, binary=False):
+        try:
+            os.stat(path)
+        except FileNotFoundError:
+            os.makedirs(path)
+        self.db = rocksdb.DB(path, rocksdb.Options(create_if_missing=True))
+        self.__binary = binary
+
+
+    def add(self, k):
+        k = str(k)
+        return RocksDbStore(k, self.db, binary=self.__binary)
+
+
+    def close(self):
+        self.db.close()
+
+
+    def ls(self):
+        it = self.db.iterkeys()
+        it.seek_to_first()
+        r = []
+        for k in it:
+            v = k.rsplit(b'.', maxsplit=1)
+            if v != k:
+                v = v[0].decode('utf-8')
+                if v not in r:
+                    r.append(v)
+        return r

tests/test_file.py

@@ -2,6 +2,7 @@
 import unittest
 import tempfile
 import os
+import shutil
 
 # local imports
 from shep.persist import PersistedState
@@ -13,17 +14,21 @@ from shep.error import (
         )
 
 
-class TestStateReport(unittest.TestCase):
+class TestFileStore(unittest.TestCase):
         
     def setUp(self):
         self.d = tempfile.mkdtemp()
         self.factory = SimpleFileStoreFactory(self.d)
-        self.states = PersistedState(self.factory.add, 4)
+        self.states = PersistedState(self.factory.add, 3)
         self.states.add('foo') 
         self.states.add('bar') 
         self.states.add('baz') 
 
 
+    def tearDown(self):
+        shutil.rmtree(self.d)
+
+
     def test_add(self):
         self.states.put('abcd', state=self.states.FOO, contents='baz')
         fp = os.path.join(self.d, 'FOO', 'abcd')
@@ -176,9 +181,17 @@ class TestStateReport(unittest.TestCase):
         f.write('barbar')
         f.close()
 
+        fp = os.path.join(self.d, 'BAR', 'yyyy')
+        f = open(fp, 'w')
+        f.close()
+
         self.states.sync()
         self.assertEqual(self.states.get('abcd'), None)
+        self.assertEqual(self.states.state('abcd'), self.states.FOO)
         self.assertEqual(self.states.get('zzzz'), 'barbar')
+        self.assertEqual(self.states.state('zzzz'), self.states.BAR)
+        self.assertEqual(self.states.get('yyyy'), None)
+        self.assertEqual(self.states.state('yyyy'), self.states.BAR)
 
 
     def test_path(self):
@@ -206,6 +219,9 @@ class TestStateReport(unittest.TestCase):
         with self.assertRaises(StateInvalid):
             self.states.next('abcd')
 
+        v = self.states.state('abcd')
+        self.assertEqual(v, self.states.BAZ)
+
         fp = os.path.join(self.d, 'FOO', 'abcd')
         with self.assertRaises(FileNotFoundError):
             os.stat(fp)
@@ -226,5 +242,20 @@ class TestStateReport(unittest.TestCase):
         self.assertEqual(r, 'foo')
 
 
+    def test_factory_ls(self):
+        self.states.put('abcd')
+        self.states.put('xxxx', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('yyyy', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('zzzz', state=self.states.BAR)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 3)
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_redis.py

@@ -0,0 +1,112 @@
+# standard imports
+import unittest
+import os
+import logging
+import sys
+import importlib
+
+# local imports
+from shep.persist import PersistedState
+from shep.error import (
+        StateExists,
+        StateInvalid,
+        StateItemExists,
+        StateItemNotFound,
+        )
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class TestRedisStore(unittest.TestCase):
+        
+    def setUp(self):
+        from shep.store.redis import RedisStoreFactory
+        self.factory = RedisStoreFactory()
+        self.factory.redis.flushall()
+        self.states = PersistedState(self.factory.add, 3)
+        self.states.add('foo') 
+        self.states.add('bar') 
+        self.states.add('baz') 
+
+
+    def test_add(self):
+        self.states.put('abcd', state=self.states.FOO, contents='baz')
+        v = self.states.get('abcd')
+        self.assertEqual(v, 'baz')
+        v = self.states.state('abcd')
+        self.assertEqual(v, self.states.FOO)
+
+
+    def test_next(self):
+        self.states.put('abcd')
+
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.FOO)
+        
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.BAR)
+
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.BAZ)
+
+        with self.assertRaises(StateInvalid):
+            self.states.next('abcd')
+
+        v = self.states.state('abcd')
+        self.assertEqual(v, self.states.BAZ)
+
+
+    def test_replace(self):
+        with self.assertRaises(StateItemNotFound):
+            self.states.replace('abcd', contents='foo')
+
+        self.states.put('abcd', state=self.states.FOO, contents='baz')
+        self.states.replace('abcd', contents='bar')
+        v = self.states.get('abcd')
+        self.assertEqual(v, 'bar')
+
+
+    def test_factory_ls(self):
+        r = self.factory.ls()
+        self.assertEqual(len(r), 0)
+
+        self.states.put('abcd')
+        self.states.put('xxxx', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('yyyy', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('zzzz', state=self.states.BAR)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 3)
+
+
+if __name__ == '__main__':
+    noredis = False
+    redis = None
+    try:
+        redis = importlib.import_module('redis')
+    except ModuleNotFoundError:
+        logg.critical('redis module not available, skipping tests.')
+        sys.exit(0)
+
+    host = os.environ.get('REDIS_HOST', 'localhost')
+    port = os.environ.get('REDIS_PORT', 6379)
+    port = int(port)
+    db = os.environ.get('REDIS_DB', 0)
+    db = int(db)
+    r = redis.Redis(host=host, port=port, db=db)
+    try:
+        r.get('foo')
+    except redis.exceptions.ConnectionError:
+        logg.critical('could not connect to redis, skipping tests.')
+        sys.exit(0)
+    except redis.exceptions.InvalidResponse as e:
+        logg.critical('is that really redis running on {}:{}? Got unexpected response: {}'.format(host, port, e))
+        sys.exit(0)
+
+    unittest.main()

tests/test_rocksdb.py

@@ -0,0 +1,100 @@
+# standard imports
+import unittest
+import os
+import logging
+import sys
+import importlib
+import tempfile
+import shutil
+
+# local imports
+from shep.persist import PersistedState
+from shep.error import (
+        StateExists,
+        StateInvalid,
+        StateItemExists,
+        StateItemNotFound,
+        )
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class TestRedisStore(unittest.TestCase):
+        
+    def setUp(self):
+        from shep.store.rocksdb import RocksDbStoreFactory
+        self.d = tempfile.mkdtemp()
+        self.factory = RocksDbStoreFactory(self.d)
+        self.states = PersistedState(self.factory.add, 3)
+        self.states.add('foo') 
+        self.states.add('bar') 
+        self.states.add('baz') 
+
+
+    def tearDown(self):
+        shutil.rmtree(self.d)
+
+
+    def test_add(self):
+        self.states.put('abcd', state=self.states.FOO, contents='baz')
+        v = self.states.get('abcd')
+        self.assertEqual(v, 'baz')
+        v = self.states.state('abcd')
+        self.assertEqual(v, self.states.FOO)
+
+
+    def test_next(self):
+        self.states.put('abcd')
+
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.FOO)
+        
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.BAR)
+
+        self.states.next('abcd')
+        self.assertEqual(self.states.state('abcd'), self.states.BAZ)
+
+        with self.assertRaises(StateInvalid):
+            self.states.next('abcd')
+
+        v = self.states.state('abcd')
+        self.assertEqual(v, self.states.BAZ)
+
+
+    def test_replace(self):
+        with self.assertRaises(StateItemNotFound):
+            self.states.replace('abcd', contents='foo')
+
+        self.states.put('abcd', state=self.states.FOO, contents='baz')
+        self.states.replace('abcd', contents='bar')
+        v = self.states.get('abcd')
+        self.assertEqual(v, 'bar')
+
+
+    def test_factory_ls(self):
+        self.states.put('abcd')
+        self.states.put('xxxx', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('yyyy', state=self.states.BAZ)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 2)
+
+        self.states.put('zzzz', state=self.states.BAR)
+        r = self.factory.ls()
+        self.assertEqual(len(r), 3)
+
+
+if __name__ == '__main__':
+    norocksdb = False
+    rocksdb = None
+    try:
+        importlib.import_module('rocksdb')
+    except ModuleNotFoundError:
+        logg.critical('rocksdb module not available, skipping tests.')
+        sys.exit(0)
+
+    unittest.main()

tests/test_store.py

@@ -21,7 +21,7 @@ class MockStore:
         self.for_state = 0
 
 
-    def add(self, k, contents=None):
+    def put(self, k, contents=None):
         self.v[k] = contents
 
 
@@ -84,6 +84,5 @@ class TestStateItems(unittest.TestCase):
         self.assertIsNone(self.mockstore.v.get(item))
 
 
-
 if __name__ == '__main__':
     unittest.main()

tests/test_verify.py

@@ -0,0 +1,31 @@
+# standard imports
+import unittest
+
+# local imports
+from shep import State
+from shep.error import (
+        StateTransitionInvalid,
+        )
+
+
+def mock_verify(state, from_state, to_state):
+    if from_state == state.FOO:
+        if to_state == state.BAR:
+            return 'bar cannot follow foo'
+
+
+class TestState(unittest.TestCase):
+
+    def test_verify(self):
+        states = State(2, verifier=mock_verify)
+        states.add('foo')
+        states.add('bar')
+        states.put('xyzzy')
+        states.next('xyzzy')
+        with self.assertRaises(StateTransitionInvalid):
+            states.next('xyzzy')
+
+
+
+if __name__ == '__main__':
+    unittest.main()