Factor out common store tests, implement for fs and rocksdb

chainsyncer/store/rocksdb.py

@@ -36,17 +36,17 @@ class SyncRocksDbStore(SyncStore):
     def __init__(self, base_path, session_id=None, state_event_callback=None, filter_state_event_callback=None):
         super(SyncRocksDbStore, self).__init__(base_path, session_id=session_id)
 
-        factory = RocksDbStoreFactory(self.session_path, binary=True)
+        self.factory = RocksDbStoreFactory(self.session_path, binary=True)
-        prefix_factory = RocksDbStoreAdder(factory, 'sync')
+        prefix_factory = RocksDbStoreAdder(self.factory, 'sync')
         self.setup_sync_state(prefix_factory, state_event_callback)
 
-        prefix_factory = RocksDbStoreAdder(factory, 'filter')
+        prefix_factory = RocksDbStoreAdder(self.factory, 'filter')
         self.setup_filter_state(prefix_factory, filter_state_event_callback)
 
         self.session_id = os.path.basename(self.session_path)
         logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path))
 
-        self.target_db = RocksDbStoreAdder(factory, '.stat').add('target')
+        self.target_db = RocksDbStoreAdder(self.factory, '.stat').add('target')
 
 
     def get_target(self):
@@ -55,3 +55,8 @@ class SyncRocksDbStore(SyncStore):
 
     def set_target(self, v):
         self.target_db.put('target')
+
+
+    def disconnect(self):
+        super(SyncRocksDbStore, self).disconnect()
+        self.factory.close()

chainsyncer/unittest/db.py

@@ -1,64 +0,0 @@
-# standard imports
-import logging
-import os
-
-# external imports
-import alembic
-import alembic.config
-
-# local imports
-from chainsyncer.db.models.base import SessionBase
-from chainsyncer.db import dsn_from_config
-from chainsyncer.db.models.base import SessionBase
-
-logg = logging.getLogger(__name__)
-
-
-class ChainSyncerDb:
-    """SQLITE database setup for unit tests
-
-    :param debug: Activate sql level debug (outputs sql statements)
-    :type debug: bool
-    """
-
-    base = SessionBase
-
-    def __init__(self, debug=False):
-        config = {
-            'DATABASE_ENGINE': 'sqlite',
-            'DATABASE_DRIVER': 'pysqlite',
-            'DATABASE_NAME': 'chainsyncer.sqlite',
-                }
-        logg.debug('config {}'.format(config))
-
-        self.dsn = dsn_from_config(config)
-
-        self.base.poolable = False
-        self.base.transactional = False
-        self.base.procedural = False
-        self.base.connect(self.dsn, debug=debug) # TODO: evaluates to "true" even if string is 0
-
-        rootdir = os.path.join(os.path.dirname(os.path.dirname(__file__)), '..')
-        dbdir = os.path.join(rootdir, 'chainsyncer', 'db')
-        #migrationsdir = os.path.join(dbdir, 'migrations', config.get('DATABASE_ENGINE'))
-        migrationsdir = os.path.join(dbdir, 'migrations', 'default')
-        logg.info('using migrations directory {}'.format(migrationsdir))
-
-        ac = alembic.config.Config(os.path.join(migrationsdir, 'alembic.ini'))
-        ac.set_main_option('sqlalchemy.url', self.dsn)
-        ac.set_main_option('script_location', migrationsdir)
-
-        alembic.command.downgrade(ac, 'base')
-        alembic.command.upgrade(ac, 'head')
-
-
-    def bind_session(self, session=None):
-        """Create session using underlying session base
-        """
-        return self.base.bind_session(session)
-
-    
-    def release_session(self, session=None):
-        """Release session using underlying session base
-        """
-        return self.base.release_session(session)

chainsyncer/unittest/store.py

@@ -0,0 +1,267 @@
+# standard imports
+import os
+import stat
+import unittest
+import shutil
+import tempfile
+import logging
+
+# local imports
+from chainsyncer.session import SyncSession
+from chainsyncer.error import (
+        LockError,
+        FilterDone,
+        IncompleteFilterError,
+        SyncDone,
+        )
+from chainsyncer.unittest import MockFilter
+
+logg = logging.getLogger(__name__)
+
+
+class TestStoreBase(unittest.TestCase):
+
+    def setUp(self):
+        self.path = tempfile.mkdtemp()
+        self.store_factory = None
+
+
+    @classmethod
+    def link(cls, target):
+        for v in [
+                "default",
+                "store_start",
+                "store_resume",
+                "sync_process_nofilter",
+                "sync_process_onefilter",
+                "sync_process_outoforder",
+                "sync_process_interrupt",
+                "sync_process_reset",
+                "sync_process_done",
+                "sync_head_future",
+                "sync_history_interrupted",
+                "sync_history_complete",
+                ]:
+            setattr(target, 'test_' + v, getattr(cls, 't_' + v))
+
+
+    def tearDown(self):
+        shutil.rmtree(self.path)
+
+
+    def t_default(self):
+        store = self.store_factory()
+        
+        fp = os.path.join(self.path, store.session_id)
+        session_id = store.session_id
+        st = os.stat(fp)
+        self.assertTrue(stat.S_ISDIR(st.st_mode))
+        self.assertTrue(store.is_default)
+        
+        fpd = os.path.join(self.path, 'default')
+        st = os.stat(fpd)
+        self.assertTrue(stat.S_ISDIR(st.st_mode))
+        self.assertTrue(store.is_default)
+
+        fpd = os.path.realpath(fpd)
+        self.assertEqual(fpd, fp)
+
+        store.disconnect()
+        store = self.store_factory()
+        fpr = os.path.join(self.path, session_id)
+        self.assertEqual(fp, fpr)
+        self.assertTrue(store.is_default)
+
+        store.disconnect()
+        store = self.store_factory('default')
+        fpr = os.path.join(self.path, session_id)
+        self.assertEqual(fp, fpr)
+        self.assertTrue(store.is_default)
+
+        store.disconnect()
+        store = self.store_factory('foo')
+        fpf = os.path.join(self.path, 'foo')
+        st = os.stat(fpf)
+        self.assertTrue(stat.S_ISDIR(st.st_mode))
+        self.assertFalse(store.is_default)
+
+
+    def t_store_start(self):
+        store = self.store_factory()
+        store.start(42)
+        self.assertTrue(store.first)
+
+        store.disconnect()
+        store = self.store_factory()
+        store.start()
+        self.assertFalse(store.first)
+
+
+    def t_store_resume(self):
+        store = self.store_factory()
+        store.start(13)
+        self.assertTrue(store.first)
+        # todo not done
+
+
+    def t_sync_process_nofilter(self):
+        store = self.store_factory()
+        session = SyncSession(store)
+        session.start()
+        o = session.get(0)
+        with self.assertRaises(FilterDone):
+            o.advance()
+
+
+    def t_sync_process_onefilter(self):
+        store = self.store_factory()
+        session = SyncSession(store)
+
+        fltr_one = MockFilter('foo')
+        store.register(fltr_one)
+
+        session.start()
+        o = session.get(0)
+        o.advance()
+        o.release()
+
+
+    def t_sync_process_outoforder(self):
+        store = self.store_factory()
+        session = SyncSession(store)
+
+        fltr_one = MockFilter('foo')
+        store.register(fltr_one)
+        fltr_two = MockFilter('two')
+        store.register(fltr_two)
+
+        session.start()
+        o = session.get(0)
+        o.advance()
+        with self.assertRaises(LockError):
+            o.advance()
+
+        o.release()
+        with self.assertRaises(LockError):
+            o.release()
+
+        o.advance()
+        o.release()
+
+
+    def t_sync_process_interrupt(self):
+        store = self.store_factory()
+        session = SyncSession(store)
+
+        fltr_one = MockFilter('foo')
+        store.register(fltr_one)
+        fltr_two = MockFilter('bar')
+        store.register(fltr_two)
+
+        session.start()
+        o = session.get(0)
+        o.advance()
+        o.release(interrupt=True)
+        with self.assertRaises(FilterDone):
+            o.advance()
+
+
+    def t_sync_process_reset(self):
+        store = self.store_factory()
+        session = SyncSession(store)
+
+        fltr_one = MockFilter('foo')
+        store.register(fltr_one)
+        fltr_two = MockFilter('bar')
+        store.register(fltr_two)
+
+        session.start()
+        o = session.get(0)
+        o.advance()
+        with self.assertRaises(LockError):
+            o.reset()
+        o.release()
+        with self.assertRaises(IncompleteFilterError):
+            o.reset()
+
+        o.advance()
+        o.release()
+
+        with self.assertRaises(FilterDone):
+            o.advance()
+
+        o.reset()
+
+    
+    def t_sync_process_done(self):
+        store = self.store_factory()
+        session = SyncSession(store)
+
+        fltr_one = MockFilter('foo')
+        store.register(fltr_one)
+
+        session.start(target=0)
+        o = session.get(0)
+        o.advance()
+        o.release()
+        with self.assertRaises(FilterDone):
+            o.advance()
+        o.reset()
+        with self.assertRaises(SyncDone):
+            o.next(advance_block=True)
+
+
+    def t_sync_head_future(self):
+        store = self.store_factory('foo')
+        session = SyncSession(store)
+
+        session.start()
+        o = session.get(0)
+        o.next(advance_block=True)
+        o.next(advance_block=True)
+        session.stop(o)
+
+        store.disconnect()
+        store = self.store_factory('foo')
+        store.start()
+        o = store.get(2)
+
+
+    def t_sync_history_interrupted(self):
+        store = self.store_factory('foo')
+        session = SyncSession(store)
+
+        session.start(target=13)
+        o = session.get(0)
+        o.next(advance_block=True)
+        o.next(advance_block=True)
+        session.stop(o)
+
+        store.disconnect()
+        store = self.store_factory('foo')
+        store.start()
+        o = store.get(0)
+        self.assertEqual(o.cursor, 2)
+        self.assertEqual(o.target, 13) 
+        o.next(advance_block=True)
+        o.next(advance_block=True)
+
+        session.stop(o)
+        store.disconnect()
+        store = self.store_factory('foo')
+        store.start()
+        self.assertEqual(o.cursor, 4)
+        self.assertEqual(o.target, 13) 
+
+
+    def t_sync_history_complete(self):
+        store = self.store_factory('foo')
+        session = SyncSession(store)
+
+        session.start(target=3)
+        o = session.get(0)
+        o.next(advance_block=True)
+        o.next(advance_block=True)
+        o.next(advance_block=True)
+        with self.assertRaises(SyncDone):
+            o.next(advance_block=True)

tests/test_fs.py

@@ -1,245 +1,32 @@
 # standard imports
 import unittest
-import tempfile
-import shutil
 import logging
-import stat
-import os
 
 # local imports
 from chainsyncer.store.fs import SyncFsStore
-from chainsyncer.session import SyncSession
+from chainsyncer.unittest.store import TestStoreBase
-from chainsyncer.error import (
-        LockError,
-        FilterDone,
-        IncompleteFilterError,
-        SyncDone,
-        )
-from chainsyncer.unittest import MockFilter
 
 logging.basicConfig(level=logging.DEBUG)
 logg = logging.getLogger()
 
 
-class TestFs(unittest.TestCase):
+class StoreFactory:
+
+    def __init__(self, path):
+        self.path = path
+
+
+    def create(self, session_id=None):
+        return SyncFsStore(self.path, session_id=session_id)
+
+
+class TestFs(TestStoreBase):
 
     def setUp(self):
-        self.path = tempfile.mkdtemp()
+        super(TestFs, self).setUp()
-
+        self.store_factory = StoreFactory(self.path).create
-
-    def tearDown(self):
-        shutil.rmtree(self.path)
-
-
-    def test_default(self):
-        store = SyncFsStore(self.path)
-        fp = os.path.join(self.path, store.session_id)
-        session_id = store.session_id
-        st = os.stat(fp)
-        self.assertTrue(stat.S_ISDIR(st.st_mode))
-        self.assertTrue(store.is_default)
-        
-        fpd = os.path.join(self.path, 'default')
-        st = os.stat(fpd)
-        self.assertTrue(stat.S_ISDIR(st.st_mode))
-        self.assertTrue(store.is_default)
-
-        fpd = os.path.realpath(fpd)
-        self.assertEqual(fpd, fp)
-
-        store = SyncFsStore(self.path)
-        fpr = os.path.join(self.path, session_id)
-        self.assertEqual(fp, fpr)
-        self.assertTrue(store.is_default)
-
-        store = SyncFsStore(self.path, 'default')
-        fpr = os.path.join(self.path, session_id)
-        self.assertEqual(fp, fpr)
-        self.assertTrue(store.is_default)
-
-        store = SyncFsStore(self.path, 'foo')
-        fpf = os.path.join(self.path, 'foo')
-        st = os.stat(fpf)
-        self.assertTrue(stat.S_ISDIR(st.st_mode))
-        self.assertFalse(store.is_default)
-
-
-    def test_store_start(self):
-        store = SyncFsStore(self.path)
-        store.start(42)
-        self.assertTrue(store.first)
-
-        store = SyncFsStore(self.path)
-        store.start()
-        self.assertFalse(store.first)
-
-
-    def test_store_resume(self):
-        store = SyncFsStore(self.path)
-        store.start(13)
-        self.assertTrue(store.first)
-        # todo not done
-
-
-    def test_sync_process_nofilter(self):
-        store = SyncFsStore(self.path)
-        session = SyncSession(store)
-        session.start()
-        o = session.get(0)
-        with self.assertRaises(FilterDone):
-            o.advance()
-
-
-    def test_sync_process_onefilter(self):
-        store = SyncFsStore(self.path)
-        session = SyncSession(store)
-
-        fltr_one = MockFilter('foo')
-        store.register(fltr_one)
-
-        session.start()
-        o = session.get(0)
-        o.advance()
-        o.release()
-
-
-    def test_sync_process_outoforder(self):
-        store = SyncFsStore(self.path)
-        session = SyncSession(store)
-
-        fltr_one = MockFilter('foo')
-        store.register(fltr_one)
-        fltr_two = MockFilter('two')
-        store.register(fltr_two)
-
-        session.start()
-        o = session.get(0)
-        o.advance()
-        with self.assertRaises(LockError):
-            o.advance()
-
-        o.release()
-        with self.assertRaises(LockError):
-            o.release()
-
-        o.advance()
-        o.release()
-
-
-    def test_sync_process_interrupt(self):
-        store = SyncFsStore(self.path)
-        session = SyncSession(store)
-
-        fltr_one = MockFilter('foo')
-        store.register(fltr_one)
-        fltr_two = MockFilter('bar')
-        store.register(fltr_two)
-
-        session.start()
-        o = session.get(0)
-        o.advance()
-        o.release(interrupt=True)
-        with self.assertRaises(FilterDone):
-            o.advance()
-
-
-    def test_sync_process_reset(self):
-        store = SyncFsStore(self.path)
-        session = SyncSession(store)
-
-        fltr_one = MockFilter('foo')
-        store.register(fltr_one)
-        fltr_two = MockFilter('bar')
-        store.register(fltr_two)
-
-        session.start()
-        o = session.get(0)
-        o.advance()
-        with self.assertRaises(LockError):
-            o.reset()
-        o.release()
-        with self.assertRaises(IncompleteFilterError):
-            o.reset()
-
-        o.advance()
-        o.release()
-
-        with self.assertRaises(FilterDone):
-            o.advance()
-
-        o.reset()
-
-    
-    def test_sync_process_done(self):
-        store = SyncFsStore(self.path)
-        session = SyncSession(store)
-
-        fltr_one = MockFilter('foo')
-        store.register(fltr_one)
-
-        session.start(target=0)
-        o = session.get(0)
-        o.advance()
-        o.release()
-        with self.assertRaises(FilterDone):
-            o.advance()
-        o.reset()
-        with self.assertRaises(SyncDone):
-            o.next(advance_block=True)
-
-
-    def test_sync_head_future(self):
-        store = SyncFsStore(self.path, session_id='foo')
-        session = SyncSession(store)
-
-        session.start()
-        o = session.get(0)
-        o.next(advance_block=True)
-        o.next(advance_block=True)
-        session.stop(o)
-
-        store = SyncFsStore(self.path, session_id='foo')
-        store.start()
-        o = store.get(2)
-
-
-    def test_sync_history_interrupted(self):
-        store = SyncFsStore(self.path, session_id='foo')
-        session = SyncSession(store)
-
-        session.start(target=13)
-        o = session.get(0)
-        o.next(advance_block=True)
-        o.next(advance_block=True)
-        session.stop(o)
-
-        store = SyncFsStore(self.path, session_id='foo')
-        store.start()
-        o = store.get(0)
-        self.assertEqual(o.cursor, 2)
-        self.assertEqual(o.target, 13) 
-        o.next(advance_block=True)
-        o.next(advance_block=True)
-
-        session.stop(o)
-        store = SyncFsStore(self.path, session_id='foo')
-        store.start()
-        self.assertEqual(o.cursor, 4)
-        self.assertEqual(o.target, 13) 
-
-
-    def test_sync_history_complete(self):
-        store = SyncFsStore(self.path, session_id='foo')
-        session = SyncSession(store)
-
-        session.start(target=3)
-        o = session.get(0)
-        o.next(advance_block=True)
-        o.next(advance_block=True)
-        o.next(advance_block=True)
-        with self.assertRaises(SyncDone):
-            o.next(advance_block=True)
 
 
 if __name__ == '__main__':
+    TestStoreBase.link(TestFs)
     unittest.main()

tests/test_rocksdb.py

@@ -1,37 +1,31 @@
 # standard imports
 import unittest
-import tempfile
-import shutil
 import logging
-import stat
-import os
 
 # local imports
 from chainsyncer.store.rocksdb import SyncRocksDbStore
-from chainsyncer.session import SyncSession
+from chainsyncer.unittest.store import TestStoreBase
-from chainsyncer.error import (
-        LockError,
-        FilterDone,
-        IncompleteFilterError,
-        SyncDone,
-        )
-from chainsyncer.unittest import MockFilter
 
 logging.basicConfig(level=logging.DEBUG)
 logg = logging.getLogger()
 
+class StoreFactory:
 
-class TestFs(unittest.TestCase):
+    def __init__(self, path):
+        self.path = path
+
+
+    def create(self, session_id=None):
+        return SyncRocksDbStore(self.path, session_id=session_id)
+
+
+class TestRocksDb(TestStoreBase):
 
     def setUp(self):
-        self.path = tempfile.mkdtemp()
+        super(TestRocksDb, self).setUp()
-
+        self.store_factory = StoreFactory(self.path).create
-
-    def test_default(self):
-        store = SyncRocksDbStore(self.path)
-        store.start(42)
-        self.assertTrue(store.first)
 
 
 if __name__ == '__main__':
+    TestStoreBase.link(TestRocksDb)
     unittest.main()