Allow memory-only syncing

CHANGELOG

@@ -1,3 +1,7 @@
+* 0.3.5
+	- Allow memory-only shep if factory set to None in store constructor
+* 0.3.4
+	- Use explicit bool check in filter interrupt check
 * 0.3.3
 	- Include shep persistent state bootstrap sync
 	- Add chainsyncer extras

chainsyncer/store/base.py

@@ -4,6 +4,7 @@ import logging
 
 # local imports
 from shep.persist import PersistedState
+from shep import State
 from shep.error import StateInvalid
 from chainsyncer.filter import FilterState
 from chainsyncer.error import (
@@ -168,13 +169,20 @@ class SyncStore:
             self.session_path = os.path.realpath(given_path)
 
 
-    def setup_sync_state(self, factory, event_callback):
+    def setup_sync_state(self, factory=None, event_callback=None):
+        if factory == None:
+            self.state = State(2, event_callback=event_callback)
+        else:
             self.state = PersistedState(factory.add, 2, event_callback=event_callback)
         self.state.add('SYNC')
         self.state.add('DONE')
 
 
-    def setup_filter_state(self, factory, event_callback):
+    def setup_filter_state(self, factory=None, event_callback=None):
+        if factory == None:
+            filter_state_backend = State(0, check_alias=False, event_callback=event_callback)
+            self.filter_state = FilterState(filter_state_backend)
+        else:
             filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback)
             self.filter_state = FilterState(filter_state_backend, scan=factory.ls)
         self.filters = []
@@ -202,7 +210,7 @@ class SyncStore:
         if self.first:
             state_bytes = sync_state_serialize(offset, 0, target)
             block_number_str = str(offset)
-            self.state.put(block_number_str, state_bytes)
+            self.state.put(block_number_str, contents=state_bytes)
             self.filter_state.put(block_number_str)
             o = SyncItem(offset, target, self.state, self.filter_state)
             self.items[offset] = o
@@ -226,7 +234,7 @@ class SyncStore:
             self.state.move(item.state_key, self.state.DONE)
 
             state_bytes = sync_state_serialize(item.cursor, 0, -1)
-            self.state.put(str(item.cursor), state_bytes)
+            self.state.put(str(item.cursor), contents=state_bytes)
 
 
     def load(self, target):

chainsyncer/store/fs.py

@@ -7,10 +7,7 @@ import logging
 from shep.store.file import SimpleFileStoreFactory
 
 # local imports 
-from chainsyncer.store import (
+from chainsyncer.store import SyncStore
-        SyncItem,
-        SyncStore,
-        )
 
 logg = logging.getLogger(__name__)
 

chainsyncer/store/mem.py

@@ -0,0 +1,34 @@
+# standard imports
+import logging
+import os
+
+# external imports
+from shep import State
+
+# local imports 
+from chainsyncer.store import SyncStore
+
+logg = logging.getLogger(__name__)
+
+
+class SyncMemStore(SyncStore):
+
+    def __init__(self, session_id=None, state_event_callback=None, filter_state_event_callback=None):
+        super(SyncMemStore, self).__init__('/dev/null', session_id=session_id)
+
+        self.session_id = os.path.basename(self.session_path)
+        logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path))
+
+        factory = None
+        self.setup_sync_state(factory, state_event_callback)
+
+        factory = None
+        self.setup_filter_state(factory, filter_state_event_callback)
+
+
+    def set_target(self, v):
+        self.target = int(v)
+
+
+    def get_target(self):
+        return self.target

chainsyncer/unittest/store.py

@@ -19,7 +19,17 @@ from chainsyncer.unittest import (
         MockItem,
         )
 
+logging.STATETRACE = 5
 logg = logging.getLogger(__name__)
+logg.setLevel(logging.STATETRACE)
+
+
+def state_change_callback(k, old_state, new_state):
+    logg.log(logging.STATETRACE, 'state change: {} {} -> {}'.format(k, old_state, new_state)) 
+
+
+def filter_change_callback(k, old_state, new_state):
+    logg.log(logging.STATETRACE, 'filter change: {} {} -> {}'.format(k, old_state, new_state)) 
 
 
 class TestStoreBase(unittest.TestCase):
@@ -27,6 +37,7 @@ class TestStoreBase(unittest.TestCase):
     def setUp(self):
         self.path = tempfile.mkdtemp()
         self.store_factory = None
+        self.persist = True
 
 
     @classmethod
@@ -58,12 +69,26 @@ class TestStoreBase(unittest.TestCase):
         
         fp = os.path.join(self.path, store.session_id)
         session_id = store.session_id
+        st = None
+        try:
             st = os.stat(fp)
+        except FileNotFoundError as e:
+            logg.warning('error {} persist {}'.format(e, self.persist))
+            if self.persist:
+                raise e
+
+        if st != None: 
             self.assertTrue(stat.S_ISDIR(st.st_mode))
             self.assertTrue(store.is_default)
 
         fpd = os.path.join(self.path, 'default')
+        try:
             st = os.stat(fpd)
+        except FileNotFoundError as e:
+            logg.warning('error {} persist {}'.format(e, self.persist))
+            if self.persist:
+                raise e
+        if st != None:
             self.assertTrue(stat.S_ISDIR(st.st_mode))
             self.assertTrue(store.is_default)
 
@@ -85,7 +110,13 @@ class TestStoreBase(unittest.TestCase):
         store.stop(bogus_item)
         store = self.store_factory('foo')
         fpf = os.path.join(self.path, 'foo')
+        try:
             st = os.stat(fpf)
+        except FileNotFoundError as e:
+            logg.warning('error {} persist {}'.format(e, self.persist))
+            if self.persist:
+                raise e
+        if st != None:
             self.assertTrue(stat.S_ISDIR(st.st_mode))
             self.assertFalse(store.is_default)
 
@@ -97,6 +128,8 @@ class TestStoreBase(unittest.TestCase):
         self.assertTrue(store.first)
 
         store.stop(bogus_item)
+
+        if self.persist:
             store = self.store_factory()
             store.start()
             self.assertFalse(store.first)
@@ -226,12 +259,16 @@ class TestStoreBase(unittest.TestCase):
         o.next(advance_block=True)
         session.stop(o)
 
+        if self.persist:
             store = self.store_factory('foo')
             store.start()
             o = store.get(2)
 
 
     def t_sync_history_interrupted(self):
+        if not self.persist:
+            return
+
         bogus_item = MockItem(0, 0, 0, 0)
         store = self.store_factory('foo')
         session = SyncSession(store)

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = chainsyncer
-version = 0.3.3
+version = 0.3.5
 description = Generic blockchain syncer driver
 author = Louis Holbrook
 author_email = dev@holbrook.no

tests/store/test_0_mem.py

@@ -0,0 +1,33 @@
+# standard imports
+import unittest
+import logging
+
+# external imports
+from shep import State
+
+# local imports
+from chainsyncer.store.mem import SyncMemStore
+from chainsyncer.unittest.store import TestStoreBase
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class StoreFactory:
+
+    def create(self, session_id=None):
+        return SyncMemStore(session_id=session_id)
+
+
+class TestMem(TestStoreBase):
+
+    def setUp(self):
+        super(TestMem, self).setUp()
+        self.store_factory = StoreFactory().create
+        self.persist = False
+
+
+if __name__ == '__main__':
+    TestStoreBase.link(TestMem)
+    # Remove tests that test persistence of state
+    unittest.main()

tests/store/test_rocksdb.py

@@ -4,7 +4,11 @@ import logging
 
 # local imports
 from chainsyncer.store.rocksdb import SyncRocksDbStore
-from chainsyncer.unittest.store import TestStoreBase
+from chainsyncer.unittest.store import (
+        TestStoreBase,
+        filter_change_callback,
+        state_change_callback,
+        )
 
 logging.basicConfig(level=logging.DEBUG)
 logg = logging.getLogger()
@@ -16,7 +20,7 @@ class StoreFactory:
 
 
     def create(self, session_id=None):
-        return SyncRocksDbStore(self.path, session_id=session_id)
+        return SyncRocksDbStore(self.path, session_id=session_id, state_event_callback=state_change_callback, filter_state_event_callback=filter_change_callback)
 
 
 class TestRocksDb(TestStoreBase):