Allow memory-only syncing

This commit is contained in:
lash 2022-04-26 07:56:04 +00:00
parent 927913bd02
commit 044e85fb99
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
8 changed files with 147 additions and 30 deletions

View File

@ -1,3 +1,7 @@
* 0.3.5
- Allow memory-only shep if factory set to None in store constructor
* 0.3.4
- Use explicit bool check in filter interrupt check
* 0.3.3
- Include shep persistent state bootstrap sync
- Add chainsyncer extras

View File

@ -4,6 +4,7 @@ import logging
# local imports
from shep.persist import PersistedState
from shep import State
from shep.error import StateInvalid
from chainsyncer.filter import FilterState
from chainsyncer.error import (
@ -168,15 +169,22 @@ class SyncStore:
self.session_path = os.path.realpath(given_path)
def setup_sync_state(self, factory, event_callback):
self.state = PersistedState(factory.add, 2, event_callback=event_callback)
def setup_sync_state(self, factory=None, event_callback=None):
if factory == None:
self.state = State(2, event_callback=event_callback)
else:
self.state = PersistedState(factory.add, 2, event_callback=event_callback)
self.state.add('SYNC')
self.state.add('DONE')
def setup_filter_state(self, factory, event_callback):
filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback)
self.filter_state = FilterState(filter_state_backend, scan=factory.ls)
def setup_filter_state(self, factory=None, event_callback=None):
if factory == None:
filter_state_backend = State(0, check_alias=False, event_callback=event_callback)
self.filter_state = FilterState(filter_state_backend)
else:
filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback)
self.filter_state = FilterState(filter_state_backend, scan=factory.ls)
self.filters = []
@ -202,7 +210,7 @@ class SyncStore:
if self.first:
state_bytes = sync_state_serialize(offset, 0, target)
block_number_str = str(offset)
self.state.put(block_number_str, state_bytes)
self.state.put(block_number_str, contents=state_bytes)
self.filter_state.put(block_number_str)
o = SyncItem(offset, target, self.state, self.filter_state)
self.items[offset] = o
@ -226,7 +234,7 @@ class SyncStore:
self.state.move(item.state_key, self.state.DONE)
state_bytes = sync_state_serialize(item.cursor, 0, -1)
self.state.put(str(item.cursor), state_bytes)
self.state.put(str(item.cursor), contents=state_bytes)
def load(self, target):

View File

@ -7,10 +7,7 @@ import logging
from shep.store.file import SimpleFileStoreFactory
# local imports
from chainsyncer.store import (
SyncItem,
SyncStore,
)
from chainsyncer.store import SyncStore
logg = logging.getLogger(__name__)

34
chainsyncer/store/mem.py Normal file
View File

@ -0,0 +1,34 @@
# standard imports
import logging
import os
# external imports
from shep import State
# local imports
from chainsyncer.store import SyncStore
logg = logging.getLogger(__name__)
class SyncMemStore(SyncStore):
def __init__(self, session_id=None, state_event_callback=None, filter_state_event_callback=None):
super(SyncMemStore, self).__init__('/dev/null', session_id=session_id)
self.session_id = os.path.basename(self.session_path)
logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path))
factory = None
self.setup_sync_state(factory, state_event_callback)
factory = None
self.setup_filter_state(factory, filter_state_event_callback)
def set_target(self, v):
self.target = int(v)
def get_target(self):
return self.target

View File

@ -19,7 +19,17 @@ from chainsyncer.unittest import (
MockItem,
)
logging.STATETRACE = 5
logg = logging.getLogger(__name__)
logg.setLevel(logging.STATETRACE)
def state_change_callback(k, old_state, new_state):
logg.log(logging.STATETRACE, 'state change: {} {} -> {}'.format(k, old_state, new_state))
def filter_change_callback(k, old_state, new_state):
logg.log(logging.STATETRACE, 'filter change: {} {} -> {}'.format(k, old_state, new_state))
class TestStoreBase(unittest.TestCase):
@ -27,6 +37,7 @@ class TestStoreBase(unittest.TestCase):
def setUp(self):
self.path = tempfile.mkdtemp()
self.store_factory = None
self.persist = True
@classmethod
@ -58,14 +69,28 @@ class TestStoreBase(unittest.TestCase):
fp = os.path.join(self.path, store.session_id)
session_id = store.session_id
st = os.stat(fp)
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertTrue(store.is_default)
st = None
try:
st = os.stat(fp)
except FileNotFoundError as e:
logg.warning('error {} persist {}'.format(e, self.persist))
if self.persist:
raise e
if st != None:
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertTrue(store.is_default)
fpd = os.path.join(self.path, 'default')
st = os.stat(fpd)
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertTrue(store.is_default)
try:
st = os.stat(fpd)
except FileNotFoundError as e:
logg.warning('error {} persist {}'.format(e, self.persist))
if self.persist:
raise e
if st != None:
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertTrue(store.is_default)
fpd = os.path.realpath(fpd)
self.assertEqual(fpd, fp)
@ -85,9 +110,15 @@ class TestStoreBase(unittest.TestCase):
store.stop(bogus_item)
store = self.store_factory('foo')
fpf = os.path.join(self.path, 'foo')
st = os.stat(fpf)
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertFalse(store.is_default)
try:
st = os.stat(fpf)
except FileNotFoundError as e:
logg.warning('error {} persist {}'.format(e, self.persist))
if self.persist:
raise e
if st != None:
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertFalse(store.is_default)
def t_store_start(self):
@ -97,9 +128,11 @@ class TestStoreBase(unittest.TestCase):
self.assertTrue(store.first)
store.stop(bogus_item)
store = self.store_factory()
store.start()
self.assertFalse(store.first)
if self.persist:
store = self.store_factory()
store.start()
self.assertFalse(store.first)
def t_store_resume(self):
@ -226,12 +259,16 @@ class TestStoreBase(unittest.TestCase):
o.next(advance_block=True)
session.stop(o)
store = self.store_factory('foo')
store.start()
o = store.get(2)
if self.persist:
store = self.store_factory('foo')
store.start()
o = store.get(2)
def t_sync_history_interrupted(self):
if not self.persist:
return
bogus_item = MockItem(0, 0, 0, 0)
store = self.store_factory('foo')
session = SyncSession(store)

View File

@ -1,6 +1,6 @@
[metadata]
name = chainsyncer
version = 0.3.3
version = 0.3.5
description = Generic blockchain syncer driver
author = Louis Holbrook
author_email = dev@holbrook.no

33
tests/store/test_0_mem.py Normal file
View File

@ -0,0 +1,33 @@
# standard imports
import unittest
import logging
# external imports
from shep import State
# local imports
from chainsyncer.store.mem import SyncMemStore
from chainsyncer.unittest.store import TestStoreBase
logging.basicConfig(level=logging.DEBUG)
logg = logging.getLogger()
class StoreFactory:
def create(self, session_id=None):
return SyncMemStore(session_id=session_id)
class TestMem(TestStoreBase):
def setUp(self):
super(TestMem, self).setUp()
self.store_factory = StoreFactory().create
self.persist = False
if __name__ == '__main__':
TestStoreBase.link(TestMem)
# Remove tests that test persistence of state
unittest.main()

View File

@ -4,7 +4,11 @@ import logging
# local imports
from chainsyncer.store.rocksdb import SyncRocksDbStore
from chainsyncer.unittest.store import TestStoreBase
from chainsyncer.unittest.store import (
TestStoreBase,
filter_change_callback,
state_change_callback,
)
logging.basicConfig(level=logging.DEBUG)
logg = logging.getLogger()
@ -16,7 +20,7 @@ class StoreFactory:
def create(self, session_id=None):
return SyncRocksDbStore(self.path, session_id=session_id)
return SyncRocksDbStore(self.path, session_id=session_id, state_event_callback=state_change_callback, filter_state_event_callback=filter_change_callback)
class TestRocksDb(TestStoreBase):