Allow memory-only syncing

This commit is contained in:
lash 2022-04-26 07:56:04 +00:00
parent 927913bd02
commit 044e85fb99
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
8 changed files with 147 additions and 30 deletions

View File

@ -1,3 +1,7 @@
* 0.3.5
- Allow memory-only shep if factory set to None in store constructor
* 0.3.4
- Use explicit bool check in filter interrupt check
* 0.3.3 * 0.3.3
- Include shep persistent state bootstrap sync - Include shep persistent state bootstrap sync
- Add chainsyncer extras - Add chainsyncer extras

View File

@ -4,6 +4,7 @@ import logging
# local imports # local imports
from shep.persist import PersistedState from shep.persist import PersistedState
from shep import State
from shep.error import StateInvalid from shep.error import StateInvalid
from chainsyncer.filter import FilterState from chainsyncer.filter import FilterState
from chainsyncer.error import ( from chainsyncer.error import (
@ -168,13 +169,20 @@ class SyncStore:
self.session_path = os.path.realpath(given_path) self.session_path = os.path.realpath(given_path)
def setup_sync_state(self, factory, event_callback): def setup_sync_state(self, factory=None, event_callback=None):
if factory == None:
self.state = State(2, event_callback=event_callback)
else:
self.state = PersistedState(factory.add, 2, event_callback=event_callback) self.state = PersistedState(factory.add, 2, event_callback=event_callback)
self.state.add('SYNC') self.state.add('SYNC')
self.state.add('DONE') self.state.add('DONE')
def setup_filter_state(self, factory, event_callback): def setup_filter_state(self, factory=None, event_callback=None):
if factory == None:
filter_state_backend = State(0, check_alias=False, event_callback=event_callback)
self.filter_state = FilterState(filter_state_backend)
else:
filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback) filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback)
self.filter_state = FilterState(filter_state_backend, scan=factory.ls) self.filter_state = FilterState(filter_state_backend, scan=factory.ls)
self.filters = [] self.filters = []
@ -202,7 +210,7 @@ class SyncStore:
if self.first: if self.first:
state_bytes = sync_state_serialize(offset, 0, target) state_bytes = sync_state_serialize(offset, 0, target)
block_number_str = str(offset) block_number_str = str(offset)
self.state.put(block_number_str, state_bytes) self.state.put(block_number_str, contents=state_bytes)
self.filter_state.put(block_number_str) self.filter_state.put(block_number_str)
o = SyncItem(offset, target, self.state, self.filter_state) o = SyncItem(offset, target, self.state, self.filter_state)
self.items[offset] = o self.items[offset] = o
@ -226,7 +234,7 @@ class SyncStore:
self.state.move(item.state_key, self.state.DONE) self.state.move(item.state_key, self.state.DONE)
state_bytes = sync_state_serialize(item.cursor, 0, -1) state_bytes = sync_state_serialize(item.cursor, 0, -1)
self.state.put(str(item.cursor), state_bytes) self.state.put(str(item.cursor), contents=state_bytes)
def load(self, target): def load(self, target):

View File

@ -7,10 +7,7 @@ import logging
from shep.store.file import SimpleFileStoreFactory from shep.store.file import SimpleFileStoreFactory
# local imports # local imports
from chainsyncer.store import ( from chainsyncer.store import SyncStore
SyncItem,
SyncStore,
)
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)

34
chainsyncer/store/mem.py Normal file
View File

@ -0,0 +1,34 @@
# standard imports
import logging
import os
# external imports
from shep import State
# local imports
from chainsyncer.store import SyncStore
logg = logging.getLogger(__name__)
class SyncMemStore(SyncStore):
def __init__(self, session_id=None, state_event_callback=None, filter_state_event_callback=None):
super(SyncMemStore, self).__init__('/dev/null', session_id=session_id)
self.session_id = os.path.basename(self.session_path)
logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path))
factory = None
self.setup_sync_state(factory, state_event_callback)
factory = None
self.setup_filter_state(factory, filter_state_event_callback)
def set_target(self, v):
self.target = int(v)
def get_target(self):
return self.target

View File

@ -19,7 +19,17 @@ from chainsyncer.unittest import (
MockItem, MockItem,
) )
logging.STATETRACE = 5
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)
logg.setLevel(logging.STATETRACE)
def state_change_callback(k, old_state, new_state):
logg.log(logging.STATETRACE, 'state change: {} {} -> {}'.format(k, old_state, new_state))
def filter_change_callback(k, old_state, new_state):
logg.log(logging.STATETRACE, 'filter change: {} {} -> {}'.format(k, old_state, new_state))
class TestStoreBase(unittest.TestCase): class TestStoreBase(unittest.TestCase):
@ -27,6 +37,7 @@ class TestStoreBase(unittest.TestCase):
def setUp(self): def setUp(self):
self.path = tempfile.mkdtemp() self.path = tempfile.mkdtemp()
self.store_factory = None self.store_factory = None
self.persist = True
@classmethod @classmethod
@ -58,12 +69,26 @@ class TestStoreBase(unittest.TestCase):
fp = os.path.join(self.path, store.session_id) fp = os.path.join(self.path, store.session_id)
session_id = store.session_id session_id = store.session_id
st = None
try:
st = os.stat(fp) st = os.stat(fp)
except FileNotFoundError as e:
logg.warning('error {} persist {}'.format(e, self.persist))
if self.persist:
raise e
if st != None:
self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertTrue(store.is_default) self.assertTrue(store.is_default)
fpd = os.path.join(self.path, 'default') fpd = os.path.join(self.path, 'default')
try:
st = os.stat(fpd) st = os.stat(fpd)
except FileNotFoundError as e:
logg.warning('error {} persist {}'.format(e, self.persist))
if self.persist:
raise e
if st != None:
self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertTrue(store.is_default) self.assertTrue(store.is_default)
@ -85,7 +110,13 @@ class TestStoreBase(unittest.TestCase):
store.stop(bogus_item) store.stop(bogus_item)
store = self.store_factory('foo') store = self.store_factory('foo')
fpf = os.path.join(self.path, 'foo') fpf = os.path.join(self.path, 'foo')
try:
st = os.stat(fpf) st = os.stat(fpf)
except FileNotFoundError as e:
logg.warning('error {} persist {}'.format(e, self.persist))
if self.persist:
raise e
if st != None:
self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertFalse(store.is_default) self.assertFalse(store.is_default)
@ -97,6 +128,8 @@ class TestStoreBase(unittest.TestCase):
self.assertTrue(store.first) self.assertTrue(store.first)
store.stop(bogus_item) store.stop(bogus_item)
if self.persist:
store = self.store_factory() store = self.store_factory()
store.start() store.start()
self.assertFalse(store.first) self.assertFalse(store.first)
@ -226,12 +259,16 @@ class TestStoreBase(unittest.TestCase):
o.next(advance_block=True) o.next(advance_block=True)
session.stop(o) session.stop(o)
if self.persist:
store = self.store_factory('foo') store = self.store_factory('foo')
store.start() store.start()
o = store.get(2) o = store.get(2)
def t_sync_history_interrupted(self): def t_sync_history_interrupted(self):
if not self.persist:
return
bogus_item = MockItem(0, 0, 0, 0) bogus_item = MockItem(0, 0, 0, 0)
store = self.store_factory('foo') store = self.store_factory('foo')
session = SyncSession(store) session = SyncSession(store)

View File

@ -1,6 +1,6 @@
[metadata] [metadata]
name = chainsyncer name = chainsyncer
version = 0.3.3 version = 0.3.5
description = Generic blockchain syncer driver description = Generic blockchain syncer driver
author = Louis Holbrook author = Louis Holbrook
author_email = dev@holbrook.no author_email = dev@holbrook.no

33
tests/store/test_0_mem.py Normal file
View File

@ -0,0 +1,33 @@
# standard imports
import unittest
import logging
# external imports
from shep import State
# local imports
from chainsyncer.store.mem import SyncMemStore
from chainsyncer.unittest.store import TestStoreBase
logging.basicConfig(level=logging.DEBUG)
logg = logging.getLogger()
class StoreFactory:
def create(self, session_id=None):
return SyncMemStore(session_id=session_id)
class TestMem(TestStoreBase):
def setUp(self):
super(TestMem, self).setUp()
self.store_factory = StoreFactory().create
self.persist = False
if __name__ == '__main__':
TestStoreBase.link(TestMem)
# Remove tests that test persistence of state
unittest.main()

View File

@ -4,7 +4,11 @@ import logging
# local imports # local imports
from chainsyncer.store.rocksdb import SyncRocksDbStore from chainsyncer.store.rocksdb import SyncRocksDbStore
from chainsyncer.unittest.store import TestStoreBase from chainsyncer.unittest.store import (
TestStoreBase,
filter_change_callback,
state_change_callback,
)
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
logg = logging.getLogger() logg = logging.getLogger()
@ -16,7 +20,7 @@ class StoreFactory:
def create(self, session_id=None): def create(self, session_id=None):
return SyncRocksDbStore(self.path, session_id=session_id) return SyncRocksDbStore(self.path, session_id=session_id, state_event_callback=state_change_callback, filter_state_event_callback=filter_change_callback)
class TestRocksDb(TestStoreBase): class TestRocksDb(TestStoreBase):