2021-02-03 20:55:39 +01:00
|
|
|
# standard imports
|
|
|
|
import logging
|
2021-02-11 09:02:17 +01:00
|
|
|
import uuid
|
2021-02-03 20:55:39 +01:00
|
|
|
|
2021-04-15 15:06:07 +02:00
|
|
|
# imports
|
2021-02-17 12:44:35 +01:00
|
|
|
from chainlib.chain import ChainSpec
|
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
# local imports
|
2021-02-11 09:02:17 +01:00
|
|
|
from chainsyncer.db.models.sync import BlockchainSync
|
2021-04-04 15:03:58 +02:00
|
|
|
from chainsyncer.db.models.filter import BlockchainSyncFilter
|
2021-02-11 09:02:17 +01:00
|
|
|
from chainsyncer.db.models.base import SessionBase
|
2021-04-15 17:16:31 +02:00
|
|
|
from .base import Backend
|
2021-02-03 20:55:39 +01:00
|
|
|
|
2021-04-10 00:30:08 +02:00
|
|
|
logg = logging.getLogger().getChild(__name__)
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
2021-04-15 17:45:35 +02:00
|
|
|
class SQLBackend(Backend):
|
2021-02-03 20:55:39 +01:00
|
|
|
"""Interface to block and transaction sync state.
|
|
|
|
|
|
|
|
:param chain_spec: Chain spec for the chain that syncer is running for.
|
|
|
|
:type chain_spec: cic_registry.chain.ChainSpec
|
|
|
|
:param object_id: Unique id for the syncer session.
|
|
|
|
:type object_id: number
|
|
|
|
"""
|
2021-06-03 13:54:49 +02:00
|
|
|
|
|
|
|
base = None
|
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
def __init__(self, chain_spec, object_id):
|
2021-04-15 17:45:35 +02:00
|
|
|
super(SQLBackend, self).__init__()
|
2021-02-03 20:55:39 +01:00
|
|
|
self.db_session = None
|
|
|
|
self.db_object = None
|
2021-04-04 15:03:58 +02:00
|
|
|
self.db_object_filter = None
|
2021-02-03 20:55:39 +01:00
|
|
|
self.chain_spec = chain_spec
|
|
|
|
self.object_id = object_id
|
|
|
|
self.connect()
|
|
|
|
self.disconnect()
|
|
|
|
|
|
|
|
|
2021-06-03 13:54:49 +02:00
|
|
|
@classmethod
|
|
|
|
def setup(cls, dsn, debug=False, *args, **kwargs):
|
|
|
|
if cls.base == None:
|
|
|
|
cls.base = SessionBase
|
|
|
|
cls.base.connect(dsn, debug=debug, pool_size=kwargs.get('pool_size', 0))
|
|
|
|
|
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
def connect(self):
|
|
|
|
"""Loads the state of the syncer session with the given id.
|
|
|
|
"""
|
2021-02-18 23:55:49 +01:00
|
|
|
if self.db_session == None:
|
|
|
|
self.db_session = SessionBase.create_session()
|
2021-04-04 15:03:58 +02:00
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
q = self.db_session.query(BlockchainSync)
|
|
|
|
q = q.filter(BlockchainSync.id==self.object_id)
|
|
|
|
self.db_object = q.first()
|
2021-04-04 15:03:58 +02:00
|
|
|
|
|
|
|
if self.db_object != None:
|
|
|
|
qtwo = self.db_session.query(BlockchainSyncFilter)
|
|
|
|
qtwo = qtwo.join(BlockchainSync)
|
|
|
|
qtwo = qtwo.filter(BlockchainSync.id==self.db_object.id)
|
|
|
|
self.db_object_filter = qtwo.first()
|
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
if self.db_object == None:
|
|
|
|
raise ValueError('sync entry with id {} not found'.format(self.object_id))
|
|
|
|
|
2021-04-04 15:03:58 +02:00
|
|
|
return self.db_session
|
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
def disconnect(self):
|
|
|
|
"""Commits state of sync to backend.
|
|
|
|
"""
|
2021-04-15 15:06:07 +02:00
|
|
|
if self.db_session == None:
|
|
|
|
return
|
|
|
|
|
2021-04-04 15:03:58 +02:00
|
|
|
if self.db_object_filter != None:
|
|
|
|
self.db_session.add(self.db_object_filter)
|
2021-02-03 20:55:39 +01:00
|
|
|
self.db_session.add(self.db_object)
|
|
|
|
self.db_session.commit()
|
|
|
|
self.db_session.close()
|
2021-02-18 23:55:49 +01:00
|
|
|
self.db_session = None
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get(self):
|
|
|
|
"""Get the current state of the syncer cursor.
|
|
|
|
|
|
|
|
:returns: Block and block transaction height, respectively
|
|
|
|
:rtype: tuple
|
|
|
|
"""
|
|
|
|
self.connect()
|
|
|
|
pair = self.db_object.cursor()
|
2021-04-04 15:03:58 +02:00
|
|
|
(filter_state, count, digest) = self.db_object_filter.cursor()
|
2021-02-03 20:55:39 +01:00
|
|
|
self.disconnect()
|
2021-04-04 15:03:58 +02:00
|
|
|
return (pair, filter_state,)
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
|
|
|
def set(self, block_height, tx_height):
|
|
|
|
"""Update the state of the syncer cursor
|
|
|
|
:param block_height: Block height of cursor
|
|
|
|
:type block_height: number
|
|
|
|
:param tx_height: Block transaction height of cursor
|
|
|
|
:type tx_height: number
|
|
|
|
:returns: Block and block transaction height, respectively
|
|
|
|
:rtype: tuple
|
|
|
|
"""
|
|
|
|
self.connect()
|
|
|
|
pair = self.db_object.set(block_height, tx_height)
|
2021-04-04 15:03:58 +02:00
|
|
|
(filter_state, count, digest)= self.db_object_filter.cursor()
|
2021-02-03 20:55:39 +01:00
|
|
|
self.disconnect()
|
2021-04-04 15:03:58 +02:00
|
|
|
return (pair, filter_state,)
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
|
|
|
def start(self):
|
|
|
|
"""Get the initial state of the syncer cursor.
|
|
|
|
|
|
|
|
:returns: Initial block and block transaction height, respectively
|
|
|
|
:rtype: tuple
|
|
|
|
"""
|
|
|
|
self.connect()
|
|
|
|
pair = self.db_object.start()
|
2021-04-04 15:03:58 +02:00
|
|
|
(filter_state, count, digest) = self.db_object_filter.start()
|
2021-02-03 20:55:39 +01:00
|
|
|
self.disconnect()
|
2021-04-04 15:03:58 +02:00
|
|
|
return (pair, filter_state,)
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
|
|
|
def target(self):
|
|
|
|
"""Get the target state (upper bound of sync) of the syncer cursor.
|
|
|
|
|
|
|
|
:returns: Target block height
|
|
|
|
:rtype: number
|
|
|
|
"""
|
|
|
|
self.connect()
|
|
|
|
target = self.db_object.target()
|
2021-04-04 15:03:58 +02:00
|
|
|
(filter_target, count, digest) = self.db_object_filter.target()
|
2021-02-03 20:55:39 +01:00
|
|
|
self.disconnect()
|
2021-04-04 15:03:58 +02:00
|
|
|
return (target, filter_target,)
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
2021-04-04 15:03:58 +02:00
|
|
|
def first(chain_spec):
|
2021-02-03 20:55:39 +01:00
|
|
|
"""Returns the model object of the most recent syncer in backend.
|
|
|
|
|
|
|
|
:param chain: Chain spec of chain that syncer is running for.
|
|
|
|
:type chain: cic_registry.chain.ChainSpec
|
|
|
|
:returns: Last syncer object
|
|
|
|
:rtype: cic_eth.db.models.BlockchainSync
|
|
|
|
"""
|
2021-04-04 15:03:58 +02:00
|
|
|
#return BlockchainSync.first(str(chain_spec))
|
|
|
|
object_id = BlockchainSync.first(str(chain_spec))
|
|
|
|
if object_id == None:
|
|
|
|
return None
|
2021-04-15 17:45:35 +02:00
|
|
|
return SQLBackend(chain_spec, object_id)
|
2021-04-04 15:03:58 +02:00
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
2021-04-04 15:03:58 +02:00
|
|
|
def initial(chain_spec, target_block_height, start_block_height=0):
|
2021-02-03 20:55:39 +01:00
|
|
|
"""Creates a new syncer session and commit its initial state to backend.
|
|
|
|
|
|
|
|
:param chain: Chain spec of chain that syncer is running for.
|
|
|
|
:type chain: cic_registry.chain.ChainSpec
|
|
|
|
:param block_height: Target block height
|
|
|
|
:type block_height: number
|
|
|
|
:returns: New syncer object
|
|
|
|
:rtype: cic_eth.db.models.BlockchainSync
|
|
|
|
"""
|
2021-04-04 15:03:58 +02:00
|
|
|
if start_block_height >= target_block_height:
|
|
|
|
raise ValueError('start block height must be lower than target block height')
|
2021-02-03 20:55:39 +01:00
|
|
|
object_id = None
|
|
|
|
session = SessionBase.create_session()
|
2021-04-04 15:03:58 +02:00
|
|
|
o = BlockchainSync(str(chain_spec), start_block_height, 0, target_block_height)
|
2021-02-03 20:55:39 +01:00
|
|
|
session.add(o)
|
|
|
|
session.commit()
|
|
|
|
object_id = o.id
|
2021-04-04 15:03:58 +02:00
|
|
|
|
|
|
|
of = BlockchainSyncFilter(o)
|
|
|
|
session.add(of)
|
|
|
|
session.commit()
|
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
session.close()
|
|
|
|
|
2021-04-15 17:45:35 +02:00
|
|
|
return SQLBackend(chain_spec, object_id)
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
2021-04-04 15:03:58 +02:00
|
|
|
def resume(chain_spec, block_height):
|
2021-02-03 20:55:39 +01:00
|
|
|
"""Retrieves and returns all previously unfinished syncer sessions.
|
|
|
|
|
|
|
|
|
2021-04-04 15:03:58 +02:00
|
|
|
:param chain_spec: Chain spec of chain that syncer is running for.
|
|
|
|
:type chain_spec: cic_registry.chain.ChainSpec
|
2021-02-03 20:55:39 +01:00
|
|
|
:param block_height: Target block height
|
|
|
|
:type block_height: number
|
|
|
|
:returns: Syncer objects of unfinished syncs
|
|
|
|
:rtype: list of cic_eth.db.models.BlockchainSync
|
|
|
|
"""
|
|
|
|
syncers = []
|
|
|
|
|
|
|
|
session = SessionBase.create_session()
|
|
|
|
|
|
|
|
object_id = None
|
|
|
|
|
2021-04-04 15:03:58 +02:00
|
|
|
highest_unsynced_block = 0
|
|
|
|
highest_unsynced_tx = 0
|
|
|
|
object_id = BlockchainSync.get_last(session=session, live=False)
|
|
|
|
if object_id != None:
|
|
|
|
q = session.query(BlockchainSync)
|
|
|
|
o = q.get(object_id)
|
|
|
|
(highest_unsynced_block, highest_unsynced_index) = o.cursor()
|
|
|
|
|
|
|
|
object_ids = BlockchainSync.get_unsynced(session=session)
|
|
|
|
session.close()
|
|
|
|
|
|
|
|
for object_id in object_ids:
|
2021-04-15 17:45:35 +02:00
|
|
|
s = SQLBackend(chain_spec, object_id)
|
2021-04-04 15:03:58 +02:00
|
|
|
logg.debug('resume unfinished {}'.format(s))
|
|
|
|
syncers.append(s)
|
|
|
|
|
|
|
|
session = SessionBase.create_session()
|
|
|
|
|
|
|
|
last_live_id = BlockchainSync.get_last(session=session)
|
|
|
|
if last_live_id != None:
|
|
|
|
|
|
|
|
q = session.query(BlockchainSync)
|
|
|
|
o = q.get(last_live_id)
|
|
|
|
|
|
|
|
(block_resume, tx_resume) = o.cursor()
|
|
|
|
session.flush()
|
|
|
|
|
|
|
|
#if block_height != block_resume:
|
|
|
|
if highest_unsynced_block < block_resume:
|
|
|
|
|
|
|
|
q = session.query(BlockchainSyncFilter)
|
|
|
|
q = q.filter(BlockchainSyncFilter.chain_sync_id==last_live_id)
|
|
|
|
of = q.first()
|
|
|
|
(flags, count, digest) = of.cursor()
|
|
|
|
|
|
|
|
session.flush()
|
|
|
|
|
|
|
|
o = BlockchainSync(str(chain_spec), block_resume, tx_resume, block_height)
|
|
|
|
session.add(o)
|
|
|
|
session.flush()
|
|
|
|
object_id = o.id
|
|
|
|
|
|
|
|
of = BlockchainSyncFilter(o, count, flags, digest)
|
|
|
|
session.add(of)
|
|
|
|
session.commit()
|
2021-02-03 20:55:39 +01:00
|
|
|
|
2021-04-15 17:45:35 +02:00
|
|
|
backend = SQLBackend(chain_spec, object_id)
|
2021-04-04 15:03:58 +02:00
|
|
|
syncers.append(backend)
|
|
|
|
|
|
|
|
logg.debug('last live session resume {}'.format(backend))
|
2021-02-03 20:55:39 +01:00
|
|
|
|
|
|
|
session.close()
|
|
|
|
|
|
|
|
return syncers
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
2021-02-17 12:44:35 +01:00
|
|
|
def live(chain_spec, block_height):
|
2021-02-03 20:55:39 +01:00
|
|
|
"""Creates a new open-ended syncer session starting at the given block height.
|
|
|
|
|
|
|
|
:param chain: Chain spec of chain that syncer is running for.
|
|
|
|
:type chain: cic_registry.chain.ChainSpec
|
|
|
|
:param block_height: Target block height
|
|
|
|
:type block_height: number
|
|
|
|
:returns: "Live" syncer object
|
|
|
|
:rtype: cic_eth.db.models.BlockchainSync
|
|
|
|
"""
|
|
|
|
session = SessionBase.create_session()
|
2021-04-04 15:03:58 +02:00
|
|
|
|
2021-02-17 12:44:35 +01:00
|
|
|
o = BlockchainSync(str(chain_spec), block_height, 0, None)
|
2021-02-03 20:55:39 +01:00
|
|
|
session.add(o)
|
2021-04-04 15:03:58 +02:00
|
|
|
session.flush()
|
2021-02-03 20:55:39 +01:00
|
|
|
object_id = o.id
|
2021-04-04 15:03:58 +02:00
|
|
|
|
|
|
|
of = BlockchainSyncFilter(o)
|
|
|
|
session.add(of)
|
|
|
|
session.commit()
|
|
|
|
|
2021-02-03 20:55:39 +01:00
|
|
|
session.close()
|
|
|
|
|
2021-04-15 17:45:35 +02:00
|
|
|
return SQLBackend(chain_spec, object_id)
|
2021-02-11 09:02:17 +01:00
|
|
|
|
|
|
|
|
2021-04-04 15:03:58 +02:00
|
|
|
def register_filter(self, name):
|
|
|
|
self.connect()
|
|
|
|
if self.db_object_filter == None:
|
|
|
|
self.db_object_filter = BlockchainSyncFilter(self.db_object)
|
|
|
|
self.db_object_filter.add(name)
|
|
|
|
self.db_session.add(self.db_object_filter)
|
|
|
|
self.disconnect()
|
|
|
|
|
|
|
|
|
|
|
|
def complete_filter(self, n):
|
2021-04-10 00:30:08 +02:00
|
|
|
self.connect()
|
2021-04-04 15:03:58 +02:00
|
|
|
self.db_object_filter.set(n)
|
2021-04-10 00:30:08 +02:00
|
|
|
self.db_session.add(self.db_object_filter)
|
|
|
|
self.db_session.commit()
|
|
|
|
self.disconnect()
|
2021-04-04 15:03:58 +02:00
|
|
|
|
|
|
|
|
2021-04-15 15:06:07 +02:00
|
|
|
def reset_filter(self):
|
|
|
|
self.connect()
|
|
|
|
self.db_object_filter.clear()
|
|
|
|
self.disconnect()
|
|
|
|
|
|
|
|
|