Allow selective sync and or queue in settings

This commit is contained in:
lash 2022-04-26 19:41:29 +00:00
parent 7e78fd0da2
commit 18d10dc8b7
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
6 changed files with 40 additions and 113 deletions

View File

@ -1,3 +1,12 @@
# standard imports
import os
# local imports
from .base import *
from .arg import ArgumentParser
from .config import Config
from .arg import process_flags
from .config import process_config
__script_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = os.path.join(os.path.dirname(__script_dir), 'data')
config_dir = os.path.join(data_dir, 'config')

View File

@ -1,17 +1,11 @@
# external imports
from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
# local imports
from .base import SyncFlag, Flag
from .base import ChaindFlag
def process_flags(argparser, flags):
if flags & ChaindFlag.SESSION:
argparser.add_argument('--session-id', dest='session_id', type=str, help='Session to store state and data under')
argparser.add_argument('--runtime-dir', dest='runtime_dir', type=str, help='Directory to store volatile data')
argparser.add_argument('--data-dir', dest='data_dir', type=str, help='Directory to store persistent data')
class ArgumentParser(BaseArgumentParser):
def process_local_flags(self, local_arg_flags):
if local_arg_flags & SyncFlag.SESSION:
self.add_argument('--session-id', dest='session_id', type=str, help='Session to store state and data under')
self.add_argument('--runtime-dir', dest='runtime_dir', type=str, help='Directory to store volatile data')
self.add_argument('--data-dir', dest='data_dir', type=str, help='Directory to store persistent data')
if local_arg_flags & SyncFlag.SYNCER:
self.add_argument('--offset', type=int, help='Block to start sync from. Default is the latest block at first run.')
self.add_argument('--until', type=int, default=-1, help='Block to stop sync on. Default is do not stop.')
if flags & ChaindFlag.SOCKET:
argparser.add_argument('--socket', type=str, help='Socket path to send transactions to (assumes -s).')

View File

@ -1,22 +1,10 @@
# standard imports
import enum
# external imports
from chainlib.eth.cli import (
argflag_std_read,
argflag_std_write,
argflag_std_base,
Flag,
)
class SyncFlag(enum.IntEnum):
class ChaindFlag(enum.IntEnum):
SESSION = 1
SYNCER = 16
QUEUE = 256
DISPATCH = 512
SOCKET = 4096
argflag_local_sync = argflag_std_base | Flag.CHAIN_SPEC | SyncFlag.SYNCER | SyncFlag.SESSION
argflag_local_queue = SyncFlag.QUEUE | Flag.CHAIN_SPEC | SyncFlag.SOCKET | SyncFlag.SESSION
argflag_local_base = ChaindFlag.SESSION

View File

@ -1,46 +1,17 @@
# standard imports
import logging
import os
# external imports
from chainlib.cli import (
Config as BaseConfig,
Flag,
)
from chaind.cli import SyncFlag
script_dir = os.path.dirname(os.path.realpath(__file__))
logg = logging.getLogger(__name__)
from chaind.cli import ChaindFlag
class Config(BaseConfig):
def process_config(config, args, flags):
args_override = {}
if flags & ChaindFlag.SESSION:
args_override['SESSION_ID'] = getattr(args, 'session_id')
args_override['SESSION_RUNTIME_DIR'] = getattr(args, 'runtime_dir')
args_override['SESSION_DATA_DIR'] = getattr(args, 'data_dir')
local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
if flags & ChaindFlag.SOCKET:
args_override['SESSION_SOCKET_PATH'] = getattr(args, 'socket')
@classmethod
def from_args(cls, engine, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
expanded_base_config_dir = [cls.local_base_config_dir]
if base_config_dir != None:
if isinstance(base_config_dir, str):
base_config_dir = [base_config_dir]
for d in base_config_dir:
expanded_base_config_dir.append(d)
config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
local_args_override = {}
if local_arg_flags & SyncFlag.SESSION:
local_args_override['SESSION_ID'] = getattr(args, 'session_id')
local_args_override['SESSION_RUNTIME_DIR'] = getattr(args, 'runtime_dir')
local_args_override['SESSION_DATA_DIR'] = getattr(args, 'data_dir')
local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
local_args_override['SYNCER_LIMIT'] = getattr(args, 'until')
if local_arg_flags & SyncFlag.SOCKET:
local_args_override['SESSION_SOCKET_PATH'] = getattr(args, 'socket')
config.dict_override(local_args_override, 'local cli args')
config.add(engine, 'CHAIND_ENGINE', False)
config.dict_override(args_override, 'local cli args')
return config

View File

@ -3,7 +3,3 @@ socket_path =
runtime_dir =
id =
data_dir =
[syncer]
offset =
limit =

View File

@ -4,20 +4,18 @@ import os
# external imports
from chainlib.chain import ChainSpec
from chainlib.eth.block import block_latest
from hexathon import (
to_int as hex_to_int,
strip_0x,
)
from chainsyncer.settings import ChainsyncerSettings
logg = logging.getLogger(__name__)
class ChaindSettings:
class ChaindSettings(ChainsyncerSettings):
def __init__(self):
def __init__(self, include_sync=False, include_queue=False):
self.o = {}
self.get = self.o.get
self.include_sync = include_sync
self.include_queue = include_queue
def process_common(self, config):
@ -25,37 +23,7 @@ class ChaindSettings:
self.o['SOCKET_PATH'] = config.get('SESSION_SOCKET_PATH')
def process_sync_range(self, config):
o = block_latest()
r = self.o['RPC'].do(o)
block_offset = int(strip_0x(r), 16) + 1
logg.info('network block height at startup is {}'.format(block_offset))
keep_alive = False
session_block_offset = 0
block_limit = 0
session_block_offset = int(config.get('SYNCER_OFFSET'))
until = int(config.get('SYNCER_LIMIT'))
if until > 0:
if until <= session_block_offset:
raise ValueError('sync termination block number must be later than offset ({} >= {})'.format(session_block_offset, until))
block_limit = until
else:
keep_alive=True
block_limit = -1
if session_block_offset == -1:
session_block_offset = block_offset
elif not config.true('_KEEP_ALIVE'):
if block_limit == 0:
lock_limit = block_offset
self.o['SYNCER_OFFSET'] = session_block_offset
self.o['SYNCER_LIMIT'] = block_limit
def process_sync_session(self, config):
def process_session(self, config):
session_id = config.get('SESSION_ID')
base_dir = os.getcwd()
@ -109,13 +77,14 @@ class ChaindSettings:
def process_sync(self, config):
self.process_sync_interface(config)
self.process_sync_session(config)
self.process_sync_range(config)
def process(self, config):
self.process_common(config)
self.process_sync(config)
self.process_session(config)
if self.include_sync:
self.process_sync(config)
def __str__(self):