Rename module

This commit is contained in:
nolash 2021-02-11 09:02:17 +01:00
parent 88df1418b0
commit 00725ebc7c
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
14 changed files with 52 additions and 74 deletions

View File

@ -1,9 +1,10 @@
# standard imports # standard imports
import logging import logging
import uuid
# local imports # local imports
from cic_syncer.db.models.sync import BlockchainSync from chainsyncer.db.models.sync import BlockchainSync
from cic_syncer.db.models.base import SessionBase from chainsyncer.db.models.base import SessionBase
logg = logging.getLogger() logg = logging.getLogger()
@ -194,3 +195,22 @@ class SyncerBackend:
session.close() session.close()
return SyncerBackend(chain, object_id) return SyncerBackend(chain, object_id)
class MemBackend:
def __init__(self, chain_spec, object_id):
self.object_id = object_id
self.chain_spec = chain_spec
self.block_height = 0
self.tx_height = 0
def set(self, block_height, tx_height):
logg.debug('stateless backend received {} {}'.format(block_height, tx_height))
self.block_height = block_height
self.tx_height = tx_height
def get(self):
return (self.block_height, self.tx_height)

View File

@ -1,8 +1,8 @@
import json import json
from cic_syncer.client import translate from chainsyncer.client import translate
from cic_syncer.client.block import Block from chainsyncer.client.block import Block
from cic_syncer.client.tx import Tx from chainsyncer.client.tx import Tx
translations = { translations = {

View File

@ -9,8 +9,8 @@ from hexathon import add_0x
# local imports # local imports
from .response import EVMResponse from .response import EVMResponse
from cic_syncer.error import RequestError from chainsyncer.error import RequestError
from cic_syncer.client.evm.response import EVMBlock from chainsyncer.client.evm.response import EVMBlock
logg = logging.getLogger() logg = logging.getLogger()
@ -44,7 +44,7 @@ class EVMWebsocketClient:
return res.get_result() return res.get_result()
def get_block_by_integer(self, n): def block_by_integer(self, n):
req_id = str(uuid.uuid4()) req_id = str(uuid.uuid4())
nhx = '0x' + n.to_bytes(8, 'big').hex() nhx = '0x' + n.to_bytes(8, 'big').hex()
req = { req = {
@ -67,7 +67,7 @@ class EVMWebsocketClient:
return EVMBlock(o['hash'], o) return EVMBlock(o['hash'], o)
def get_block_by_hash(self, hx_in): def block_by_hash(self, hx_in):
req_id = str(uuid.uuid4()) req_id = str(uuid.uuid4())
hx = add_0x(hx_in) hx = add_0x(hx_in)
req ={ req ={

View File

@ -3,7 +3,7 @@ import os
import logging import logging
# local imports # local imports
from cic_syncer.db.models.base import SessionBase from chainsyncer.db.models.base import SessionBase
logg = logging.getLogger() logg = logging.getLogger()

View File

@ -10,12 +10,11 @@ class Syncer:
running_global = True running_global = True
def __init__(self, backend, handler): def __init__(self, backend):
self.cursor = None self.cursor = None
self.running = True self.running = True
self.backend = backend self.backend = backend
self.filter = [] self.filter = []
self.handler = handler
def chain(self): def chain(self):
@ -27,11 +26,14 @@ class Syncer:
return self.bc_cache.chain() return self.bc_cache.chain()
def add_filter(self, f):
self.filter.append(f)
class MinedSyncer(Syncer): class MinedSyncer(Syncer):
def __init__(self, backend, handler): def __init__(self, backend):
super(MinedSyncer, self).__init__(backend, handler) super(MinedSyncer, self).__init__(backend)
def loop(self, interval, getter): def loop(self, interval, getter):
@ -46,13 +48,13 @@ class MinedSyncer(Syncer):
class HeadSyncer(MinedSyncer): class HeadSyncer(MinedSyncer):
def __init__(self, backend, handler): def __init__(self, backend):
super(HeadSyncer, self).__init__(backend, handler) super(HeadSyncer, self).__init__(backend)
def process(self, getter, block): def process(self, getter, block):
logg.debug('process {}'.format(block)) logg.debug('process {}'.format(block))
block = getter.get_block_by_hash(block.hash) block = getter.block_by_hash(block.hash)
i = 0 i = 0
tx = None tx = None
while True: while True:
@ -62,7 +64,7 @@ class HeadSyncer(MinedSyncer):
logg.debug('tx {}'.format(tx)) logg.debug('tx {}'.format(tx))
self.backend.set(block.number(), i) self.backend.set(block.number(), i)
for f in self.filter: for f in self.filter:
f.handle(getter, block, tx) f(getter, block, tx)
except IndexError as e: except IndexError as e:
self.backend.set(block.number() + 1, 0) self.backend.set(block.number() + 1, 0)
break break
@ -73,7 +75,7 @@ class HeadSyncer(MinedSyncer):
(block_number, tx_number) = self.backend.get() (block_number, tx_number) = self.backend.get()
block_hash = [] block_hash = []
uu = uuid.uuid4() uu = uuid.uuid4()
res = getter.get_block_by_integer(block_number) res = getter.block_by_integer(block_number)
logg.debug('get {}'.format(res)) logg.debug('get {}'.format(res))
return res return res

View File

@ -9,12 +9,12 @@ import re
# third-party imports # third-party imports
import confini import confini
from cic_syncer.driver import HeadSyncer from chainlib.eth.connection import HTTPConnection
from cic_syncer.db import dsn_from_config from chainsyncer.driver import HeadSyncer
from cic_syncer.db.models.base import SessionBase from chainsyncer.db import dsn_from_config
from cic_syncer.client.evm.websocket import EVMWebsocketClient from chainsyncer.db.models.base import SessionBase
from cic_syncer.backend import SyncerBackend from chainsyncer.backend import SyncerBackend
from cic_syncer.error import LoopDone from chainsyncer.error import LoopDone
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
@ -55,7 +55,7 @@ config = confini.Config(config_dir, args.env_prefix)
config.process() config.process()
# override args # override args
args_override = { args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'), 'CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'), 'ETH_PROVIDER': getattr(args, 'p'),
} }
config.dict_override(args_override, 'cli flag') config.dict_override(args_override, 'cli flag')
@ -70,58 +70,15 @@ queue = args.q
dsn = dsn_from_config(config) dsn = dsn_from_config(config)
SessionBase.connect(dsn) SessionBase.connect(dsn)
c = HTTPConnection(config.get('ETH_PROVIDER'))
transfer_callbacks = [] chain = config.get('CHAIN_SPEC')
for cb in config.get('TASKS_SYNCER_CALLBACKS', '').split(','):
task_split = cb.split(':')
task_queue = queue
if len(task_split) > 1:
task_queue = task_split[0]
task_pair = (task_split[1], task_queue)
transfer_callbacks.append(task_pair)
def tx_filter(w3, tx, rcpt, chain_spec):
tx_hash_hex = tx.hash.hex()
otx = Otx.load(tx_hash_hex)
if otx == None:
logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
return None
logg.info('otx found {}'.format(otx.tx_hash))
s = celery.signature(
'cic_eth.queue.tx.set_final_status',
[
tx_hash_hex,
rcpt.blockNumber,
rcpt.status == 0,
],
queue=queue,
)
t = s.apply_async()
return t
re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
c = EVMWebsocketClient(config.get('ETH_PROVIDER'))
chain = config.get('CIC_CHAIN_SPEC')
def main(): def main():
block_offset = c.block_number() block_offset = c.block_number()
#syncer_backend = SyncerBackend.live(chain, block_offset+1)
syncer_backend = SyncerBackend.live(chain, 0) syncer_backend = SyncerBackend.live(chain, 0)
syncer = HeadSyncer(syncer_backend, handler) syncer = HeadSyncer(syncer_backend)
for cb in config.get('TASKS_SYNCER_CALLBACKS', '').split(','):
task_split = cb.split(':')
task_queue = queue
if len(task_split) > 1:
task_queue = task_split[0]
task_pair = (task_split[1], task_queue)
h = Handler(task_pair[0], task_pair[1])
syncer.filter.append(h)
try: try:
logg.debug('block offset {} {}'.format(block_offset, c)) logg.debug('block offset {} {}'.format(block_offset, c))

View File

@ -1,9 +1,8 @@
websocket-client==0.57.0
psycopg2==2.8.6 psycopg2==2.8.6
SQLAlchemy==1.3.20 SQLAlchemy==1.3.20
py-evm==0.3.0a20 py-evm==0.3.0a20
eth-tester==0.5.0b3 eth-tester==0.5.0b3
web3==5.12.2
confini==0.3.6b2 confini==0.3.6b2
semver==2.13.0 semver==2.13.0
hexathon==0.0.1a2 hexathon==0.0.1a2
chainlib=0.0.1a4