diff --git a/cic_syncer/backend.py b/chainsyncer/backend.py similarity index 90% rename from cic_syncer/backend.py rename to chainsyncer/backend.py index 8a8a564..d265ab7 100644 --- a/cic_syncer/backend.py +++ b/chainsyncer/backend.py @@ -1,9 +1,10 @@ # standard imports import logging +import uuid # local imports -from cic_syncer.db.models.sync import BlockchainSync -from cic_syncer.db.models.base import SessionBase +from chainsyncer.db.models.sync import BlockchainSync +from chainsyncer.db.models.base import SessionBase logg = logging.getLogger() @@ -194,3 +195,22 @@ class SyncerBackend: session.close() return SyncerBackend(chain, object_id) + + +class MemBackend: + + def __init__(self, chain_spec, object_id): + self.object_id = object_id + self.chain_spec = chain_spec + self.block_height = 0 + self.tx_height = 0 + + + def set(self, block_height, tx_height): + logg.debug('stateless backend received {} {}'.format(block_height, tx_height)) + self.block_height = block_height + self.tx_height = tx_height + + + def get(self): + return (self.block_height, self.tx_height) diff --git a/cic_syncer/client/__init__.py b/chainsyncer/client/__init__.py similarity index 100% rename from cic_syncer/client/__init__.py rename to chainsyncer/client/__init__.py diff --git a/cic_syncer/client/block.py b/chainsyncer/client/block.py similarity index 100% rename from cic_syncer/client/block.py rename to chainsyncer/client/block.py diff --git a/cic_syncer/client/evm/response.py b/chainsyncer/client/evm/response.py similarity index 89% rename from cic_syncer/client/evm/response.py rename to chainsyncer/client/evm/response.py index 9d4f17a..bac12f2 100644 --- a/cic_syncer/client/evm/response.py +++ b/chainsyncer/client/evm/response.py @@ -1,8 +1,8 @@ import json -from cic_syncer.client import translate -from cic_syncer.client.block import Block -from cic_syncer.client.tx import Tx +from chainsyncer.client import translate +from chainsyncer.client.block import Block +from chainsyncer.client.tx import Tx translations = { diff --git a/cic_syncer/client/evm/websocket.py b/chainsyncer/client/evm/websocket.py similarity index 92% rename from cic_syncer/client/evm/websocket.py rename to chainsyncer/client/evm/websocket.py index 3a3ba4a..f857800 100644 --- a/cic_syncer/client/evm/websocket.py +++ b/chainsyncer/client/evm/websocket.py @@ -9,8 +9,8 @@ from hexathon import add_0x # local imports from .response import EVMResponse -from cic_syncer.error import RequestError -from cic_syncer.client.evm.response import EVMBlock +from chainsyncer.error import RequestError +from chainsyncer.client.evm.response import EVMBlock logg = logging.getLogger() @@ -44,7 +44,7 @@ class EVMWebsocketClient: return res.get_result() - def get_block_by_integer(self, n): + def block_by_integer(self, n): req_id = str(uuid.uuid4()) nhx = '0x' + n.to_bytes(8, 'big').hex() req = { @@ -67,7 +67,7 @@ class EVMWebsocketClient: return EVMBlock(o['hash'], o) - def get_block_by_hash(self, hx_in): + def block_by_hash(self, hx_in): req_id = str(uuid.uuid4()) hx = add_0x(hx_in) req ={ diff --git a/cic_syncer/client/translate.py b/chainsyncer/client/translate.py similarity index 100% rename from cic_syncer/client/translate.py rename to chainsyncer/client/translate.py diff --git a/cic_syncer/client/tx.py b/chainsyncer/client/tx.py similarity index 100% rename from cic_syncer/client/tx.py rename to chainsyncer/client/tx.py diff --git a/cic_syncer/db/__init__.py b/chainsyncer/db/__init__.py similarity index 96% rename from cic_syncer/db/__init__.py rename to chainsyncer/db/__init__.py index 6a33fef..bfb64c9 100644 --- a/cic_syncer/db/__init__.py +++ b/chainsyncer/db/__init__.py @@ -3,7 +3,7 @@ import os import logging # local imports -from cic_syncer.db.models.base import SessionBase +from chainsyncer.db.models.base import SessionBase logg = logging.getLogger() diff --git a/cic_syncer/db/models/base.py b/chainsyncer/db/models/base.py similarity index 100% rename from cic_syncer/db/models/base.py rename to chainsyncer/db/models/base.py diff --git a/cic_syncer/db/models/sync.py b/chainsyncer/db/models/sync.py similarity index 100% rename from cic_syncer/db/models/sync.py rename to chainsyncer/db/models/sync.py diff --git a/cic_syncer/driver.py b/chainsyncer/driver.py similarity index 78% rename from cic_syncer/driver.py rename to chainsyncer/driver.py index 2068ea3..28073da 100644 --- a/cic_syncer/driver.py +++ b/chainsyncer/driver.py @@ -10,12 +10,11 @@ class Syncer: running_global = True - def __init__(self, backend, handler): + def __init__(self, backend): self.cursor = None self.running = True self.backend = backend self.filter = [] - self.handler = handler def chain(self): @@ -27,11 +26,14 @@ class Syncer: return self.bc_cache.chain() + def add_filter(self, f): + self.filter.append(f) + class MinedSyncer(Syncer): - def __init__(self, backend, handler): - super(MinedSyncer, self).__init__(backend, handler) + def __init__(self, backend): + super(MinedSyncer, self).__init__(backend) def loop(self, interval, getter): @@ -46,13 +48,13 @@ class MinedSyncer(Syncer): class HeadSyncer(MinedSyncer): - def __init__(self, backend, handler): - super(HeadSyncer, self).__init__(backend, handler) + def __init__(self, backend): + super(HeadSyncer, self).__init__(backend) def process(self, getter, block): logg.debug('process {}'.format(block)) - block = getter.get_block_by_hash(block.hash) + block = getter.block_by_hash(block.hash) i = 0 tx = None while True: @@ -62,7 +64,7 @@ class HeadSyncer(MinedSyncer): logg.debug('tx {}'.format(tx)) self.backend.set(block.number(), i) for f in self.filter: - f.handle(getter, block, tx) + f(getter, block, tx) except IndexError as e: self.backend.set(block.number() + 1, 0) break @@ -73,7 +75,7 @@ class HeadSyncer(MinedSyncer): (block_number, tx_number) = self.backend.get() block_hash = [] uu = uuid.uuid4() - res = getter.get_block_by_integer(block_number) + res = getter.block_by_integer(block_number) logg.debug('get {}'.format(res)) return res diff --git a/cic_syncer/error.py b/chainsyncer/error.py similarity index 100% rename from cic_syncer/error.py rename to chainsyncer/error.py diff --git a/cic_syncer/runnable/tracker.py b/chainsyncer/runnable/tracker.py similarity index 59% rename from cic_syncer/runnable/tracker.py rename to chainsyncer/runnable/tracker.py index 7b9c5e3..9a46d90 100644 --- a/cic_syncer/runnable/tracker.py +++ b/chainsyncer/runnable/tracker.py @@ -9,12 +9,12 @@ import re # third-party imports import confini -from cic_syncer.driver import HeadSyncer -from cic_syncer.db import dsn_from_config -from cic_syncer.db.models.base import SessionBase -from cic_syncer.client.evm.websocket import EVMWebsocketClient -from cic_syncer.backend import SyncerBackend -from cic_syncer.error import LoopDone +from chainlib.eth.connection import HTTPConnection +from chainsyncer.driver import HeadSyncer +from chainsyncer.db import dsn_from_config +from chainsyncer.db.models.base import SessionBase +from chainsyncer.backend import SyncerBackend +from chainsyncer.error import LoopDone logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -55,7 +55,7 @@ config = confini.Config(config_dir, args.env_prefix) config.process() # override args args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), + 'CHAIN_SPEC': getattr(args, 'i'), 'ETH_PROVIDER': getattr(args, 'p'), } config.dict_override(args_override, 'cli flag') @@ -70,58 +70,15 @@ queue = args.q dsn = dsn_from_config(config) SessionBase.connect(dsn) - -transfer_callbacks = [] -for cb in config.get('TASKS_SYNCER_CALLBACKS', '').split(','): - task_split = cb.split(':') - task_queue = queue - if len(task_split) > 1: - task_queue = task_split[0] - task_pair = (task_split[1], task_queue) - transfer_callbacks.append(task_pair) - - -def tx_filter(w3, tx, rcpt, chain_spec): - tx_hash_hex = tx.hash.hex() - otx = Otx.load(tx_hash_hex) - if otx == None: - logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex)) - return None - logg.info('otx found {}'.format(otx.tx_hash)) - s = celery.signature( - 'cic_eth.queue.tx.set_final_status', - [ - tx_hash_hex, - rcpt.blockNumber, - rcpt.status == 0, - ], - queue=queue, - ) - t = s.apply_async() - return t - - -re_websocket = re.compile('^wss?://') -re_http = re.compile('^https?://') -c = EVMWebsocketClient(config.get('ETH_PROVIDER')) -chain = config.get('CIC_CHAIN_SPEC') +c = HTTPConnection(config.get('ETH_PROVIDER')) +chain = config.get('CHAIN_SPEC') def main(): block_offset = c.block_number() - #syncer_backend = SyncerBackend.live(chain, block_offset+1) syncer_backend = SyncerBackend.live(chain, 0) - syncer = HeadSyncer(syncer_backend, handler) - - for cb in config.get('TASKS_SYNCER_CALLBACKS', '').split(','): - task_split = cb.split(':') - task_queue = queue - if len(task_split) > 1: - task_queue = task_split[0] - task_pair = (task_split[1], task_queue) - h = Handler(task_pair[0], task_pair[1]) - syncer.filter.append(h) + syncer = HeadSyncer(syncer_backend) try: logg.debug('block offset {} {}'.format(block_offset, c)) diff --git a/requirements.txt b/requirements.txt index b2a8af7..6deff1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,8 @@ -websocket-client==0.57.0 psycopg2==2.8.6 SQLAlchemy==1.3.20 py-evm==0.3.0a20 eth-tester==0.5.0b3 -web3==5.12.2 confini==0.3.6b2 semver==2.13.0 hexathon==0.0.1a2 +chainlib=0.0.1a4