Add missing lock file

CHANGELOG

@@ -1,15 +1,3 @@
-- 0.2.9
-	* Minimize instantiations of adapters in filter execution
-- 0.2.8
-	* Upgrade chainsyncer
-- 0.2.7
-	* Upgrade chainlib
-- 0.2.6
-	* Deps upgrade
-- 0.2.5
-	* Deps upgrade
-- 0.2.4
-	* Allow omission of state store sync in queue store backend
 - 0.2.2
 	* Fix missing symbol crashes related to race conditions
 - 0.2.1

chaind/adapters/base.py

@@ -13,13 +13,13 @@ logg = logging.getLogger(__name__)
 
 class ChaindAdapter:
 
-    def __init__(self, chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, store_sync=True):
+    def __init__(self, chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0):
         self.cache_adapter = cache_adapter
         self.dispatcher = dispatcher
         store_lock = StoreLock()
         while True:
             try:
-                self.store = QueueStore(chain_spec, state_store, index_store, counter_store, cache=cache, sync=store_sync)
+                self.store = QueueStore(chain_spec, state_store, index_store, counter_store, cache=cache)
                 break
             except FileNotFoundError as e:
                 logg.debug('queuestore instantiation failed, possible race condition (will try again): {}'.format(e))

chaind/adapters/fs.py

@@ -26,13 +26,13 @@ logg = logging.getLogger(__name__)
 
 class ChaindFsAdapter(ChaindAdapter):
 
-    def __init__(self, chain_spec, path, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, digest_bytes=32, event_callback=None, store_sync=True):
+    def __init__(self, chain_spec, path, cache_adapter, dispatcher, cache=None, pending_retry_threshold=0, error_retry_threshold=0, digest_bytes=32, event_callback=None):
         factory = SimpleFileStoreFactory(path, use_lock=True).add
         state_store = Status(factory, allow_invalid=True, event_callback=event_callback)
         index_path = os.path.join(path, 'tx')
         index_store = IndexStore(index_path, digest_bytes=digest_bytes)
         counter_store = CounterStore(path)
-        super(ChaindFsAdapter, self).__init__(chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=cache, pending_retry_threshold=pending_retry_threshold, error_retry_threshold=error_retry_threshold, store_sync=store_sync)
+        super(ChaindFsAdapter, self).__init__(chain_spec, state_store, index_store, counter_store, cache_adapter, dispatcher, cache=cache, pending_retry_threshold=pending_retry_threshold, error_retry_threshold=error_retry_threshold)
 
 
     def put(self, signed_tx):
@@ -51,11 +51,11 @@ class ChaindFsAdapter(ChaindAdapter):
                 logg.error('I am just a simple syncer and do not know how to handle the state which the tx {} is in: {}'.format(tx_hash, e))
                 return None
             except FileNotFoundError as e:
-                logg.debug('queuestore get (file missing) {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
+                logg.debug('queuestore get {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
                 store_lock.again()
                 continue
             except StateLockedKey as e:
-                logg.debug('queuestore get (statelock) {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
+                logg.debug('queuestore get {} failed, possible race condition (will try again): {}'.format(tx_hash, e))
                 store_lock.again()
                 continue
 
@@ -90,19 +90,12 @@ class ChaindFsAdapter(ChaindAdapter):
     def succeed(self, block, tx):
         if self.store.is_reserved(tx.hash):
             raise QueueLockError(tx.hash)
-        r = self.store.final(tx.hash, block, tx, error=False)
-        (k, v) = self.store.get(tx.hash)
-        self.store.purge(k)
-        return r
+
+        return self.store.final(tx.hash, block, tx, error=False)
 
 
     def fail(self, block, tx):
-        if self.store.is_reserved(tx.hash):
-            raise QueueLockError(tx.hash)
-        r = self.store.final(tx.hash, block, tx, error=True)
-        (k, v) = self.store.get(tx.hash)
-        self.store.purge(k)
-        return r
+        return self.store.final(tx.hash, block, tx, error=True)
 
 
     def sendfail(self):
@@ -135,8 +128,7 @@ class ChaindFsAdapter(ChaindAdapter):
         r = None
         try:
             r = self.dispatcher.send(tx_wire)
-        except RPCException as e:
-            logg.error('dispatch send failed for {}: {}'.format(tx_hash, e))
+        except RPCException:
             self.store.fail(tx_hash)
             return False
 

chaind/dispatch.py

@@ -1,33 +0,0 @@
-# standard imports
-import logging
-
-# local ipmorts
-from chaind.adapters.fs import ChaindFsAdapter
-from chaind.eth.cache import EthCacheTx
-
-logg = logging.getLogger(__name__)
-
-
-class DispatchProcessor:
-
-    def __init__(self, chain_spec, queue_dir, dispatcher):
-        self.dispatcher = dispatcher
-        self.chain_spec = chain_spec,
-        self.queue_dir = queue_dir
-
-
-    def process(self, rpc, limit=50):
-        adapter = ChaindFsAdapter(
-            self.chain_spec,
-            self.queue_dir, 
-            EthCacheTx,
-            self.dispatcher,
-            )
-        
-        upcoming = adapter.upcoming(limit=limit)
-        logg.info('processor has {} candidates for {}, processing with limit {}'.format(len(upcoming), self.chain_spec, limit))
-        i = 0
-        for tx_hash in upcoming:
-            if adapter.dispatch(tx_hash):
-                i += 1
-        return i

chaind/filter.py

@@ -26,69 +26,43 @@ class StateFilter(SyncFilter):
         self.adapter_path = adapter_path
         self.tx_adapter = tx_adapter
         self.throttler = throttler
-        self.last_block_height = 0
-        self.adapter = None
-        self.store_lock = None
-
-
-    def __get_adapter(self, block, force_reload=False):
-        if self.store_lock == None:
-            self.store_lock = StoreLock()
-
-        reload = False
-        if block.number != self.last_block_height:
-            reload = True
-        elif self.adapter == None:
-            reload = True
-        elif force_reload:
-            reload = True
-        
-        self.last_block_height = block.number
-
-        if reload:
-            while True:
-                logg.info('reloading adapter')
-                try:
-                    self.adapter = ChaindFsAdapter(
-                        self.chain_spec,
-                        self.adapter_path,
-                        self.tx_adapter,
-                        None,
-                        )
-                    break
-                except BackendError as e:
-                    logg.error('adapter instantiation failed: {}, one more try'.format(e))
-                    self.store_lock.again()
-                    continue
-
-        return self.adapter
 
 
     def filter(self, conn, block, tx, session=None):
         cache_tx = None
-        queue_adapter = self.__get_adapter(block)
-        
-        self.store_lock.reset()
-    
+        store_lock = StoreLock()
+        queue_adapter = None
         while True:
+            try:
+                queue_adapter = ChaindFsAdapter(
+                    self.chain_spec,
+                    self.adapter_path,
+                    self.tx_adapter,
+                    None,
+                    )
+            except BackendError as e:
+                logg.error('adapter instantiation failed: {}, one more try'.format(e))
+                store_lock.again()
+                continue
+
+            store_lock.reset()
+
             try:
                 cache_tx = queue_adapter.get(tx.hash)
                 break
             except NotLocalTxError:
                 logg.debug('skipping not local transaction {}'.format(tx.hash))
-                self.__stop_adapter()
                 return False
             except BackendError as e:
-                logg.error('adapter get failed: {}, one more try'.format(e))
-                self.store_lock.again()
-                queue_adapter = self.__get_adapter(block, force_reload=True)
+                logg.error('adapter instantiation failed: {}, one more try'.format(e))
+                queue_adapter = None
+                store_lock.again()
                 continue
 
         if cache_tx == None:
             raise NotLocalTxError(tx.hash)
 
-        self.store_lock.reset()
-
+        store_lock = StoreLock()
         queue_lock = StoreLock(error=QueueLockError)
         while True:
             try:
@@ -102,18 +76,15 @@ class StateFilter(SyncFilter):
                 queue_lock.again()
             except FileNotFoundError as e:
                 logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
-                self.store_lock.again()
-                queue_adapter = self.__get_adapter(block, force_reload=True)
+                store_lock.again()
                 continue
             except NotLocalTxError as e:
                 logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
-                self.store_lock.again()
-                queue_adapter = self.__get_adapter(block, force_reload=True)
+                store_lock.again()
                 continue
             except StateLockedKey as e:
                 logg.debug('queue item {} not found, possible race condition, will retry: {}'.format(tx.hash, e))
-                self.store_lock.again()
-                queue_adapter = self.__get_adapter(block, force_reload=True)
+                store_lock.again()
                 continue
 
         logg.info('filter registered {} for {} in {}'.format(tx.status.name, tx.hash, block))

chaind/lock.py

@@ -5,7 +5,7 @@ import time
 from .error import BackendError
 
 BASE_DELAY = 0.01
-BASE_DELAY_LIMIT = 10.0
+BASE_DELAY_LIMIT = 3.0
 
 
 class StoreLock:

chaind/session.py

@@ -22,7 +22,7 @@ logg = logging.getLogger(__name__)
 
 class SessionController:
 
-    def __init__(self, config, processor):
+    def __init__(self, config, adapter, processor):
         self.dead = False
         os.makedirs(os.path.dirname(config.get('SESSION_SOCKET_PATH')), exist_ok=True)
         try:
@@ -37,6 +37,7 @@ class SessionController:
         self.srv.settimeout(float(config.get('SESSION_DISPATCH_DELAY')))
         self.processor = processor
         self.chain_spec = config.get('CHAIN_SPEC')
+        self.adapter = adapter
 
 
     def shutdown(self, signo, frame):
@@ -64,7 +65,7 @@ class SessionController:
         r = None
         while True:
             try:
-                r = self.processor(conn)
+                r = self.processor(self.chain_spec, self.adapter, conn)
                 break
             except BackendError as e:
                 state_lock.again(e)

chaind/unittest/common.py

@@ -29,6 +29,8 @@ class MockDispatcher:
 
 
     def send(self, v):
+        import sys
+        sys.stderr.write('susu v {} {}\n'.format(v, self.fails))
         if v in self.fails:
             raise RPCException('{} is in fails'.format(v))
         pass

requirements.txt

@@ -1,6 +1,6 @@
-chainlib~=0.1.2
-chainqueue~=0.1.12
-chainsyncer~=0.4.4
+chainlib~=0.1.1
+chainqueue~=0.1.8
+chainsyncer~=0.4.3
 confini~=0.6.0
 funga~=0.5.2
 pyxdg~=0.26

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = chaind
-version = 0.2.9
+version = 0.2.2
 description = Base package for chain queue service
 author = Louis Holbrook
 author_email = dev@holbrook.no