2022-03-11 20:38:12 +01:00
|
|
|
|
# standard imports
|
|
|
|
|
import re
|
2022-03-12 09:48:19 +01:00
|
|
|
|
import datetime
|
2022-04-10 16:00:01 +02:00
|
|
|
|
import logging
|
2022-05-01 08:27:52 +02:00
|
|
|
|
import time
|
2022-03-11 20:38:12 +01:00
|
|
|
|
|
2022-03-12 14:48:40 +01:00
|
|
|
|
# local imports
|
|
|
|
|
from chainqueue.cache import CacheTx
|
2022-03-13 16:40:45 +01:00
|
|
|
|
from chainqueue.entry import QueueEntry
|
2022-05-01 08:44:33 +02:00
|
|
|
|
from chainqueue.error import NotLocalTxError
|
2022-04-30 07:42:32 +02:00
|
|
|
|
from chainqueue.enum import (
|
|
|
|
|
StatusBits,
|
|
|
|
|
all_errors,
|
|
|
|
|
)
|
2022-03-12 14:48:40 +01:00
|
|
|
|
|
2022-04-10 16:00:01 +02:00
|
|
|
|
logg = logging.getLogger(__name__)
|
|
|
|
|
|
2022-03-12 14:48:40 +01:00
|
|
|
|
|
2022-03-12 09:48:19 +01:00
|
|
|
|
def to_key(t, n, k):
|
|
|
|
|
return '{}_{}_{}'.format(t, n, k)
|
2022-03-11 22:49:23 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def from_key(k):
|
2022-03-12 09:48:19 +01:00
|
|
|
|
(ts_str, seq_str, tx_hash) = k.split('_')
|
|
|
|
|
return (float(ts_str), int(seq_str), tx_hash, )
|
2022-03-11 22:49:23 +01:00
|
|
|
|
|
2022-04-30 07:42:32 +02:00
|
|
|
|
all_local_errors = all_errors() - StatusBits.NETWORK_ERROR
|
2022-03-11 22:49:23 +01:00
|
|
|
|
|
2022-03-11 20:38:12 +01:00
|
|
|
|
re_u = r'^[^_][_A-Z]+$'
|
|
|
|
|
class Store:
|
|
|
|
|
|
2022-05-04 07:44:47 +02:00
|
|
|
|
def __init__(self, chain_spec, state_store, index_store, counter, cache=None, sync=True):
|
2022-03-12 14:48:40 +01:00
|
|
|
|
self.chain_spec = chain_spec
|
2022-03-12 09:48:19 +01:00
|
|
|
|
self.cache = cache
|
2022-03-11 20:38:12 +01:00
|
|
|
|
self.state_store = state_store
|
|
|
|
|
self.index_store = index_store
|
2022-03-12 09:48:19 +01:00
|
|
|
|
self.counter = counter
|
2022-03-11 20:38:12 +01:00
|
|
|
|
for s in dir(self.state_store):
|
|
|
|
|
if not re.match(re_u, s):
|
|
|
|
|
continue
|
|
|
|
|
v = self.state_store.from_name(s)
|
|
|
|
|
setattr(self, s, v)
|
2022-03-13 18:22:39 +01:00
|
|
|
|
for v in [
|
|
|
|
|
'state',
|
|
|
|
|
'change',
|
|
|
|
|
'set',
|
|
|
|
|
'unset',
|
|
|
|
|
'name',
|
|
|
|
|
'modified',
|
2022-05-02 22:21:51 +02:00
|
|
|
|
'purge',
|
2022-03-13 18:22:39 +01:00
|
|
|
|
]:
|
2022-03-11 20:38:12 +01:00
|
|
|
|
setattr(self, v, getattr(self.state_store, v))
|
2022-04-30 20:31:02 +02:00
|
|
|
|
|
2022-05-04 07:44:47 +02:00
|
|
|
|
if not sync:
|
|
|
|
|
return
|
|
|
|
|
|
2022-04-30 20:31:02 +02:00
|
|
|
|
sync_err = None
|
2022-05-01 08:44:33 +02:00
|
|
|
|
try:
|
|
|
|
|
self.state_store.sync()
|
|
|
|
|
except Exception as e:
|
|
|
|
|
sync_err = e
|
2022-04-30 20:31:02 +02:00
|
|
|
|
|
|
|
|
|
if sync_err != None:
|
2022-05-01 08:44:33 +02:00
|
|
|
|
raise FileNotFoundError(sync_err)
|
2022-03-11 20:38:12 +01:00
|
|
|
|
|
2022-03-12 15:12:02 +01:00
|
|
|
|
|
2022-03-15 09:06:39 +01:00
|
|
|
|
def put(self, v, cache_adapter=CacheTx):
|
2022-03-15 10:00:15 +01:00
|
|
|
|
tx = cache_adapter(self.chain_spec)
|
2022-03-15 09:06:39 +01:00
|
|
|
|
tx.deserialize(v)
|
|
|
|
|
k = tx.hash
|
2022-03-12 09:48:19 +01:00
|
|
|
|
n = self.counter.next()
|
|
|
|
|
t = datetime.datetime.now().timestamp()
|
|
|
|
|
s = to_key(t, n, k)
|
|
|
|
|
self.index_store.put(k, s)
|
2022-04-12 15:44:29 +02:00
|
|
|
|
self.state_store.put(s, v)
|
2022-03-12 09:48:19 +01:00
|
|
|
|
if self.cache != None:
|
2022-03-12 14:48:40 +01:00
|
|
|
|
self.cache.put(self.chain_spec, tx)
|
2022-03-15 09:06:39 +01:00
|
|
|
|
return (s, k,)
|
2022-03-11 20:38:12 +01:00
|
|
|
|
|
|
|
|
|
|
2022-03-11 22:49:23 +01:00
|
|
|
|
def get(self, k):
|
2022-04-30 20:31:02 +02:00
|
|
|
|
v = None
|
2022-05-01 08:44:33 +02:00
|
|
|
|
s = self.index_store.get(k)
|
|
|
|
|
err = None
|
|
|
|
|
try:
|
|
|
|
|
v = self.state_store.get(s)
|
|
|
|
|
except FileNotFoundError as e:
|
|
|
|
|
err = e
|
2022-04-30 20:31:02 +02:00
|
|
|
|
if v == None:
|
2022-05-01 08:44:33 +02:00
|
|
|
|
raise NotLocalTxError('could not find tx {}: {}'.format(k, err))
|
2022-03-12 09:48:19 +01:00
|
|
|
|
return (s, v,)
|
2022-03-11 20:38:12 +01:00
|
|
|
|
|
|
|
|
|
|
2022-05-06 10:30:14 +02:00
|
|
|
|
def by_state(self, state=0, not_state=0, include_pending=False, limit=4096, strict=False, threshold=None):
|
2022-03-11 20:38:12 +01:00
|
|
|
|
hashes = []
|
|
|
|
|
i = 0
|
2022-05-06 10:30:14 +02:00
|
|
|
|
|
|
|
|
|
refs_state = []
|
|
|
|
|
if state > 0:
|
|
|
|
|
if self.state_store.is_pure(state):
|
|
|
|
|
refs_state = self.state_store.list(state)
|
|
|
|
|
elif strict:
|
|
|
|
|
refs_state = self.state_store.list(state)
|
|
|
|
|
else:
|
|
|
|
|
for v in self.state_store.elements(state, numeric=True):
|
|
|
|
|
refs_state += self.state_store.list(v)
|
2022-05-06 16:00:05 +02:00
|
|
|
|
refs_state = list(set(refs_state))
|
2022-05-06 10:30:14 +02:00
|
|
|
|
if include_pending:
|
|
|
|
|
refs_state += self.state_store.list(0)
|
2022-03-11 22:49:23 +01:00
|
|
|
|
|
2022-04-30 18:43:55 +02:00
|
|
|
|
refs_state.sort()
|
2022-05-06 10:30:14 +02:00
|
|
|
|
|
2022-03-13 16:40:45 +01:00
|
|
|
|
for ref in refs_state:
|
|
|
|
|
v = from_key(ref)
|
|
|
|
|
hsh = v[2]
|
|
|
|
|
|
2022-04-30 18:43:55 +02:00
|
|
|
|
item_state = self.state_store.state(ref)
|
|
|
|
|
|
2022-03-13 16:40:45 +01:00
|
|
|
|
if strict:
|
2022-03-11 20:43:00 +01:00
|
|
|
|
if item_state & state != item_state:
|
2022-03-11 20:38:12 +01:00
|
|
|
|
continue
|
2022-03-13 18:22:39 +01:00
|
|
|
|
|
2022-04-30 18:43:55 +02:00
|
|
|
|
if item_state & not_state > 0:
|
|
|
|
|
continue
|
|
|
|
|
|
2022-05-04 07:44:47 +02:00
|
|
|
|
item_state_str = self.state_store.name(item_state)
|
|
|
|
|
logg.info('state {} {} ({})'.format(ref, item_state_str, item_state))
|
|
|
|
|
|
2022-03-13 18:22:39 +01:00
|
|
|
|
if threshold != None:
|
|
|
|
|
v = self.state_store.modified(ref)
|
|
|
|
|
if v > threshold:
|
|
|
|
|
continue
|
|
|
|
|
|
2022-03-13 16:40:45 +01:00
|
|
|
|
hashes.append(hsh)
|
2022-03-11 22:49:23 +01:00
|
|
|
|
|
2022-04-30 07:42:32 +02:00
|
|
|
|
i += 1
|
|
|
|
|
if limit > 0 and i == limit:
|
|
|
|
|
break
|
2022-03-13 18:22:39 +01:00
|
|
|
|
|
2022-05-06 10:30:14 +02:00
|
|
|
|
#hashes.sort()
|
2022-03-13 16:40:45 +01:00
|
|
|
|
return hashes
|
2022-03-12 14:48:40 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def upcoming(self, limit=4096):
|
|
|
|
|
return self.by_state(state=self.QUEUED, limit=limit)
|
2022-03-13 16:40:45 +01:00
|
|
|
|
|
|
|
|
|
|
2022-03-13 18:22:39 +01:00
|
|
|
|
def deferred(self, limit=4096, threshold=None):
|
|
|
|
|
return self.by_state(state=self.DEFERRED, limit=limit, threshold=threshold)
|
2022-03-13 16:40:45 +01:00
|
|
|
|
|
|
|
|
|
|
2022-04-30 07:42:32 +02:00
|
|
|
|
def failed(self, limit=4096):
|
|
|
|
|
#return self.by_state(state=all_local_errors, limit=limit)
|
|
|
|
|
r = []
|
|
|
|
|
r += self.by_state(state=self.LOCAL_ERROR, limit=limit)
|
|
|
|
|
r += self.by_state(state=self.NODE_ERROR, limit=limit)
|
|
|
|
|
r.sort()
|
|
|
|
|
if len(r) > limit:
|
|
|
|
|
r = r[:limit]
|
|
|
|
|
return r
|
|
|
|
|
|
|
|
|
|
|
2022-03-13 16:40:45 +01:00
|
|
|
|
def pending(self, limit=4096):
|
2022-05-09 21:40:41 +02:00
|
|
|
|
return self.by_state(include_pending=True, limit=limit, strict=True)
|
2022-03-13 16:40:45 +01:00
|
|
|
|
|
|
|
|
|
|
2022-03-14 20:53:54 +01:00
|
|
|
|
def reserve(self, k):
|
|
|
|
|
entry = QueueEntry(self, k)
|
|
|
|
|
entry.load()
|
|
|
|
|
entry.reserve()
|
|
|
|
|
|
|
|
|
|
|
2022-03-13 16:40:45 +01:00
|
|
|
|
def enqueue(self, k):
|
|
|
|
|
entry = QueueEntry(self, k)
|
|
|
|
|
entry.load()
|
|
|
|
|
try:
|
|
|
|
|
entry.retry()
|
|
|
|
|
except StateTransitionInvalid:
|
|
|
|
|
entry.readysend()
|
2022-03-13 16:45:48 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def fail(self, k):
|
|
|
|
|
entry = QueueEntry(self, k)
|
2022-03-13 17:04:14 +01:00
|
|
|
|
entry.load()
|
2022-04-30 07:42:32 +02:00
|
|
|
|
logg.debug('fail {}'.format(k))
|
2022-03-13 17:04:14 +01:00
|
|
|
|
entry.sendfail()
|
2022-03-14 20:53:54 +01:00
|
|
|
|
|
|
|
|
|
|
2022-03-14 22:17:00 +01:00
|
|
|
|
def final(self, k, block, tx, error=False):
|
|
|
|
|
entry = QueueEntry(self, k)
|
|
|
|
|
entry.load()
|
|
|
|
|
if error:
|
|
|
|
|
entry.fail(block, tx)
|
|
|
|
|
else:
|
|
|
|
|
entry.succeed(block, tx)
|
|
|
|
|
|
|
|
|
|
|
2022-03-14 20:53:54 +01:00
|
|
|
|
def send_start(self, k):
|
|
|
|
|
entry = QueueEntry(self, k)
|
|
|
|
|
entry.load()
|
|
|
|
|
entry.reserve()
|
|
|
|
|
return entry
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def send_end(self, k):
|
|
|
|
|
entry = QueueEntry(self, k)
|
|
|
|
|
entry.load()
|
|
|
|
|
entry.sent()
|
2022-04-29 08:28:01 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_reserved(self, k):
|
|
|
|
|
entry = QueueEntry(self, k)
|
|
|
|
|
entry.load()
|
|
|
|
|
return entry.test(self.RESERVED)
|
2022-04-30 20:31:02 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sync(self):
|
|
|
|
|
self.state_store.sync()
|