kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rehabilitate tests, tx_list tests still failing

Browse Source
This commit is contained in:
nolash 2021-04-02 23:26:13 +02:00
parent 68e02ba5b8
commit ffe7416040
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
32 changed files with 332 additions and 1219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
apps/cic-eth/cic_eth/admin/nonce.py
View File

@ -5,11 +5,13 @@ import logging
import celery import celery
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainlib.eth.tx import unpack from chainlib.eth.tx import unpack
from chainqueue.query import get_tx
from chainqueue.state import set_cancel
from chainqueue.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache
# local imports # local imports
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import Nonce from cic_eth.db.models.nonce import Nonce
from cic_eth.admin.ctrl import ( from cic_eth.admin.ctrl import (
lock_send, lock_send,
@ -17,14 +19,8 @@ from cic_eth.admin.ctrl import (
lock_queue, lock_queue,
unlock_queue, unlock_queue,
) )
from cic_eth.queue.tx import ( from cic_eth.queue.tx import queue_create
get_tx, from cic_eth.eth.gas import create_check_gas_task
set_cancel,
)
from cic_eth.queue.tx import create as queue_create
from cic_eth.eth.gas import (
create_check_gas_task,
)
celery_app = celery.current_app celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()

53
apps/cic-eth/cic_eth/api/api_admin.py
View File

@ -22,23 +22,21 @@ from hexathon import (
add_0x, add_0x,
) )
from chainlib.eth.gas import balance from chainlib.eth.gas import balance
from chainqueue.db.enum import (
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import Nonce
from cic_eth.db.enum import (
StatusEnum, StatusEnum,
StatusBits, StatusBits,
is_alive, is_alive,
is_error_status, is_error_status,
status_str, status_str,
) )
from chainqueue.error import TxStateChangeError
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.nonce import Nonce
from cic_eth.error import InitializationError from cic_eth.error import InitializationError
from cic_eth.db.error import TxStateChangeError from cic_eth.queue.query import get_tx
from cic_eth.queue.tx import get_tx
app = celery.current_app app = celery.current_app
@ -138,8 +136,9 @@ class AdminApi:
logg.debug('resend {}'.format(tx_hash_hex)) logg.debug('resend {}'.format(tx_hash_hex))
s_get_tx_cache = celery.signature( s_get_tx_cache = celery.signature(
'cic_eth.queue.tx.get_tx_cache', 'cic_eth.queue.query.get_tx_cache',
[ [
chain_spec.asdict(),
tx_hash_hex, tx_hash_hex,
], ],
queue=self.queue, queue=self.queue,
@ -188,8 +187,9 @@ class AdminApi:
def check_nonce(self, address): def check_nonce(self, address):
s = celery.signature( s = celery.signature(
'cic_eth.queue.tx.get_account_tx', 'cic_eth.queue.query.get_account_tx',
[ [
chain_spec.asdict(),
address, address,
True, True,
False, False,
@ -204,8 +204,9 @@ class AdminApi:
last_nonce = -1 last_nonce = -1
for k in txs.keys(): for k in txs.keys():
s_get_tx = celery.signature( s_get_tx = celery.signature(
'cic_eth.queue.tx.get_tx', 'cic_eth.queue.query.get_tx',
[ [
chain_spec.asdict(),
k, k,
], ],
queue=self.queue, queue=self.queue,
@ -243,8 +244,9 @@ class AdminApi:
def fix_nonce(self, address, nonce, chain_spec): def fix_nonce(self, address, nonce, chain_spec):
s = celery.signature( s = celery.signature(
'cic_eth.queue.tx.get_account_tx', 'cic_eth.queue.query.get_account_tx',
[ [
chain_spec.asdict(),
address, address,
True, True,
False, False,
@ -294,8 +296,9 @@ class AdminApi:
""" """
last_nonce = -1 last_nonce = -1
s = celery.signature( s = celery.signature(
'cic_eth.queue.tx.get_account_tx', 'cic_eth.queue.query.get_account_tx',
[ [
chain_spec.asdict(),
address, address,
], ],
queue=self.queue, queue=self.queue,
@ -306,8 +309,11 @@ class AdminApi:
for tx_hash in txs.keys(): for tx_hash in txs.keys():
errors = [] errors = []
s = celery.signature( s = celery.signature(
'cic_eth.queue.tx.get_tx_cache', 'cic_eth.queue.query.get_tx_cache',
[tx_hash], [
chain_spec.asdict(),
tx_hash,
],
queue=self.queue, queue=self.queue,
) )
tx_dict = s.apply_async().get() tx_dict = s.apply_async().get()
@ -367,12 +373,16 @@ class AdminApi:
#tx_hash = self.w3.keccak(hexstr=tx_raw).hex() #tx_hash = self.w3.keccak(hexstr=tx_raw).hex()
s = celery.signature( s = celery.signature(
'cic_eth.queue.tx.get_tx_cache', 'cic_eth.queue.query.get_tx_cache',
[tx_hash], [
chain_spec.asdict(),
tx_hash,
],
queue=self.queue, queue=self.queue,
) )
tx = s.apply_async().get() t = s.apply_async()
tx = t.get()
source_token = None source_token = None
if tx['source_token'] != ZERO_ADDRESS: if tx['source_token'] != ZERO_ADDRESS:
@ -524,8 +534,9 @@ class AdminApi:
tx['data'] = tx_unpacked['data'] tx['data'] = tx_unpacked['data']
s = celery.signature( s = celery.signature(
'cic_eth.queue.tx.get_state_log', 'cic_eth.queue.state.get_state_log',
[ [
chain_spec.asdict(),
tx_hash, tx_hash,
], ],
queue=self.queue, queue=self.queue,

14
apps/cic-eth/cic_eth/api/api_task.py
View File

@ -93,7 +93,9 @@ class Api:
) )
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[], [
self.chain_spec.asdict(),
],
queue=self.queue, queue=self.queue,
) )
s_tokens = celery.signature( s_tokens = celery.signature(
@ -155,7 +157,9 @@ class Api:
) )
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[], [
self.chain_spec.asdict(),
],
queue=self.queue, queue=self.queue,
) )
s_tokens = celery.signature( s_tokens = celery.signature(
@ -215,6 +219,7 @@ class Api:
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
self.chain_spec.asdict(),
from_address, from_address,
], ],
queue=self.queue, queue=self.queue,
@ -361,6 +366,7 @@ class Api:
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
self.chain_spec.asdict(),
'ACCOUNT_REGISTRY_WRITER', 'ACCOUNT_REGISTRY_WRITER',
], ],
queue=self.queue, queue=self.queue,
@ -399,6 +405,7 @@ class Api:
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
self.chain_spec.asdict(),
'GAS_GIFTER', 'GAS_GIFTER',
], ],
queue=self.queue, queue=self.queue,
@ -439,8 +446,9 @@ class Api:
""" """
offset = 0 offset = 0
s_local = celery.signature( s_local = celery.signature(
'cic_eth.queue.tx.get_account_tx', 'cic_eth.queue.query.get_account_tx',
[ [
self.chain_spec.asdict(),
address, address,
], ],
queue=self.queue, queue=self.queue,

4
apps/cic-eth/cic_eth/db/__init__.py
View File

@ -11,10 +11,6 @@ logg = logging.getLogger()
# an Engine, which the Session will use for connection # an Engine, which the Session will use for connection
# resources # resources
# TODO: Remove the package exports, all models should be imported using full path
from .models.otx import Otx
from .models.convert import TxConvertTransfer
def dsn_from_config(config): def dsn_from_config(config):
"""Generate a dsn string from the provided config dict. """Generate a dsn string from the provided config dict.

2
apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
View File

@ -24,7 +24,7 @@ def upgrade():
sa.Column('blockchain', sa.String), sa.Column('blockchain', sa.String),
sa.Column("flags", sa.BIGINT(), nullable=False, default=0), sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
sa.Column("date_created", sa.DateTime, nullable=False), sa.Column("date_created", sa.DateTime, nullable=False),
sa.Column("otx_id", sa.Integer, nullable=True), sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
) )
op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True) op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)

10
apps/cic-eth/cic_eth/db/models/lock.py
View File

@ -5,11 +5,11 @@ import logging
# third-party imports # third-party imports
from sqlalchemy import Column, String, Integer, DateTime, ForeignKey from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.constant import ZERO_ADDRESS
from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
# local imports # local imports
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.otx import Otx
logg = logging.getLogger() logg = logging.getLogger()
@ -22,10 +22,12 @@ class Lock(SessionBase):
__tablename__ = "lock" __tablename__ = "lock"
blockchain = Column(String) blockchain = Column(String)
address = Column(String, ForeignKey('tx_cache.sender')) #address = Column(String, ForeignKey('tx_cache.sender'))
address = Column(String, ForeignKey(TxCache.sender))
flags = Column(Integer) flags = Column(Integer)
date_created = Column(DateTime, default=datetime.datetime.utcnow) date_created = Column(DateTime, default=datetime.datetime.utcnow)
otx_id = Column(Integer, ForeignKey('otx.id')) otx_id = Column(Integer, ForeignKey(Otx.id))
#otx_id = Column(Integer)
def chain(self): def chain(self):

680
apps/cic-eth/cic_eth/db/models/otx.py
View File

@ -1,680 +0,0 @@
# standard imports
import datetime
import logging
# external imports
from sqlalchemy import Column, Enum, String, Integer, DateTime, Text, or_, ForeignKey
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
# local imports
from .base import SessionBase
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
status_str,
is_error_status,
)
from cic_eth.db.error import TxStateChangeError
logg = logging.getLogger()
class OtxStateLog(SessionBase):
__tablename__ = 'otx_state_log'
date = Column(DateTime, default=datetime.datetime.utcnow)
status = Column(Integer)
otx_id = Column(Integer, ForeignKey('otx.id'))
def __init__(self, otx):
self.otx_id = otx.id
self.status = otx.status
class Otx(SessionBase):
"""Outgoing transactions with local origin.
:param nonce: Transaction nonce
:type nonce: number
:param address: Ethereum address of recipient - NOT IN USE, REMOVE
:type address: str
:param tx_hash: Tranasction hash
:type tx_hash: str, 0x-hex
:param signed_tx: Signed raw transaction data
:type signed_tx: str, 0x-hex
"""
__tablename__ = 'otx'
tracing = False
"""Whether to enable queue state tracing"""
nonce = Column(Integer)
date_created = Column(DateTime, default=datetime.datetime.utcnow)
tx_hash = Column(String(66))
signed_tx = Column(Text)
status = Column(Integer)
block = Column(Integer)
def __set_status(self, status, session):
self.status |= status
session.add(self)
session.flush()
def __reset_status(self, status, session):
status_edit = ~status & self.status
self.status &= status_edit
session.add(self)
session.flush()
def __status_already_set(self, status):
r = bool(self.status & status)
if r:
logg.warning('status bit {} already set on {}'.format(status.name, self.tx_hash))
return r
def __status_not_set(self, status):
r = not(self.status & status)
if r:
logg.warning('status bit {} not set on {}'.format(status.name, self.tx_hash))
return r
def set_block(self, block, session=None):
"""Set block number transaction was mined in.
Only manipulates object, does not transaction or commit to backend.
:param block: Block number
:type block: number
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
session = SessionBase.bind_session(session)
if self.block != None:
SessionBase.release_session(session)
raise TxStateChangeError('Attempted set block {} when block was already {}'.format(block, self.block))
self.block = block
session.add(self)
session.flush()
SessionBase.release_session(session)
def waitforgas(self, session=None):
"""Marks transaction as suspended pending gas funding.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.GAS_ISSUES):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.GAS_ISSUES, session)
self.__reset_status(StatusBits.QUEUED | StatusBits.DEFERRED, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def fubar(self, session=None):
"""Marks transaction as "fubar." Any transaction marked this way is an anomaly and may be a symptom of a serious problem.
Only manipulates object, does not transaction or commit to backend.
"""
if self.__status_already_set(StatusBits.UNKNOWN_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('FUBAR cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('FUBAR cannot be set on an entry with an error state already set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.UNKNOWN_ERROR | StatusBits.FINAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def reject(self, session=None):
"""Marks transaction as "rejected," which means the node rejected sending the transaction to the network. The nonce has not been spent, and the transaction should be replaced.
Only manipulates object, does not transaction or commit to backend.
"""
if self.__status_already_set(StatusBits.NODE_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('REJECTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('REJECTED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('REJECTED cannot be set on an entry with an error state already set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.NODE_ERROR | StatusBits.FINAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def override(self, manual=False, session=None):
"""Marks transaction as manually overridden.
Only manipulates object, does not transaction or commit to backend.
"""
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status)))
if self.status & StatusBits.OBSOLETE:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already OBSOLETE ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.OBSOLETE, session)
#if manual:
# self.__set_status(StatusBits.MANUAL, session)
self.__reset_status(StatusBits.QUEUED | StatusBits.IN_NETWORK, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def manual(self, session=None):
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.MANUAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def retry(self, session=None):
"""Marks transaction as ready to retry after a timeout following a sendfail or a completed gas funding.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.QUEUED):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('RETRY cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if not is_error_status(self.status) and not StatusBits.IN_NETWORK & self.status > 0:
SessionBase.release_session(session)
raise TxStateChangeError('RETRY cannot be set on an entry that has no error ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.QUEUED, session)
self.__reset_status(StatusBits.GAS_ISSUES, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def readysend(self, session=None):
"""Marks transaction as ready for initial send attempt.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.QUEUED):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('READYSEND cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('READYSEND cannot be set on an errored state ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.QUEUED, session)
self.__reset_status(StatusBits.GAS_ISSUES, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def sent(self, session=None):
"""Marks transaction as having been sent to network.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.IN_NETWORK):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('SENT cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.IN_NETWORK, session)
self.__reset_status(StatusBits.DEFERRED | StatusBits.QUEUED | StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def sendfail(self, session=None):
"""Marks that an attempt to send the transaction to the network has failed.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.NODE_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('SENDFAIL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('SENDFAIL cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.LOCAL_ERROR | StatusBits.DEFERRED, session)
self.__reset_status(StatusBits.QUEUED | StatusBits.GAS_ISSUES, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def dequeue(self, session=None):
"""Marks that a process to execute send attempt is underway
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_not_set(StatusBits.QUEUED):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('QUEUED cannot be unset on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('QUEUED cannot be unset on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
self.__reset_status(StatusBits.QUEUED, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def minefail(self, block, session=None):
"""Marks that transaction was mined but code execution did not succeed.
Only manipulates object, does not transaction or commit to backend.
:param block: Block number transaction was mined in.
:type block: number
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.NETWORK_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('REVERTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if not self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('REVERTED cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status)))
if block != None:
self.block = block
self.__set_status(StatusBits.NETWORK_ERROR | StatusBits.FINAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def cancel(self, confirmed=False, session=None):
"""Marks that the transaction has been succeeded by a new transaction with same nonce.
If set to confirmed, the previous state must be OBSOLETED, and will transition to CANCELLED - a finalized state. Otherwise, the state must follow a non-finalized state, and will be set to OBSOLETED.
Only manipulates object, does not transaction or commit to backend.
:param confirmed: Whether transition is to a final state.
:type confirmed: bool
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('CANCEL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if confirmed:
if self.status > 0 and not self.status & StatusBits.OBSOLETE:
SessionBase.release_session(session)
raise TxStateChangeError('CANCEL can only be set on an entry marked OBSOLETE ({})'.format(status_str(self.status)))
self.__set_status(StatusEnum.CANCELLED, session)
else:
self.__set_status(StatusEnum.OBSOLETED, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def success(self, block, session=None):
"""Marks that transaction was successfully mined.
Only manipulates object, does not transaction or commit to backend.
:param block: Block number transaction was mined in.
:type block: number
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('SUCCESS cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if not self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('SUCCESS cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('SUCCESS cannot be set on an entry with error state set ({})'.format(status_str(self.status)))
if block != None:
self.block = block
self.__set_status(StatusEnum.SUCCESS, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
@staticmethod
def get(status=0, limit=4096, status_exact=True, session=None):
"""Returns outgoing transaction lists by status.
Status may either be matched exactly, or be an upper bound of the integer value of the status enum.
:param status: Status value to use in query
:type status: cic_eth.db.enum.StatusEnum
:param limit: Max results to return
:type limit: number
:param status_exact: Whether or not to perform exact status match
:type bool:
:returns: List of transaction hashes
:rtype: tuple, where first element is transaction hash
"""
e = None
session = SessionBase.bind_session(session)
if status_exact:
e = session.query(Otx.tx_hash).filter(Otx.status==status).order_by(Otx.date_created.asc()).limit(limit).all()
else:
e = session.query(Otx.tx_hash).filter(Otx.status<=status).order_by(Otx.date_created.asc()).limit(limit).all()
SessionBase.release_session(session)
return e
@staticmethod
def load(tx_hash, session=None):
"""Retrieves the outgoing transaction record by transaction hash.
:param tx_hash: Transaction hash
:type tx_hash: str, 0x-hex
"""
session = SessionBase.bind_session(session)
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
SessionBase.release_session(session)
return q.first()
@staticmethod
def account(account_address):
"""Retrieves all transaction hashes for which the given Ethereum address is sender or recipient.
:param account_address: Ethereum address to use in query.
:type account_address: str, 0x-hex
:returns: Outgoing transactions
:rtype: tuple, where first element is transaction hash
"""
session = Otx.create_session()
q = session.query(Otx.tx_hash)
q = q.join(TxCache)
q = q.filter(or_(TxCache.sender==account_address, TxCache.recipient==account_address))
txs = q.all()
session.close()
return list(txs)
def __state_log(self, session):
l = OtxStateLog(self)
session.add(l)
# TODO: it is not safe to return otx here unless session has been passed in
@staticmethod
def add(nonce, address, tx_hash, signed_tx, session=None):
external_session = session != None
session = SessionBase.bind_session(session)
otx = Otx(nonce, address, tx_hash, signed_tx)
session.add(otx)
session.flush()
if otx.tracing:
otx.__state_log(session=session)
session.flush()
SessionBase.release_session(session)
if not external_session:
return None
return otx
def __init__(self, nonce, address, tx_hash, signed_tx):
self.nonce = nonce
self.tx_hash = tx_hash
self.signed_tx = signed_tx
self.status = StatusEnum.PENDING
signed_tx_bytes = bytes.fromhex(signed_tx[2:])
# sender_address = address_hex_from_signed_tx(signed_tx_bytes)
# logg.debug('decoded tx {}'.format(sender_address))
# TODO: Most of the methods on this object are obsolete, but it contains a static function for retrieving "expired" outgoing transactions that should be moved to Otx instead.
class OtxSync(SessionBase):
"""Obsolete
"""
__tablename__ = 'otx_sync'
blockchain = Column(String)
block_height_backlog = Column(Integer)
tx_height_backlog = Column(Integer)
block_height_session = Column(Integer)
tx_height_session = Column(Integer)
block_height_head = Column(Integer)
tx_height_head = Column(Integer)
date_created = Column(DateTime, default=datetime.datetime.utcnow)
date_updated = Column(DateTime)
def backlog(self, block_height=None, tx_height=None):
#session = OtxSync.create_session()
if block_height != None:
if tx_height == None:
raise ValueError('tx height missing')
self.block_height_backlog = block_height
self.tx_height_backlog = tx_height
#session.add(self)
self.date_updated = datetime.datetime.utcnow()
#session.commit()
block_height = self.block_height_backlog
tx_height = self.tx_height_backlog
#session.close()
return (block_height, tx_height)
def session(self, block_height=None, tx_height=None):
#session = OtxSync.create_session()
if block_height != None:
if tx_height == None:
raise ValueError('tx height missing')
self.block_height_session = block_height
self.tx_height_session = tx_height
#session.add(self)
self.date_updated = datetime.datetime.utcnow()
#session.commit()
block_height = self.block_height_session
tx_height = self.tx_height_session
#session.close()
return (block_height, tx_height)
def head(self, block_height=None, tx_height=None):
#session = OtxSync.create_session()
if block_height != None:
if tx_height == None:
raise ValueError('tx height missing')
self.block_height_head = block_height
self.tx_height_head = tx_height
#session.add(self)
self.date_updated = datetime.datetime.utcnow()
#session.commit()
block_height = self.block_height_head
tx_height = self.tx_height_head
#session.close()
return (block_height, tx_height)
@hybrid_property
def synced(self):
#return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.block_height_backlog
return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.tx_height_backlog
@staticmethod
def load(blockchain_string, session):
q = session.query(OtxSync)
q = q.filter(OtxSync.blockchain==blockchain_string)
return q.first()
@staticmethod
def latest(nonce):
session = SessionBase.create_session()
otx = session.query(Otx).filter(Otx.nonce==nonce).order_by(Otx.created.desc()).first()
session.close()
return otx
@staticmethod
def get_expired(datetime_threshold):
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.date_created<datetime_threshold)
q = q.filter(Otx.status==StatusEnum.SENT)
q = q.order_by(Otx.date_created.desc())
q = q.group_by(Otx.nonce)
q = q.group_by(Otx.id)
otxs = q.all()
session.close()
return otxs
def chain(self):
return self.blockchain
def __init__(self, blockchain):
self.blockchain = blockchain
self.block_height_head = 0
self.tx_height_head = 0
self.block_height_session = 0
self.tx_height_session = 0
self.block_height_backlog = 0
self.tx_height_backlog = 0

151
apps/cic-eth/cic_eth/db/models/tx.py
View File

@ -1,151 +0,0 @@
# standard imports
import datetime
# third-party imports
from sqlalchemy import Column, String, Integer, DateTime, Enum, ForeignKey, Boolean, NUMERIC
from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
#from sqlalchemy.orm import relationship, backref
#from sqlalchemy.ext.declarative import declarative_base
# local imports
from .base import SessionBase
from .otx import Otx
from cic_eth.db.util import num_serialize
from cic_eth.error import NotLocalTxError
from cic_eth.db.error import TxStateChangeError
class TxCache(SessionBase):
"""Metadata expansions for outgoing transactions.
These records are not essential for handling of outgoing transaction queues. It is implemented to reduce the amount of computation spent of parsing and analysing raw signed transaction data.
Instantiation of the object will fail if an outgoing transaction record with the same transaction hash does not exist.
Typically three types of transactions are recorded:
- Token transfers; where source and destination token values and addresses are identical, sender and recipient differ.
- Token conversions; source and destination token values and addresses differ, sender and recipient are identical.
- Any other transaction; source and destination token addresses are zero-address.
:param tx_hash: Transaction hash
:type tx_hash: str, 0x-hex
:param sender: Ethereum address of transaction sender
:type sender: str, 0x-hex
:param recipient: Ethereum address of transaction beneficiary (e.g. token transfer recipient)
:type recipient: str, 0x-hex
:param source_token_address: Contract address of token that sender spent from
:type source_token_address: str, 0x-hex
:param destination_token_address: Contract address of token that recipient will receive balance of
:type destination_token_address: str, 0x-hex
:param from_value: Amount of source tokens spent
:type from_value: number
:param to_value: Amount of destination tokens received
:type to_value: number
:param block_number: Block height the transaction was mined at, or None if not yet mined
:type block_number: number or None
:param tx_number: Block transaction height the transaction was mined at, or None if not yet mined
:type tx_number: number or None
:raises FileNotFoundError: Outgoing transaction for given transaction hash does not exist
"""
__tablename__ = 'tx_cache'
otx_id = Column(Integer, ForeignKey('otx.id'))
source_token_address = Column(String(42))
destination_token_address = Column(String(42))
sender = Column(String(42))
recipient = Column(String(42))
from_value = Column(NUMERIC())
to_value = Column(NUMERIC())
block_number = Column(Integer())
tx_index = Column(Integer())
date_created = Column(DateTime, default=datetime.datetime.utcnow)
date_updated = Column(DateTime, default=datetime.datetime.utcnow)
date_checked = Column(DateTime, default=datetime.datetime.utcnow)
def check(self):
"""Update the "checked" timestamp to current time.
Only manipulates object, does not transaction or commit to backend.
"""
self.date_checked = datetime.datetime.now()
@staticmethod
def clone(
tx_hash_original,
tx_hash_new,
session=None,
):
"""Copy tx cache data and associate it with a new transaction.
:param tx_hash_original: tx cache data to copy
:type tx_hash_original: str, 0x-hex
:param tx_hash_new: tx hash to associate the copied entry with
:type tx_hash_new: str, 0x-hex
"""
session = SessionBase.bind_session(session)
q = session.query(TxCache)
q = q.join(Otx)
q = q.filter(Otx.tx_hash==tx_hash_original)
txc = q.first()
if txc == None:
SessionBase.release_session(session)
raise NotLocalTxError('original {}'.format(tx_hash_original))
if txc.block_number != None:
SessionBase.release_session(session)
raise TxStateChangeError('cannot clone tx cache of confirmed tx {}'.format(tx_hash_original))
session.flush()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash_new)
otx = q.first()
if otx == None:
SessionBase.release_session(session)
raise NotLocalTxError('new {}'.format(tx_hash_new))
txc_new = TxCache(
otx.tx_hash,
txc.sender,
txc.recipient,
txc.source_token_address,
txc.destination_token_address,
int(txc.from_value),
int(txc.to_value),
session=session,
)
session.add(txc_new)
session.commit()
SessionBase.release_session(session)
def __init__(self, tx_hash, sender, recipient, source_token_address, destination_token_address, from_value, to_value, block_number=None, tx_index=None, session=None):
session = SessionBase.bind_session(session)
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
tx = q.first()
if tx == None:
SessionBase.release_session(session)
raise FileNotFoundError('outgoing transaction record unknown {} (add a Tx first)'.format(tx_hash))
self.otx_id = tx.id
self.sender = sender
self.recipient = recipient
self.source_token_address = source_token_address
self.destination_token_address = destination_token_address
self.from_value = from_value
self.to_value = to_value
self.block_number = block_number
self.tx_index = tx_index
# not automatically set in sqlite, it seems:
self.date_created = datetime.datetime.utcnow()
self.date_updated = self.date_created
self.date_checked = self.date_created
SessionBase.release_session(session)

2
apps/cic-eth/cic_eth/eth/account.py
View File

@ -21,6 +21,7 @@ from chainlib.eth.tx import (
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from eth_accounts_index import AccountRegistry from eth_accounts_index import AccountRegistry
from sarafu_faucet import MinterFaucet as Faucet from sarafu_faucet import MinterFaucet as Faucet
from chainqueue.db.models.tx import TxCache
# local import # local import
from cic_eth_registry import CICRegistry from cic_eth_registry import CICRegistry
@ -31,7 +32,6 @@ from cic_eth.eth.gas import (
from cic_eth.db.models.nonce import Nonce from cic_eth.db.models.nonce import Nonce
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.tx import TxCache
from cic_eth.error import ( from cic_eth.error import (
RoleMissingError, RoleMissingError,
SignerError, SignerError,

11
apps/cic-eth/cic_eth/eth/erc20.py
View File

@ -14,18 +14,22 @@ from chainlib.eth.tx import (
from cic_eth_registry import CICRegistry from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token from cic_eth_registry.erc20 import ERC20Token
from hexathon import strip_0x from hexathon import strip_0x
from chainqueue.db.models.tx import TxCache
from chainqueue.error import NotLocalTxError
# local imports # local imports
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole from cic_eth.db.models.role import AccountRole
from cic_eth.error import TokenCountError, PermanentTxError, OutOfGasError, NotLocalTxError from cic_eth.error import (
TokenCountError,
PermanentTxError,
OutOfGasError,
)
from cic_eth.queue.tx import register_tx from cic_eth.queue.tx import register_tx
from cic_eth.eth.gas import ( from cic_eth.eth.gas import (
create_check_gas_task, create_check_gas_task,
MaxGasOracle, MaxGasOracle,
) )
#from cic_eth.eth.factory import TxFactory
from cic_eth.ext.address import translate_address from cic_eth.ext.address import translate_address
from cic_eth.task import ( from cic_eth.task import (
CriticalSQLAlchemyTask, CriticalSQLAlchemyTask,
@ -90,6 +94,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
:rtype: str, 0x-hex :rtype: str, 0x-hex
""" """
# we only allow one token, one transfer # we only allow one token, one transfer
logg.debug('tokens {}'.format(tokens))
if len(tokens) != 1: if len(tokens) != 1:
raise TokenCountError raise TokenCountError
t = tokens[0] t = tokens[0]

40
apps/cic-eth/cic_eth/eth/tx.py
View File

@ -37,29 +37,29 @@ from hexathon import (
add_0x, add_0x,
strip_0x, strip_0x,
) )
from chainqueue.db.models.tx import Otx
from chainqueue.db.models.tx import TxCache
from chainqueue.db.enum import StatusBits
# local imports # local imports
from cic_eth.db import ( from cic_eth.db import SessionBase
Otx,
SessionBase,
)
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import NonceReservation from cic_eth.db.models.nonce import NonceReservation
from cic_eth.db.models.lock import Lock from cic_eth.db.models.lock import Lock
from cic_eth.db.models.role import AccountRole from cic_eth.db.models.role import AccountRole
from cic_eth.db.enum import ( from cic_eth.db.enum import (
LockEnum, LockEnum,
StatusBits,
) )
from cic_eth.error import PermanentTxError from cic_eth.error import PermanentTxError
from cic_eth.error import TemporaryTxError from cic_eth.error import TemporaryTxError
from cic_eth.error import NotLocalTxError from cic_eth.error import NotLocalTxError
#from cic_eth.queue.tx import create as queue_create #from cic_eth.queue.tx import create as queue_create
from cic_eth.queue.tx import ( from cic_eth.queue.query import (
get_tx, get_tx,
register_tx,
get_nonce_tx, get_nonce_tx,
create as queue_create, )
from cic_eth.queue.tx import (
queue_create,
register_tx,
) )
from cic_eth.error import OutOfGasError from cic_eth.error import OutOfGasError
from cic_eth.error import LockedError from cic_eth.error import LockedError
@ -145,6 +145,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
chain_spec_dict,
address, address,
gas_provider, gas_provider,
], ],
@ -164,6 +165,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
s = celery.signature( s = celery.signature(
'cic_eth.queue.state.set_waitforgas', 'cic_eth.queue.state.set_waitforgas',
[ [
chain_spec_dict,
tx_hash, tx_hash,
], ],
queue=queue, queue=queue,
@ -177,6 +179,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
chain_spec_dict,
address, address,
gas_provider, gas_provider,
], ],
@ -197,6 +200,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
s = celery.signature( s = celery.signature(
'cic_eth.queue.state.set_ready', 'cic_eth.queue.state.set_ready',
[ [
chain_spec_dict,
tx_hash, tx_hash,
], ],
queue=queue, queue=queue,
@ -368,6 +372,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
s_status = celery.signature( s_status = celery.signature(
'cic_eth.queue.state.set_ready', 'cic_eth.queue.state.set_ready',
[ [
chain_spec.asdict(),
tx_hash_hex, tx_hash_hex,
], ],
queue=queue, queue=queue,
@ -395,16 +400,14 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
""" """
session = SessionBase.create_session() session = SessionBase.create_session()
q = session.query(Otx) otx = Otx.load(txold_hash_hex, session)
q = q.filter(Otx.tx_hash==txold_hash_hex)
otx = q.first()
if otx == None: if otx == None:
session.close() session.close()
raise NotLocalTxError(txold_hash_hex) raise NotLocalTxError(txold_hash_hex)
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(otx.signed_tx[2:]) tx_signed_raw_bytes = bytes.fromhex(otx.signed_tx)
tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id()) tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
logg.debug('resend otx {} {}'.format(tx, otx.signed_tx)) logg.debug('resend otx {} {}'.format(tx, otx.signed_tx))
@ -438,11 +441,11 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
tx['gasPrice'] = new_gas_price tx['gasPrice'] = new_gas_price
(tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx) (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
queue_create( queue_create(
chain_spec,
tx['nonce'], tx['nonce'],
tx['from'], tx['from'],
tx_hash_hex, tx_hash_hex,
tx_signed_raw_hex, tx_signed_raw_hex,
chain_spec,
session=session, session=session,
) )
TxCache.clone(txold_hash_hex, tx_hash_hex, session=session) TxCache.clone(txold_hash_hex, tx_hash_hex, session=session)
@ -462,7 +465,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask) @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
def reserve_nonce(self, chained_input, signer_address=None): def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None):
self.log_banner() self.log_banner()
@ -473,7 +476,6 @@ def reserve_nonce(self, chained_input, signer_address=None):
address = chained_input address = chained_input
logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input)) logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
else: else:
#if web3.Web3.isChecksumAddress(signer_address):
if is_checksum_address(signer_address): if is_checksum_address(signer_address):
address = signer_address address = signer_address
logg.debug('explicit address for reserve nonce {}'.format(signer_address)) logg.debug('explicit address for reserve nonce {}'.format(signer_address))
@ -485,7 +487,7 @@ def reserve_nonce(self, chained_input, signer_address=None):
raise ValueError('invalid result when resolving address for nonce {}'.format(address)) raise ValueError('invalid result when resolving address for nonce {}'.format(address))
root_id = self.request.root_id root_id = self.request.root_id
r = NonceReservation.next(address, root_id) r = NonceReservation.next(address, root_id, session=session)
logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0])) logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
session.commit() session.commit()
@ -609,6 +611,8 @@ def cache_gas_data(
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id()) tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
session = SessionBase.create_session()
tx_cache = TxCache( tx_cache = TxCache(
tx_hash_hex, tx_hash_hex,
tx['from'], tx['from'],
@ -617,9 +621,9 @@ def cache_gas_data(
ZERO_ADDRESS, ZERO_ADDRESS,
tx['value'], tx['value'],
tx['value'], tx['value'],
session=session,
) )
session = SessionBase.create_session()
session.add(tx_cache) session.add(tx_cache)
session.commit() session.commit()
cache_id = tx_cache.id cache_id = tx_cache.id

14
apps/cic-eth/cic_eth/ext/tx.py
View File

@ -18,13 +18,14 @@ from chainlib.eth.erc20 import ERC20
from hexathon import strip_0x from hexathon import strip_0x
from cic_eth_registry import CICRegistry from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token from cic_eth_registry.erc20 import ERC20Token
from chainqueue.db.models.otx import Otx
from chainqueue.db.enum import StatusEnum
from chainqueue.query import get_tx_cache
# local imports # local imports
from cic_eth.db.models.otx import Otx
from cic_eth.db.enum import StatusEnum
from cic_eth.queue.tx import get_tx_cache
from cic_eth.queue.time import tx_times from cic_eth.queue.time import tx_times
from cic_eth.task import BaseTask from cic_eth.task import BaseTask
from cic_eth.db.models.base import SessionBase
celery_app = celery.current_app celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()
@ -168,6 +169,8 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
if isinstance(tx_batches, dict): if isinstance(tx_batches, dict):
tx_batches = [tx_batches] tx_batches = [tx_batches]
session = SessionBase.create_session()
for b in tx_batches: for b in tx_batches:
for v in b.values(): for v in b.values():
tx = None tx = None
@ -175,7 +178,7 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
try: try:
hx = strip_0x(v) hx = strip_0x(v)
tx = unpack(bytes.fromhex(hx), chain_spec.chain_id()) tx = unpack(bytes.fromhex(hx), chain_spec.chain_id())
txc = get_tx_cache(tx['hash']) txc = get_tx_cache(chain_spec, tx['hash'], session)
txc['timestamp'] = int(txc['date_created'].timestamp()) txc['timestamp'] = int(txc['date_created'].timestamp())
txc['hash'] = txc['tx_hash'] txc['hash'] = txc['tx_hash']
tx = txc tx = txc
@ -185,6 +188,8 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
k = '{}.{}.{}'.format(tx['timestamp'], tx['sender'], tx['nonce']) k = '{}.{}.{}'.format(tx['timestamp'], tx['sender'], tx['nonce'])
txs_by_block[k] = tx txs_by_block[k] = tx
session.close()
txs = [] txs = []
ks = list(txs_by_block.keys()) ks = list(txs_by_block.keys())
ks.sort() ks.sort()
@ -192,4 +197,5 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
ks.reverse() ks.reverse()
for k in ks: for k in ks:
txs.append(txs_by_block[k]) txs.append(txs_by_block[k])
return txs return txs

2
apps/cic-eth/cic_eth/queue/__init__.py
View File

@ -1,4 +1,2 @@
# third-party imports # third-party imports
import celery import celery
from .tx import get_tx

2
apps/cic-eth/cic_eth/queue/lock.py
View File

@ -7,6 +7,8 @@ from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.db import SessionBase from cic_eth.db import SessionBase
from cic_eth.db.models.lock import Lock from cic_eth.db.models.lock import Lock
celery_app = celery.current_app
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def get_lock(address=None): def get_lock(address=None):

41
apps/cic-eth/cic_eth/queue/query.py
View File

@ -1,37 +1,63 @@
# standard imports
import datetime
# external imports # external imports
import celery
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainlib.eth.tx import unpack from chainlib.eth.tx import unpack
import chainqueue.query import chainqueue.query
from chainqueue.db.enum import (
StatusEnum,
is_alive,
)
from sqlalchemy import func from sqlalchemy import func
from sqlalchemy import or_ from sqlalchemy import or_
from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
# local imports # local imports
from cic_eth.db.enum import LockEnum
from cic_eth.task import CriticalSQLAlchemyTask from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.db.models.lock import Lock from cic_eth.db.models.lock import Lock
from cic_eth.db.models.base import SessionBase
celery_app = celery.current_app
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def get_tx_cache(chain_spec_dict, tx_hash): def get_tx_cache(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.query.get_tx_cache(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.query.get_tx_cache(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def get_tx(chain_spec_dict, tx_hash): def get_tx(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.query.get_tx(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.query.get_tx(chain_spec, tx_hash)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None): def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None) session = SessionBase.create_session()
r = chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None): def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack): session = SessionBase.create_session()
r = chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
session.close()
return r
def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None): def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
@ -43,10 +69,10 @@ def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=No
def get_nonce_tx(chain_spec, nonce, sender): def get_nonce_tx(chain_spec, nonce, sender):
return get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack): return get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack)
def get_upcoming_tx(status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, chain_id=0, session=None): def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
"""Returns the next pending transaction, specifically the transaction with the lowest nonce, for every recipient that has pending transactions. """Returns the next pending transaction, specifically the transaction with the lowest nonce, for every recipient that has pending transactions.
Will omit addresses that have the LockEnum.SEND bit in Lock set. Will omit addresses that have the LockEnum.SEND bit in Lock set.
@ -65,6 +91,7 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, not_status=None, recipient=None
:returns: Transactions :returns: Transactions
:rtype: dict, with transaction hash as key, signed raw transaction as value :rtype: dict, with transaction hash as key, signed raw transaction as value
""" """
chain_id = chain_spec.chain_id()
session = SessionBase.bind_session(session) session = SessionBase.bind_session(session)
q_outer = session.query( q_outer = session.query(
TxCache.sender, TxCache.sender,
@ -109,7 +136,7 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, not_status=None, recipient=None
if o == None: if o == None:
continue continue
tx_signed_bytes = bytes.fromhex(o.signed_tx[2:]) tx_signed_bytes = bytes.fromhex(o.signed_tx)
tx = unpack(tx_signed_bytes, chain_id) tx = unpack(tx_signed_bytes, chain_id)
txs[o.tx_hash] = o.signed_tx txs[o.tx_hash] = o.signed_tx

57
apps/cic-eth/cic_eth/queue/state.py
View File

@ -3,65 +3,98 @@ from chainlib.chain import ChainSpec
import chainqueue.state import chainqueue.state
# local imports # local imports
import celery
from cic_eth.task import CriticalSQLAlchemyTask from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.db.models.base import SessionBase
celery_app = celery.current_app
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_sent_status(chain_spec_dict, tx_hash, fail=False): def set_sent_status(chain_spec_dict, tx_hash, fail=False):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_sent_status(chain_spec, tx_hash, fail) session = SessionBase.create_session()
r = chainqueue.state.set_sent_status(chain_spec, tx_hash, fail, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_final_status(chain_spec_dict, tx_hash, block=None, fail=False): def set_final_status(chain_spec_dict, tx_hash, block=None, fail=False):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_final_status(chain_spec, tx_hash, block, fail) session = SessionBase.create_session()
r = chainqueue.state.set_final_status(chain_spec, tx_hash, block, fail, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_cancel(chain_spec_dict, tx_hash, manual=False): def set_cancel(chain_spec_dict, tx_hash, manual=False):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_cancel(chain_spec, tx_hash, manual) session = SessionBase.create_session()
r = chainqueue.state.set_cancel(chain_spec, tx_hash, manual, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_rejected(chain_spec_dict, tx_hash): def set_rejected(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_rejected(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.state.set_rejected(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_fubar(chain_spec_dict, tx_hash): def set_fubar(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_fubar(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.state.set_fubar(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_manual(chain_spec_dict, tx_hash): def set_manual(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_manual(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.state.set_manual(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_ready(chain_spec_dict, tx_hash): def set_ready(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_ready(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.state.set_ready(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_reserved(chain_spec_dict, tx_hash): def set_reserved(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_reserved(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.state.set_reserved(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def set_waitforgas(chain_spec_dict, tx_hash): def set_waitforgas(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.set_waitforgas(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.state.set_waitforgas(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def get_state_log(chain_spec_dict, tx_hash): def get_state_log(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
return chainqueue.state.get_state_log(chain_spec, tx_hash) session = SessionBase.create_session()
r = chainqueue.state.get_state_log(chain_spec, tx_hash, session=session)
session.close()
return r

9
apps/cic-eth/cic_eth/queue/time.py
View File

@ -18,7 +18,10 @@ celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()
def tx_times(tx_hash, chain_spec): def tx_times(tx_hash, chain_spec, session=None):
session = SessionBase.bind_session(session)
rpc = RPCConnection.connect(chain_spec, 'default') rpc = RPCConnection.connect(chain_spec, 'default')
time_pair = { time_pair = {
'network': None, 'network': None,
@ -35,8 +38,10 @@ def tx_times(tx_hash, chain_spec):
logg.debug('error with getting timestamp details for {}: {}'.format(tx_hash, e)) logg.debug('error with getting timestamp details for {}: {}'.format(tx_hash, e))
pass pass
otx = Otx.load(tx_hash) otx = Otx.load(tx_hash, session=session)
if otx != None: if otx != None:
time_pair['queue'] = int(otx['date_created'].timestamp()) time_pair['queue'] = int(otx['date_created'].timestamp())
SessionBase.release_session(session)
return time_pair return time_pair

20
apps/cic-eth/cic_eth/queue/tx.py
View File

@ -22,6 +22,7 @@ from chainqueue.db.enum import (
is_alive, is_alive,
dead, dead,
) )
from chainqueue.tx import create
from chainqueue.error import NotLocalTxError from chainqueue.error import NotLocalTxError
from chainqueue.db.enum import status_str from chainqueue.db.enum import status_str
@ -36,6 +37,21 @@ celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()
def queue_create(chain_spec, nonce, holder_address, tx_hash, signed_tx, session=None):
session = SessionBase.bind_session(session)
lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session)
if lock > 0:
SessionBase.release_session(session)
raise LockedError(lock)
tx_hash = create(chain_spec, nonce, holder_address, tx_hash, signed_tx, chain_spec, session=session)
SessionBase.release_session(session)
return tx_hash
def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=None, session=None): def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=None, session=None):
"""Signs the provided transaction, and adds it to the transaction queue cache (with status PENDING). """Signs the provided transaction, and adds it to the transaction queue cache (with status PENDING).
@ -55,12 +71,12 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No
tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex)) tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw, chain_id=chain_spec.chain_id()) tx = unpack(tx_signed_raw, chain_id=chain_spec.chain_id())
create( tx_hash = queue_create(
chain_spec,
tx['nonce'], tx['nonce'],
tx['from'], tx['from'],
tx_hash_hex, tx_hash_hex,
tx_signed_raw_hex, tx_signed_raw_hex,
chain_spec,
session=session, session=session,
) )

3
apps/cic-eth/tests/fixtures_celery.py
View File

@ -27,6 +27,9 @@ def celery_includes():
'cic_eth.eth.tx', 'cic_eth.eth.tx',
'cic_eth.ext.tx', 'cic_eth.ext.tx',
'cic_eth.queue.tx', 'cic_eth.queue.tx',
'cic_eth.queue.lock',
'cic_eth.queue.query',
'cic_eth.queue.state',
'cic_eth.queue.balance', 'cic_eth.queue.balance',
'cic_eth.admin.ctrl', 'cic_eth.admin.ctrl',
'cic_eth.admin.nonce', 'cic_eth.admin.nonce',

27
apps/cic-eth/tests/task/api/test_admin.py
View File

@ -16,27 +16,24 @@ from hexathon import (
strip_0x, strip_0x,
add_0x, add_0x,
) )
from chainqueue.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache
from chainqueue.db.enum import (
StatusEnum,
StatusBits,
status_str,
)
from chainqueue.query import get_tx
# local imports # local imports
from cic_eth.api import AdminApi from cic_eth.api import AdminApi
from cic_eth.db.models.role import AccountRole from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.otx import Otx from cic_eth.db.enum import LockEnum
from cic_eth.db.models.tx import TxCache
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
status_str,
LockEnum,
)
from cic_eth.error import InitializationError from cic_eth.error import InitializationError
from cic_eth.eth.tx import ( from cic_eth.eth.tx import (
cache_gas_data, cache_gas_data,
) )
#from cic_eth.eth.gas import cache_gas_tx from cic_eth.queue.tx import queue_create
from cic_eth.queue.tx import (
create as queue_create,
get_tx,
)
logg = logging.getLogger() logg = logging.getLogger()
@ -278,7 +275,7 @@ def test_tx(
eth_signer, eth_signer,
agent_roles, agent_roles,
contract_roles, contract_roles,
celery_session_worker, celery_worker,
): ):
chain_id = default_chain_spec.chain_id() chain_id = default_chain_spec.chain_id()
@ -286,7 +283,7 @@ def test_tx(
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id) c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED) (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id) tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
queue_create(tx['nonce'], agent_roles['ALICE'], tx_hash_hex, tx_signed_raw_hex, default_chain_spec, session=init_database) queue_create(default_chain_spec, tx['nonce'], agent_roles['ALICE'], tx_hash_hex, tx_signed_raw_hex)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT']) api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])

19
apps/cic-eth/tests/task/api/test_list.py
View File

@ -1,23 +1,29 @@
# standard imports # standard imports
import logging import logging
# local imports # external imports
import pytest
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.erc20 import ERC20 from chainlib.eth.erc20 import ERC20
from chainlib.eth.tx import receipt from chainlib.eth.tx import receipt
# local imports
from cic_eth.api.api_task import Api from cic_eth.api.api_task import Api
from tests.mock.filter import (
block_filter,
tx_filter,
)
from cic_eth.db.models.nonce import ( from cic_eth.db.models.nonce import (
Nonce, Nonce,
NonceReservation, NonceReservation,
) )
# test imports
from tests.mock.filter import (
block_filter,
tx_filter,
)
logg = logging.getLogger() logg = logging.getLogger()
@pytest.mark.xfail()
def test_list_tx( def test_list_tx(
default_chain_spec, default_chain_spec,
init_database, init_database,
@ -29,7 +35,7 @@ def test_list_tx(
foo_token, foo_token,
register_tokens, register_tokens,
init_eth_tester, init_eth_tester,
celery_session_worker, celery_worker,
): ):
chain_id = default_chain_spec.chain_id() chain_id = default_chain_spec.chain_id()
@ -110,7 +116,6 @@ def test_list_tx(
r = t.get_leaf() r = t.get_leaf()
assert t.successful() assert t.successful()
assert len(r) == 3 assert len(r) == 3
logg.debug('rrrr {}'.format(r)) logg.debug('rrrr {}'.format(r))

7
apps/cic-eth/tests/task/test_task_account.py
View File

@ -11,13 +11,12 @@ from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import receipt from chainlib.eth.tx import receipt
from eth_accounts_index import AccountRegistry from eth_accounts_index import AccountRegistry
from hexathon import strip_0x from hexathon import strip_0x
from chainqueue.db.enum import StatusEnum
from chainqueue.db.models.otx import Otx
# local imports # local imports
from cic_eth.error import OutOfGasError from cic_eth.error import OutOfGasError
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.db.enum import StatusEnum
from cic_eth.db.enum import StatusEnum
from cic_eth.db.models.nonce import Nonce from cic_eth.db.models.nonce import Nonce
from cic_eth.db.models.role import AccountRole from cic_eth.db.models.role import AccountRole
@ -77,6 +76,7 @@ def test_register_account(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
eth_empty_accounts[0], eth_empty_accounts[0],
default_chain_spec.asdict(),
custodial_roles['ACCOUNT_REGISTRY_WRITER'], custodial_roles['ACCOUNT_REGISTRY_WRITER'],
], ],
queue=None, queue=None,
@ -171,6 +171,7 @@ def test_gift(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
agent_roles['ALICE'], agent_roles['ALICE'],
default_chain_spec.asdict(),
], ],
queue=None, queue=None,
) )

2
apps/cic-eth/tests/task/test_task_erc20.py
View File

@ -105,6 +105,7 @@ def test_erc20_transfer_task(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
[token_object], [token_object],
default_chain_spec.asdict(),
custodial_roles['FOO_TOKEN_GIFTER'], custodial_roles['FOO_TOKEN_GIFTER'],
], ],
queue=None, queue=None,
@ -146,6 +147,7 @@ def test_erc20_approve_task(
'cic_eth.eth.tx.reserve_nonce', 'cic_eth.eth.tx.reserve_nonce',
[ [
[token_object], [token_object],
default_chain_spec.asdict(),
custodial_roles['FOO_TOKEN_GIFTER'], custodial_roles['FOO_TOKEN_GIFTER'],
], ],
queue=None, queue=None,

10
apps/cic-eth/tests/task/test_task_tx.py
View File

@ -13,16 +13,15 @@ from chainlib.eth.tx import (
receipt, receipt,
) )
from hexathon import strip_0x from hexathon import strip_0x
from chainqueue.db.models.otx import Otx
# local imports # local imports
from cic_eth.queue.tx import register_tx from cic_eth.queue.tx import register_tx
from cic_eth.eth.tx import cache_gas_data from cic_eth.eth.tx import cache_gas_data
from cic_eth.db.models.otx import Otx
logg = logging.getLogger() logg = logging.getLogger()
@pytest.mark.skip()
def test_tx_send( def test_tx_send(
init_database, init_database,
default_chain_spec, default_chain_spec,
@ -62,12 +61,11 @@ def test_tx_send(
assert rcpt['status'] == 1 assert rcpt['status'] == 1
@pytest.mark.skip()
def test_sync_tx( def test_sync_tx(
default_chain_spec, default_chain_spec,
eth_rpc, eth_rpc,
eth_signer, eth_signer,
celery_worker, celery_session_worker,
): ):
pass pass
@ -78,7 +76,7 @@ def test_resend_with_higher_gas(
eth_rpc, eth_rpc,
eth_signer, eth_signer,
agent_roles, agent_roles,
celery_worker, celery_session_worker,
): ):
chain_id = default_chain_spec.chain_id() chain_id = default_chain_spec.chain_id()
@ -102,7 +100,7 @@ def test_resend_with_higher_gas(
r = t.get_leaf() r = t.get_leaf()
q = init_database.query(Otx) q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==r) q = q.filter(Otx.tx_hash==strip_0x(r))
otx = q.first() otx = q.first()
if otx == None: if otx == None:
raise NotLocalTxError(r) raise NotLocalTxError(r)

29
apps/cic-eth/tests/unit/db/test_block_sync.py
View File

@ -1,29 +0,0 @@
# standard imports
import logging
# local imports
from cic_eth.db.models.otx import OtxSync
logg = logging.getLogger()
def test_db_block_sync(
init_database,
):
s = OtxSync('eip155:8995:bloxberg')
s.head(666, 12)
assert s.head() == (666, 12)
s.session(42, 13)
assert s.session() == (42, 13)
s.backlog(13, 2)
assert s.backlog() == (13, 2)
assert not s.synced
s.backlog(42, 13)
assert s.backlog() == (42, 13)
assert s.synced

2
apps/cic-eth/tests/unit/db/test_db_convert_transfer.py
View File

@ -3,7 +3,7 @@ import logging
import pytest import pytest
from cic_eth.db import TxConvertTransfer from cic_eth.db.models.convert import TxConvertTransfer
from cic_eth.db.error import UnknownConvertError from cic_eth.db.error import UnknownConvertError
logg = logging.getLogger() logg = logging.getLogger()

107
apps/cic-eth/tests/unit/db/test_otx.py
View File

@ -1,107 +0,0 @@
# standard imports
import os
import logging
# third-party imports
import pytest
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.otx import OtxStateLog
from cic_eth.db.models.otx import Otx
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
is_alive,
)
logg = logging.getLogger()
#def test_get(
# rpc_eth,
# rpc_signer,
# agent_roles,
# init_database,
# ):
#
# rpc = RPCConnection.connect(default_chain_spec, 'default')
# nonce_oracle = RPCNonceOracle(agent_roles['ALICE'])
# gas_oracle = RPCGasOracle(eth_rpc)
# c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
#
# for i in range(10):
#
# (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)),
#
# tx_def = {
# 'from': init_w3.eth.accounts[0],
# 'to': init_w3.eth.accounts[1],
# 'nonce': 0,
# 'value': 101,
# 'gasPrice': 2000000000,
# 'gas': 21000,
# 'data': '',
# 'chainId': 1,
# }
#
# session = init_database
# txs = []
# for i in range(10):
# nonce = init_w3.eth.getTransactionCount(init_w3.eth.accounts[0], 'pending')
# tx_def['nonce'] = nonce
# tx = init_w3.eth.sign_transaction(tx_def)
# tx_hash = init_w3.eth.send_raw_transaction(tx['raw'])
# logg.debug('tx {}'.format(tx))
#
# address = init_w3.eth.accounts[i%3]
# otx = Otx(int((i/3)+1), address, '0x'+tx_hash.hex(), tx['raw'])
# txs.append(otx)
# session.add(otx)
# session.flush()
#
# logg.debug(txs)
# session.commit()
#
# txs[0].status = 0
# session.add(txs[0])
# session.commit()
# session.close()
#
# get_txs = Otx.get()
# logg.debug(get_txs)
def test_state_log(
init_database,
):
Otx.tracing = True
address = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex()
otx = Otx.add(0, address, tx_hash, signed_tx, session=init_database)
otx.waitforgas(session=init_database)
init_database.commit()
otx.readysend(session=init_database)
init_database.commit()
otx.sent(session=init_database)
init_database.commit()
otx.success(1024, session=init_database)
init_database.commit()
q = init_database.query(OtxStateLog)
q = q.filter(OtxStateLog.otx_id==otx.id)
q = q.order_by(OtxStateLog.date.asc())
logs = q.all()
assert logs[0].status == StatusEnum.PENDING
assert logs[1].status == StatusEnum.WAITFORGAS
assert logs[2].status & StatusBits.QUEUED
assert logs[3].status & StatusBits.IN_NETWORK
assert not is_alive(logs[4].status)

97
apps/cic-eth/tests/unit/db/test_status.py
View File

@ -1,97 +0,0 @@
# standard imports
import os
# third-party imports
import pytest
# local imports
from cic_eth.db.models.otx import Otx
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
is_alive,
)
@pytest.fixture(scope='function')
def otx(
init_database,
):
bogus_hash = '0x' + os.urandom(32).hex()
bogus_address = '0x' + os.urandom(20).hex()
bogus_tx_raw = '0x' + os.urandom(128).hex()
return Otx(0, bogus_address, bogus_hash, bogus_tx_raw)
def test_status_chain_gas(
init_database,
otx,
):
otx.waitforgas(init_database)
otx.readysend(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_straight_success(
init_database,
otx,
):
otx.readysend(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_straight_revert(
init_database,
otx,
):
otx.readysend(init_database)
otx.sent(init_database)
otx.minefail(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_nodeerror(
init_database,
otx,
):
otx.readysend(init_database)
otx.sendfail(init_database)
otx.retry(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_nodeerror_multiple(
init_database,
otx,
):
otx.readysend(init_database)
otx.sendfail(init_database)
otx.retry(init_database)
otx.sendfail(init_database)
otx.retry(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_nodeerror(
init_database,
otx,
):
otx.readysend(init_database)
otx.reject(init_database)
assert not is_alive(otx.status)

14
apps/cic-eth/tests/unit/db/test_tx.py
View File

@ -18,10 +18,8 @@ from hexathon import (
add_0x, add_0x,
strip_0x, strip_0x,
) )
from chainqueue.db.models.tx import TxCache
# local imports from chainqueue.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.otx import Otx
# test imports # test imports
from tests.util.gas import StaticGasOracle from tests.util.gas import StaticGasOracle
@ -46,7 +44,6 @@ def test_set(
otx = Otx( otx = Otx(
tx['nonce'], tx['nonce'],
tx['from'],
tx_hash_hex, tx_hash_hex,
tx_signed_raw_hex, tx_signed_raw_hex,
) )
@ -66,6 +63,7 @@ def test_set(
to_value, to_value,
666, 666,
13, 13,
session=init_database,
) )
init_database.add(txc) init_database.add(txc)
init_database.commit() init_database.commit()
@ -112,7 +110,6 @@ def test_clone(
tx_dict = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id) tx_dict = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
otx = Otx( otx = Otx(
tx_dict['nonce'], tx_dict['nonce'],
tx_dict['from'],
tx_hash_hex, tx_hash_hex,
tx_signed_raw_hex, tx_signed_raw_hex,
) )
@ -130,15 +127,16 @@ def test_clone(
ZERO_ADDRESS, ZERO_ADDRESS,
txs[0]['value'], txs[0]['value'],
txs[0]['value'], txs[0]['value'],
session=init_database,
) )
init_database.add(txc) init_database.add(txc)
init_database.commit() init_database.commit()
TxCache.clone(txs[0]['hash'], txs[1]['hash']) TxCache.clone(txs[0]['hash'], txs[1]['hash'], session=init_database)
q = init_database.query(TxCache) q = init_database.query(TxCache)
q = q.join(Otx) q = q.join(Otx)
q = q.filter(Otx.tx_hash==txs[1]['hash']) q = q.filter(Otx.tx_hash==strip_0x(txs[1]['hash']))
txc_clone = q.first() txc_clone = q.first()
assert txc_clone != None assert txc_clone != None

30
apps/cic-eth/tests/unit/queue/test_balances.py
View File

@ -59,17 +59,15 @@ def test_assemble():
assert r[1].get('balance_xyzzy') != None assert r[1].get('balance_xyzzy') != None
@pytest.mark.skip()
def test_outgoing_balance( def test_outgoing_balance(
default_chain_spec, default_chain_spec,
init_database, init_database,
): ):
chain_str = str(default_chain_spec)
recipient = '0x' + os.urandom(20).hex() recipient = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex() tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex() signed_tx = '0x' + os.urandom(128).hex()
otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database) otx = Otx.add(0, tx_hash, signed_tx, session=init_database)
init_database.add(otx) init_database.add(otx)
init_database.commit() init_database.commit()
@ -83,6 +81,7 @@ def test_outgoing_balance(
token_address, token_address,
1000, 1000,
1000, 1000,
session=init_database,
) )
init_database.add(txc) init_database.add(txc)
init_database.commit() init_database.commit()
@ -91,33 +90,35 @@ def test_outgoing_balance(
'address': token_address, 'address': token_address,
'converters': [], 'converters': [],
} }
b = balance_outgoing([token_data], sender, chain_str) b = balance_outgoing([token_data], sender, default_chain_spec.asdict())
assert b[0]['balance_outgoing'] == 1000 assert b[0]['balance_outgoing'] == 1000
otx.readysend(session=init_database)
init_database.flush()
otx.reserve(session=init_database)
init_database.flush()
otx.sent(session=init_database) otx.sent(session=init_database)
init_database.commit() init_database.commit()
b = balance_outgoing([token_data], sender, chain_str) b = balance_outgoing([token_data], sender, default_chain_spec.asdict())
assert b[0]['balance_outgoing'] == 1000 assert b[0]['balance_outgoing'] == 1000
otx.success(block=1024, session=init_database) otx.success(block=1024, session=init_database)
init_database.commit() init_database.commit()
b = balance_outgoing([token_data], sender, chain_str) b = balance_outgoing([token_data], sender, default_chain_spec.asdict())
assert b[0]['balance_outgoing'] == 0 assert b[0]['balance_outgoing'] == 0
@pytest.mark.skip()
def test_incoming_balance( def test_incoming_balance(
default_chain_spec, default_chain_spec,
init_database, init_database,
): ):
chain_str = str(default_chain_spec)
recipient = '0x' + os.urandom(20).hex() recipient = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex() tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex() signed_tx = '0x' + os.urandom(128).hex()
otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database) otx = Otx.add(0, tx_hash, signed_tx, session=init_database)
init_database.add(otx) init_database.add(otx)
init_database.commit() init_database.commit()
@ -131,6 +132,7 @@ def test_incoming_balance(
token_address, token_address,
1000, 1000,
1000, 1000,
session=init_database,
) )
init_database.add(txc) init_database.add(txc)
init_database.commit() init_database.commit()
@ -139,19 +141,23 @@ def test_incoming_balance(
'address': token_address, 'address': token_address,
'converters': [], 'converters': [],
} }
b = balance_incoming([token_data], recipient, chain_str) b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 0 assert b[0]['balance_incoming'] == 0
otx.readysend(session=init_database)
init_database.flush()
otx.reserve(session=init_database)
init_database.flush()
otx.sent(session=init_database) otx.sent(session=init_database)
init_database.commit() init_database.commit()
b = balance_incoming([token_data], recipient, chain_str) b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 1000 assert b[0]['balance_incoming'] == 1000
otx.success(block=1024, session=init_database) otx.success(block=1024, session=init_database)
init_database.commit() init_database.commit()
b = balance_incoming([token_data], recipient, chain_str) b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 0 assert b[0]['balance_incoming'] == 0

58
apps/cic-eth/tests/unit/queue/test_query.py Normal file
View File

@ -0,0 +1,58 @@
# external imports
from chainqueue.db.enum import (
StatusEnum,
StatusBits,
)
from chainlib.connection import RPCConnection
from chainlib.eth.gas import (
RPCGasOracle,
Gas,
)
from chainlib.chain import ChainSpec
# local imports
from cic_eth.db.enum import LockEnum
from cic_eth.db.models.lock import Lock
from cic_eth.queue.query import get_upcoming_tx
from cic_eth.queue.tx import register_tx
from cic_eth.eth.tx import cache_gas_data
# test imports
from tests.util.nonce import StaticNonceOracle
def test_upcoming_with_lock(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = StaticNonceOracle(42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
(tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6))
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
assert len(txs.keys()) == 1
Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE'])
txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
assert len(txs.keys()) == 0
(tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6))
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
assert len(txs.keys()) == 1

10
apps/cic-eth/tests/unit/queue/test_queue_lock.py
View File

@ -3,12 +3,12 @@ import os
# third-party imports # third-party imports
import pytest import pytest
from chainqueue.tx import create as queue_create
# local imports # local imports
from cic_eth.db.models.lock import Lock from cic_eth.db.models.lock import Lock
from cic_eth.db.enum import LockEnum from cic_eth.db.enum import LockEnum
from cic_eth.error import LockedError from cic_eth.error import LockedError
from cic_eth.queue.tx import queue_create
def test_queue_lock( def test_queue_lock(
@ -25,41 +25,41 @@ def test_queue_lock(
Lock.set(chain_str, LockEnum.QUEUE) Lock.set(chain_str, LockEnum.QUEUE)
with pytest.raises(LockedError): with pytest.raises(LockedError):
queue_create( queue_create(
default_chain_spec,
0, 0,
address, address,
tx_hash, tx_hash,
tx_raw, tx_raw,
chain_str
) )
Lock.set(chain_str, LockEnum.QUEUE, address=address) Lock.set(chain_str, LockEnum.QUEUE, address=address)
with pytest.raises(LockedError): with pytest.raises(LockedError):
queue_create( queue_create(
default_chain_spec,
0, 0,
address, address,
tx_hash, tx_hash,
tx_raw, tx_raw,
chain_str
) )
Lock.reset(chain_str, LockEnum.QUEUE) Lock.reset(chain_str, LockEnum.QUEUE)
with pytest.raises(LockedError): with pytest.raises(LockedError):
queue_create( queue_create(
default_chain_spec,
0, 0,
address, address,
tx_hash, tx_hash,
tx_raw, tx_raw,
chain_str
) )
Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash) Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
with pytest.raises(LockedError): with pytest.raises(LockedError):
queue_create( queue_create(
default_chain_spec,
0, 0,
address, address,
tx_hash, tx_hash,
tx_raw, tx_raw,
chain_str
) )