kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rehabilitate transfer, approve

Browse Source 
Signed-off-by: nolash <dev@holbrook.no>
This commit is contained in:
Louis Holbrook
2021-03-29 13:27:53 +00:00
parent 299385f320
commit b65ab8a0ca
222 changed files with 3272 additions and 809800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
apps/cic-eth/cic_eth/queue/balance.py
View File

@@ -3,10 +3,10 @@ import logging
# third-party imports
import celery
from chainlib.chain import ChainSpec
from hexathon import strip_0x
# local imports
from cic_registry.chain import ChainSpec
from cic_eth.db import SessionBase
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
@@ -21,7 +21,7 @@ celery_app = celery.current_app
logg = logging.getLogger()
def __balance_outgoing_compatible(token_address, holder_address, chain_str):
def __balance_outgoing_compatible(token_address, holder_address):
session = SessionBase.create_session()
q = session.query(TxCache.from_value)
q = q.join(Otx)
@@ -37,7 +37,7 @@ def __balance_outgoing_compatible(token_address, holder_address, chain_str):
@celery_app.task(base=CriticalSQLAlchemyTask)
def balance_outgoing(tokens, holder_address, chain_str):
def balance_outgoing(tokens, holder_address, chain_spec_dict):
"""Retrieve accumulated value of unprocessed transactions sent from the given address.
:param tokens: list of token spec dicts with addresses to retrieve balances for
@@ -49,15 +49,15 @@ def balance_outgoing(tokens, holder_address, chain_str):
:returns: Tokens dicts with outgoing balance added
:rtype: dict
"""
chain_spec = ChainSpec.from_chain_str(chain_str)
chain_spec = ChainSpec.from_dict(chain_spec_dict)
for t in tokens:
b = __balance_outgoing_compatible(t['address'], holder_address, chain_str)
b = __balance_outgoing_compatible(t['address'], holder_address)
t['balance_outgoing'] = b
return tokens
def __balance_incoming_compatible(token_address, receiver_address, chain_str):
def __balance_incoming_compatible(token_address, receiver_address):
session = SessionBase.create_session()
q = session.query(TxCache.to_value)
q = q.join(Otx)
@@ -75,7 +75,7 @@ def __balance_incoming_compatible(token_address, receiver_address, chain_str):
@celery_app.task(base=CriticalSQLAlchemyTask)
def balance_incoming(tokens, receipient_address, chain_str):
def balance_incoming(tokens, receipient_address, chain_spec_dict):
"""Retrieve accumulated value of unprocessed transactions to be received by the given address.
:param tokens: list of token spec dicts with addresses to retrieve balances for
@@ -87,9 +87,9 @@ def balance_incoming(tokens, receipient_address, chain_str):
:returns: Tokens dicts with outgoing balance added
:rtype: dict
"""
chain_spec = ChainSpec.from_chain_str(chain_str)
chain_spec = ChainSpec.from_dict(chain_spec_dict)
for t in tokens:
b = __balance_incoming_compatible(t['address'], receipient_address, chain_str)
b = __balance_incoming_compatible(t['address'], receipient_address)
t['balance_incoming'] = b
return tokens
@@ -107,6 +107,7 @@ def assemble_balances(balances_collection):
:rtype: list of dicts
"""
tokens = {}
logg.debug('received collection {}'.format(balances_collection))
for c in balances_collection:
for b in c:
address = b['address']

23
apps/cic-eth/cic_eth/queue/time.py
View File

@@ -2,12 +2,13 @@
import logging
# third-party imports
import web3
import celery
from cic_registry.chain import ChainSpec
from chainlib.chain import ChainSpec
from chainlib.connection import RPCConnection
from chainlib.eth.block import block_by_hash
from chainlib.eth.tx import receipt
# local imports
from cic_eth.eth.rpc import RpcClient
from cic_eth.db.models.otx import Otx
from cic_eth.error import NotLocalTxError
from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
@@ -17,21 +18,21 @@ celery_app = celery.current_app
logg = logging.getLogger()
# TODO: This method does not belong in the _queue_ module, it operates across queue and network
@celery_app.task(base=CriticalSQLAlchemyAndWeb3Task)
def tx_times(tx_hash, chain_str):
chain_spec = ChainSpec.from_chain_str(chain_str)
c = RpcClient(chain_spec)
def tx_times(tx_hash, chain_spec):
rpc = RPCConnection.connect(chain_spec, 'default')
time_pair = {
'network': None,
'queue': None,
}
try:
rcpt = c.w3.eth.getTransactionReceipt(tx_hash)
block = c.w3.eth.getBlock(rcpt['blockHash'])
o = receipt(tx_hash)
r = rpc.do(o)
o = block_by_hash(r['block_hash'])
block = rpc.do(o)
logg.debug('rcpt {}'.format(block))
time_pair['network'] = block['timestamp']
except web3.exceptions.TransactionNotFound:
except Exception as e:
logg.debug('error with getting timestamp details for {}: {}'.format(tx_hash, e))
pass
otx = Otx.load(tx_hash)

143
apps/cic-eth/cic_eth/queue/tx.py
View File

@@ -3,17 +3,16 @@ import logging
import time
import datetime
# third-party imports
# external imports
import celery
from hexathon import strip_0x
from sqlalchemy import or_
from sqlalchemy import not_
from sqlalchemy import tuple_
from sqlalchemy import func
from chainlib.eth.tx import unpack
# local imports
from cic_registry import CICRegistry
from cic_registry.chain import ChainSpec
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.otx import OtxStateLog
from cic_eth.db.models.tx import TxCache
@@ -27,7 +26,6 @@ from cic_eth.db.enum import (
dead,
)
from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.eth.util import unpack_signed_raw_tx # TODO: should not be in same sub-path as package that imports queue.tx
from cic_eth.error import NotLocalTxError
from cic_eth.error import LockedError
from cic_eth.db.enum import status_str
@@ -37,7 +35,7 @@ celery_app = celery.current_app
logg = logging.getLogger()
def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predecessors=True, session=None):
def create(nonce, holder_address, tx_hash, signed_tx, chain_spec, obsolete_predecessors=True, session=None):
"""Create a new transaction queue record.
:param nonce: Transaction nonce
@@ -48,13 +46,13 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
:type tx_hash: str, 0x-hex
:param signed_tx: Signed raw transaction
:type signed_tx: str, 0x-hex
:param chain_str: Chain spec string representation to create transaction for
:type chain_str: str
:param chain_spec: Chain spec to create transaction for
:type chain_spec: ChainSpec
:returns: transaction hash
:rtype: str, 0x-hash
"""
session = SessionBase.bind_session(session)
lock = Lock.check_aggregate(chain_str, LockEnum.QUEUE, holder_address, session=session)
lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session)
if lock > 0:
SessionBase.release_session(session)
raise LockedError(lock)
@@ -69,17 +67,26 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
session.flush()
if obsolete_predecessors:
# TODO: obsolete previous txs from same holder with same nonce
q = session.query(Otx)
q = q.join(TxCache)
q = q.filter(Otx.nonce==nonce)
q = q.filter(TxCache.sender==holder_address)
q = q.filter(Otx.tx_hash!=tx_hash)
q = q.filter(Otx.status<=StatusEnum.SENT)
q = q.filter(Otx.status.op('&')(StatusBits.FINAL)==0)
for otx in q.all():
logg.info('otx {} obsoleted by {}'.format(otx.tx_hash, tx_hash))
otx.cancel(confirmed=False, session=session)
try:
otx.cancel(confirmed=False, session=session)
except TxStateChangeError as e:
logg.exception('obsolete fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('obsolete UNEXPECTED fail: {}'.format(e))
session.close()
raise(e)
session.commit()
SessionBase.release_session(session)
@@ -87,6 +94,50 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
return tx_hash
def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=None, session=None):
"""Signs the provided transaction, and adds it to the transaction queue cache (with status PENDING).
:param tx: Standard ethereum transaction data
:type tx: dict
:param chain_spec: Chain spec of transaction to add to queue
:type chain_spec: chainlib.chain.ChainSpec
:param queue: Task queue
:type queue: str
:param cache_task: Cache task to call with signed transaction. If None, no task will be called.
:type cache_task: str
:raises: sqlalchemy.exc.DatabaseError
:returns: Tuple; Transaction hash, signed raw transaction data
:rtype: tuple
"""
logg.debug('adding queue tx {}:{} -> {}'.format(chain_spec, tx_hash_hex, tx_signed_raw_hex))
tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw, chain_id=chain_spec.chain_id())
create(
tx['nonce'],
tx['from'],
tx_hash_hex,
tx_signed_raw_hex,
chain_spec,
session=session,
)
if cache_task != None:
logg.debug('adding cache task {} tx {}'.format(cache_task, tx_hash_hex))
s_cache = celery.signature(
cache_task,
[
tx_hash_hex,
tx_signed_raw_hex,
chain_spec.asdict(),
],
queue=queue,
)
s_cache.apply_async()
return (tx_hash_hex, tx_signed_raw_hex,)
# TODO: Replace set_* with single task for set status
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_sent_status(tx_hash, fail=False):
@@ -109,10 +160,20 @@ def set_sent_status(tx_hash, fail=False):
session.close()
return False
if fail:
o.sendfail(session=session)
else:
o.sent(session=session)
try:
if fail:
o.sendfail(session=session)
else:
o.sent(session=session)
except TxStateChangeError as e:
logg.exception('set sent fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('set sent UNEXPECED fail: {}'.format(e))
session.close()
raise(e)
session.commit()
session.close()
@@ -156,10 +217,20 @@ def set_final_status(tx_hash, block=None, fail=False):
q = q.filter(Otx.tx_hash==tx_hash)
o = q.first()
if fail:
o.minefail(block, session=session)
else:
o.success(block, session=session)
try:
if fail:
o.minefail(block, session=session)
else:
o.success(block, session=session)
session.commit()
except TxStateChangeError as e:
logg.exception('set final fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('set final UNEXPECED fail: {}'.format(e))
session.close()
raise(e)
q = session.query(Otx)
q = q.join(TxCache)
@@ -168,8 +239,16 @@ def set_final_status(tx_hash, block=None, fail=False):
q = q.filter(Otx.tx_hash!=tx_hash)
for otwo in q.all():
otwo.cancel(True, session=session)
try:
otwo.cancel(True, session=session)
except TxStateChangeError as e:
logg.exception('cancel non-final fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('cancel non-final UNEXPECTED fail: {}'.format(e))
session.close()
raise(e)
session.commit()
session.close()
@@ -197,12 +276,16 @@ def set_cancel(tx_hash, manual=False):
session.flush()
if manual:
o.override(session=session)
else:
o.cancel(session=session)
session.commit()
try:
if manual:
o.override(session=session)
else:
o.cancel(session=session)
session.commit()
except TxStateChangeError as e:
logg.exception('set cancel fail: {}'.format(e))
except Exception as e:
logg.exception('set cancel UNEXPECTED fail: {}'.format(e))
session.close()
return tx_hash
@@ -513,7 +596,7 @@ def get_nonce_tx(nonce, sender, chain_id):
txs = {}
for r in q.all():
tx_signed_bytes = bytes.fromhex(r.signed_tx[2:])
tx = unpack_signed_raw_tx(tx_signed_bytes, chain_id)
tx = unpack(tx_signed_bytes, chain_id)
if sender == None or tx['from'] == sender:
txs[r.tx_hash] = r.signed_tx
@@ -558,7 +641,7 @@ def get_paused_txs(status=None, sender=None, chain_id=0, session=None):
for r in q.all():
tx_signed_bytes = bytes.fromhex(r.signed_tx[2:])
tx = unpack_signed_raw_tx(tx_signed_bytes, chain_id)
tx = unpack(tx_signed_bytes, chain_id)
if sender == None or tx['from'] == sender:
#gas += tx['gas'] * tx['gasPrice']
txs[r.tx_hash] = r.signed_tx
@@ -664,7 +747,7 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
continue
tx_signed_bytes = bytes.fromhex(o.signed_tx[2:])
tx = unpack_signed_raw_tx(tx_signed_bytes, chain_id)
tx = unpack(tx_signed_bytes, chain_id)
txs[o.tx_hash] = o.signed_tx
q = session.query(TxCache)