kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cic-eth: Make nonce separate task

Browse Source
This commit is contained in:
Louis Holbrook
2021-03-06 17:55:51 +00:00
parent 618769a0d2
commit f0088f20de
51 changed files with 1140 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
apps/cic-eth/cic_eth/eth/account.py
View File

@@ -36,6 +36,7 @@ class AccountTxFactory(TxFactory):
self,
address,
chain_spec,
uuid,
session=None,
):
"""Register an Ethereum account address with the on-chain account registry
@@ -59,7 +60,7 @@ class AccountTxFactory(TxFactory):
'gas': gas,
'gasPrice': self.gas_price,
'chainId': chain_spec.chain_id(),
'nonce': self.next_nonce(session=session),
'nonce': self.next_nonce(uuid, session=session),
'value': 0,
})
return tx_add
@@ -69,6 +70,7 @@ class AccountTxFactory(TxFactory):
self,
address,
chain_spec,
uuid,
session=None,
):
"""Trigger the on-chain faucet to disburse tokens to the provided Ethereum account
@@ -90,7 +92,7 @@ class AccountTxFactory(TxFactory):
'gas': gas,
'gasPrice': self.gas_price,
'chainId': chain_spec.chain_id(),
'nonce': self.next_nonce(session=session),
'nonce': self.next_nonce(uuid, session=session),
'value': 0,
})
return tx_add
@@ -156,19 +158,9 @@ def create(password, chain_str):
logg.debug('created account {}'.format(a))
# Initialize nonce provider record for account
# TODO: this can safely be set to zero, since we are randomly creating account
n = c.w3.eth.getTransactionCount(a, 'pending')
session = SessionBase.create_session()
q = session.query(Nonce)
q = q.filter(Nonce.address_hex==a)
o = q.first()
session.flush()
if o == None:
o = Nonce()
o.address_hex = a
o.nonce = n
session.add(o)
session.commit()
Nonce.init(a, session=session)
session.commit()
session.close()
return a
@@ -203,7 +195,7 @@ def register(self, account_address, chain_str, writer_address=None):
c = RpcClient(chain_spec, holder_address=writer_address)
txf = AccountTxFactory(writer_address, c)
tx_add = txf.add(account_address, chain_spec, session=session)
tx_add = txf.add(account_address, chain_spec, self.request.root_id, session=session)
(tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_add, chain_str, queue, 'cic_eth.eth.account.cache_account_data', session=session)
session.close()
@@ -243,7 +235,7 @@ def gift(self, account_address, chain_str):
txf = AccountTxFactory(account_address, c)
session = SessionBase.create_session()
tx_add = txf.gift(account_address, chain_spec, session=session)
tx_add = txf.gift(account_address, chain_spec, self.request.root_id, session=session)
(tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_add, chain_str, queue, 'cic_eth.eth.account.cache_gift_data', session=session)
session.close()

6
apps/cic-eth/cic_eth/eth/factory.py
View File

@@ -32,10 +32,10 @@ class TxFactory:
logg.debug('txfactory instance address {} gas price'.format(self.address, self.gas_price))
def next_nonce(self, session=None):
"""Returns the current cached nonce value, and increments it for next transaction.
def next_nonce(self, uuid, session=None):
"""Returns the current reserved nonce value, and increments it for next transaction.
:returns: Nonce
:rtype: number
"""
return self.nonce_oracle.next(session=session)
return self.nonce_oracle.next_by_task_uuid(uuid, session=session)

14
apps/cic-eth/cic_eth/eth/nonce.py
View File

@@ -1,5 +1,8 @@
# local imports
from cic_eth.db.models.nonce import Nonce
from cic_eth.db.models.nonce import (
Nonce,
NonceReservation,
)
class NonceOracle():
"""Ensures atomic nonce increments for all transactions across all tasks and threads.
@@ -14,10 +17,15 @@ class NonceOracle():
self.default_nonce = default_nonce
def next(self, session=None):
def next(self):
"""Get next unique nonce.
:returns: Nonce
:rtype: number
"""
return Nonce.next(self.address, self.default_nonce, session=session)
raise AttributeError('this should not be called')
return Nonce.next(self.address, self.default_nonce)
def next_by_task_uuid(self, uuid, session=None):
return NonceReservation.release(uuid, session=session)

10
apps/cic-eth/cic_eth/eth/token.py
View File

@@ -46,6 +46,7 @@ class TokenTxFactory(TxFactory):
spender_address,
amount,
chain_spec,
uuid,
session=None,
):
"""Create an ERC20 "approve" transaction
@@ -74,7 +75,7 @@ class TokenTxFactory(TxFactory):
'gas': source_token_gas,
'gasPrice': self.gas_price,
'chainId': chain_spec.chain_id(),
'nonce': self.next_nonce(session=session),
'nonce': self.next_nonce(uuid, session=session),
})
return tx_approve
@@ -85,6 +86,7 @@ class TokenTxFactory(TxFactory):
receiver_address,
value,
chain_spec,
uuid,
session=None,
):
"""Create an ERC20 "transfer" transaction
@@ -114,7 +116,7 @@ class TokenTxFactory(TxFactory):
'gas': source_token_gas,
'gasPrice': self.gas_price,
'chainId': chain_spec.chain_id(),
'nonce': self.next_nonce(session=session),
'nonce': self.next_nonce(uuid, session=session),
})
return tx_transfer
@@ -248,7 +250,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_str):
txf = TokenTxFactory(holder_address, c)
session = SessionBase.create_session()
tx_transfer = txf.transfer(t['address'], receiver_address, value, chain_spec, session=session)
tx_transfer = txf.transfer(t['address'], receiver_address, value, chain_spec, self.request.root_id, session=session)
(tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, cache_task='cic_eth.eth.token.otx_cache_transfer', session=session)
session.close()
@@ -304,7 +306,7 @@ def approve(self, tokens, holder_address, spender_address, value, chain_str):
txf = TokenTxFactory(holder_address, c)
session = SessionBase.create_session()
tx_transfer = txf.approve(t['address'], spender_address, value, chain_spec, session=session)
tx_transfer = txf.approve(t['address'], spender_address, value, chain_spec, self.request.root_id, session=session)
(tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, cache_task='cic_eth.eth.token.otx_cache_approve', session=session)
session.close()

72
apps/cic-eth/cic_eth/eth/tx.py
View File

@@ -12,6 +12,7 @@ from cic_registry.chain import ChainSpec
from .rpc import RpcClient
from cic_eth.db import Otx, SessionBase
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import NonceReservation
from cic_eth.db.models.lock import Lock
from cic_eth.db.enum import (
LockEnum,
@@ -84,15 +85,23 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required))
if gas_required > balance:
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[
address,
c.gas_provider(),
],
queue=queue,
)
s_refill_gas = celery.signature(
'cic_eth.eth.tx.refill_gas',
[
address,
chain_str,
],
queue=queue,
)
s_refill_gas.apply_async()
s_nonce.link(s_refill_gas)
s_nonce.apply_async()
wait_tasks = []
for tx_hash in tx_hashes:
s = celery.signature(
@@ -108,15 +117,23 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
safe_gas = c.safe_threshold_amount()
if balance < safe_gas:
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[
address,
c.gas_provider(),
],
queue=queue,
)
s_refill_gas = celery.signature(
'cic_eth.eth.tx.refill_gas',
[
address,
chain_str,
],
queue=queue,
)
s_refill_gas.apply_async()
s_nonce.link(s_refill)
s_nonce.apply_async()
logg.debug('requested refill from {} to {}'.format(c.gas_provider(), address))
ready_tasks = []
for tx_hash in tx_hashes:
@@ -287,11 +304,10 @@ class ParityNodeHandler:
s_debug = celery.signature(
'cic_eth.admin.debug.alert',
[
tx_hash_hex,
tx_hash_hex,
debugstr,
],
queue=queue,
queue=self.queue,
)
s_set_reject.link(s_debug)
s_lock.link(s_set_reject)
@@ -299,7 +315,7 @@ class ParityNodeHandler:
return (t, PermanentTxError, 'Reject invalid {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id())))
def handle_default(self, tx_hash_hex, tx_hex):
def handle_default(self, tx_hash_hex, tx_hex, debugstr):
tx_bytes = bytes.fromhex(tx_hex[2:])
tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id())
s_lock = celery.signature(
@@ -317,9 +333,18 @@ class ParityNodeHandler:
[],
queue=self.queue,
)
s_debug = celery.signature(
'cic_eth.admin.debug.alert',
[
tx_hash_hex,
debugstr,
],
queue=self.queue,
)
s_set_fubar.link(s_debug)
s_lock.link(s_set_fubar)
t = s_lock.apply_async()
return (t, PermanentTxError, 'Fubar {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id())))
return (t, PermanentTxError, 'Fubar {} {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()), debugstr))
# TODO: A lock should be introduced to ensure that the send status change and the transaction send is atomic.
@@ -407,6 +432,7 @@ def refill_gas(self, recipient_address, chain_str):
"""
chain_spec = ChainSpec.from_chain_str(chain_str)
zero_amount = False
session = SessionBase.create_session()
status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
q = session.query(Otx.tx_hash)
@@ -416,8 +442,11 @@ def refill_gas(self, recipient_address, chain_str):
q = q.filter(TxCache.recipient==recipient_address)
c = q.count()
if c > 0:
session.close()
raise AlreadyFillingGasError(recipient_address)
#session.close()
#raise AlreadyFillingGasError(recipient_address)
logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address))))
zero_amount = True
session.flush()
queue = self.request.delivery_info['routing_key']
@@ -426,10 +455,13 @@ def refill_gas(self, recipient_address, chain_str):
logg.debug('refill gas from provider address {}'.format(c.gas_provider()))
default_nonce = c.w3.eth.getTransactionCount(c.gas_provider(), 'pending')
nonce_generator = NonceOracle(c.gas_provider(), default_nonce)
nonce = nonce_generator.next(session=session)
#nonce = nonce_generator.next(session=session)
nonce = nonce_generator.next_by_task_uuid(self.request.root_id, session=session)
gas_price = c.gas_price()
gas_limit = c.default_gas_limit
refill_amount = c.refill_amount()
refill_amount = 0
if not zero_amount:
refill_amount = c.refill_amount()
logg.debug('tx send gas price {} nonce {}'.format(gas_price, nonce))
# create and sign transaction
@@ -475,6 +507,7 @@ def refill_gas(self, recipient_address, chain_str):
queue=queue,
)
celery.group(s_tx_cache, s_status)()
return tx_send_gas_signed['raw']
@@ -554,6 +587,21 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
return tx_hash_hex
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
def reserve_nonce(self, chained_input, address=None):
session = SessionBase.create_session()
if address == None:
address = chained_input
root_id = self.request.root_id
nonce = NonceReservation.next(address, root_id)
session.close()
return chained_input
@celery_app.task(bind=True, throws=(web3.exceptions.TransactionNotFound,), base=CriticalWeb3Task)
def sync_tx(self, tx_hash_hex, chain_str):
"""Force update of network status of a simgle transaction

118
apps/cic-eth/cic_eth/eth/util.py
View File

@@ -3,10 +3,11 @@ import logging
import sha3
import web3
# third-party imports
# external imports
from rlp import decode as rlp_decode
from rlp import encode as rlp_encode
from eth_keys import KeyAPI
from chainlib.eth.tx import unpack
logg = logging.getLogger()
@@ -22,64 +23,65 @@ field_debugs = [
's',
]
unpack_signed_raw_tx = unpack
def unpack_signed_raw_tx(tx_raw_bytes, chain_id):
d = rlp_decode(tx_raw_bytes)
logg.debug('decoding using chain id {}'.format(chain_id))
j = 0
for i in d:
logg.debug('decoded {}: {}'.format(field_debugs[j], i.hex()))
j += 1
vb = chain_id
if chain_id != 0:
v = int.from_bytes(d[6], 'big')
vb = v - (chain_id * 2) - 35
while len(d[7]) < 32:
d[7] = b'\x00' + d[7]
while len(d[8]) < 32:
d[8] = b'\x00' + d[8]
s = b''.join([d[7], d[8], bytes([vb])])
so = KeyAPI.Signature(signature_bytes=s)
h = sha3.keccak_256()
h.update(rlp_encode(d))
signed_hash = h.digest()
d[6] = chain_id
d[7] = b''
d[8] = b''
h = sha3.keccak_256()
h.update(rlp_encode(d))
unsigned_hash = h.digest()
p = so.recover_public_key_from_msg_hash(unsigned_hash)
a = p.to_checksum_address()
logg.debug('decoded recovery byte {}'.format(vb))
logg.debug('decoded address {}'.format(a))
logg.debug('decoded signed hash {}'.format(signed_hash.hex()))
logg.debug('decoded unsigned hash {}'.format(unsigned_hash.hex()))
to = d[3].hex() or None
if to != None:
to = web3.Web3.toChecksumAddress('0x' + to)
return {
'from': a,
'nonce': int.from_bytes(d[0], 'big'),
'gasPrice': int.from_bytes(d[1], 'big'),
'gas': int.from_bytes(d[2], 'big'),
'to': to,
'value': int.from_bytes(d[4], 'big'),
'data': '0x' + d[5].hex(),
'v': chain_id,
'r': '0x' + s[:32].hex(),
's': '0x' + s[32:64].hex(),
'chainId': chain_id,
'hash': '0x' + signed_hash.hex(),
'hash_unsigned': '0x' + unsigned_hash.hex(),
}
#def unpack_signed_raw_tx(tx_raw_bytes, chain_id):
# d = rlp_decode(tx_raw_bytes)
#
# logg.debug('decoding {} using chain id {}'.format(tx_raw_bytes.hex(), chain_id))
# j = 0
# for i in d:
# logg.debug('decoded {}: {}'.format(field_debugs[j], i.hex()))
# j += 1
# vb = chain_id
# if chain_id != 0:
# v = int.from_bytes(d[6], 'big')
# vb = v - (chain_id * 2) - 35
# while len(d[7]) < 32:
# d[7] = b'\x00' + d[7]
# while len(d[8]) < 32:
# d[8] = b'\x00' + d[8]
# s = b''.join([d[7], d[8], bytes([vb])])
# so = KeyAPI.Signature(signature_bytes=s)
#
# h = sha3.keccak_256()
# h.update(rlp_encode(d))
# signed_hash = h.digest()
#
# d[6] = chain_id
# d[7] = b''
# d[8] = b''
#
# h = sha3.keccak_256()
# h.update(rlp_encode(d))
# unsigned_hash = h.digest()
#
# p = so.recover_public_key_from_msg_hash(unsigned_hash)
# a = p.to_checksum_address()
# logg.debug('decoded recovery byte {}'.format(vb))
# logg.debug('decoded address {}'.format(a))
# logg.debug('decoded signed hash {}'.format(signed_hash.hex()))
# logg.debug('decoded unsigned hash {}'.format(unsigned_hash.hex()))
#
# to = d[3].hex() or None
# if to != None:
# to = web3.Web3.toChecksumAddress('0x' + to)
#
# return {
# 'from': a,
# 'nonce': int.from_bytes(d[0], 'big'),
# 'gasPrice': int.from_bytes(d[1], 'big'),
# 'gas': int.from_bytes(d[2], 'big'),
# 'to': to,
# 'value': int.from_bytes(d[4], 'big'),
# 'data': '0x' + d[5].hex(),
# 'v': chain_id,
# 'r': '0x' + s[:32].hex(),
# 's': '0x' + s[32:64].hex(),
# 'chainId': chain_id,
# 'hash': '0x' + signed_hash.hex(),
# 'hash_unsigned': '0x' + unsigned_hash.hex(),
# }
def unpack_signed_raw_tx_hex(tx_raw_hex, chain_id):