kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Syncer refactor

Browse Source
This commit is contained in:
Louis Holbrook
2021-03-01 20:15:17 +00:00
parent 451079d004
commit 5001113267
56 changed files with 833 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
apps/cic-eth/cic_eth/db/migrations/default/versions/ec40ac0974c1_add_chain_syncer.py Normal file
View File

@@ -0,0 +1,28 @@
"""Add chain syncer
Revision ID: ec40ac0974c1
Revises: 6ac7a1dadc46
Create Date: 2021-02-23 06:10:19.246304
"""
from alembic import op
import sqlalchemy as sa
from chainsyncer.db.migrations.sqlalchemy import (
chainsyncer_upgrade,
chainsyncer_downgrade,
)
# revision identifiers, used by Alembic.
revision = 'ec40ac0974c1'
down_revision = '6ac7a1dadc46'
branch_labels = None
depends_on = None
def upgrade():
chainsyncer_upgrade(0, 0, 1)
def downgrade():
chainsyncer_downgrade(0, 0, 1)

28
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/ec40ac0974c1_add_chain_syncer.py Normal file
View File

@@ -0,0 +1,28 @@
"""Add chain syncer
Revision ID: ec40ac0974c1
Revises: 6ac7a1dadc46
Create Date: 2021-02-23 06:10:19.246304
"""
from alembic import op
import sqlalchemy as sa
from chainsyncer.db.migrations.sqlalchemy import (
chainsyncer_upgrade,
chainsyncer_downgrade,
)
# revision identifiers, used by Alembic.
revision = 'ec40ac0974c1'
down_revision = '6ac7a1dadc46'
branch_labels = None
depends_on = None
def upgrade():
chainsyncer_upgrade(0, 0, 1)
def downgrade():
chainsyncer_downgrade(0, 0, 1)

1
apps/cic-eth/cic_eth/db/models/base.py
View File

@@ -114,7 +114,6 @@ class SessionBase(Model):
@staticmethod
def release_session(session=None):
session.flush()
session_key = str(id(session))
if SessionBase.localsessions.get(session_key) != None:
logg.debug('destroying session {}'.format(session_key))

43
apps/cic-eth/cic_eth/db/models/lock.py
View File

@@ -55,11 +55,9 @@ class Lock(SessionBase):
:returns: New flag state of entry
:rtype: number
"""
localsession = session
if localsession == None:
localsession = SessionBase.create_session()
session = SessionBase.bind_session(session)
q = localsession.query(Lock)
q = session.query(Lock)
#q = q.join(TxCache, isouter=True)
q = q.filter(Lock.address==address)
q = q.filter(Lock.blockchain==chain_str)
@@ -71,7 +69,8 @@ class Lock(SessionBase):
lock.address = address
lock.blockchain = chain_str
if tx_hash != None:
q = localsession.query(Otx)
session.flush()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
otx = q.first()
if otx != None:
@@ -80,12 +79,11 @@ class Lock(SessionBase):
lock.flags |= flags
r = lock.flags
localsession.add(lock)
localsession.commit()
session.add(lock)
session.commit()
SessionBase.release_session(session)
if session == None:
localsession.close()
return r
@@ -110,11 +108,9 @@ class Lock(SessionBase):
:returns: New flag state of entry
:rtype: number
"""
localsession = session
if localsession == None:
localsession = SessionBase.create_session()
session = SessionBase.bind_session(session)
q = localsession.query(Lock)
q = session.query(Lock)
#q = q.join(TxCache, isouter=True)
q = q.filter(Lock.address==address)
q = q.filter(Lock.blockchain==chain_str)
@@ -124,14 +120,13 @@ class Lock(SessionBase):
if lock != None:
lock.flags &= ~flags
if lock.flags == 0:
localsession.delete(lock)
session.delete(lock)
else:
localsession.add(lock)
session.add(lock)
r = lock.flags
localsession.commit()
session.commit()
if session == None:
localsession.close()
SessionBase.release_session(session)
return r
@@ -156,22 +151,20 @@ class Lock(SessionBase):
:rtype: number
"""
localsession = session
if localsession == None:
localsession = SessionBase.create_session()
session = SessionBase.bind_session(session)
q = localsession.query(Lock)
q = session.query(Lock)
#q = q.join(TxCache, isouter=True)
q = q.filter(Lock.address==address)
q = q.filter(Lock.blockchain==chain_str)
q = q.filter(Lock.flags.op('&')(flags)==flags)
lock = q.first()
if session == None:
localsession.close()
r = 0
if lock != None:
r = lock.flags & flags
SessionBase.release_session(session)
return r

52
apps/cic-eth/cic_eth/db/models/nonce.py
View File

@@ -21,12 +21,9 @@ class Nonce(SessionBase):
@staticmethod
def get(address, session=None):
localsession = session
if localsession == None:
localsession = SessionBase.create_session()
session = SessionBase.bind_session(session)
q = localsession.query(Nonce)
q = session.query(Nonce)
q = q.filter(Nonce.address_hex==address)
nonce = q.first()
@@ -34,28 +31,29 @@ class Nonce(SessionBase):
if nonce != None:
nonce_value = nonce.nonce;
if session == None:
localsession.close()
SessionBase.release_session(session)
return nonce_value
@staticmethod
def __get(conn, address):
r = conn.execute("SELECT nonce FROM nonce WHERE address_hex = '{}'".format(address))
def __get(session, address):
r = session.execute("SELECT nonce FROM nonce WHERE address_hex = '{}'".format(address))
nonce = r.fetchone()
session.flush()
if nonce == None:
return None
return nonce[0]
@staticmethod
def __set(conn, address, nonce):
conn.execute("UPDATE nonce set nonce = {} WHERE address_hex = '{}'".format(nonce, address))
def __set(session, address, nonce):
session.execute("UPDATE nonce set nonce = {} WHERE address_hex = '{}'".format(nonce, address))
session.flush()
@staticmethod
def next(address, initial_if_not_exists=0):
def next(address, initial_if_not_exists=0, session=None):
"""Generate next nonce for the given address.
If there is no previous nonce record for the address, the nonce may be initialized to a specified value, or 0 if no value has been given.
@@ -67,20 +65,32 @@ class Nonce(SessionBase):
:returns: Nonce
:rtype: number
"""
conn = Nonce.engine.connect()
session = SessionBase.bind_session(session)
SessionBase.release_session(session)
session.begin_nested()
#conn = Nonce.engine.connect()
if Nonce.transactional:
conn.execute('BEGIN')
conn.execute('LOCK TABLE nonce IN SHARE ROW EXCLUSIVE MODE')
nonce = Nonce.__get(conn, address)
#session.execute('BEGIN')
session.execute('LOCK TABLE nonce IN SHARE ROW EXCLUSIVE MODE')
session.flush()
nonce = Nonce.__get(session, address)
logg.debug('get nonce {} for address {}'.format(nonce, address))
if nonce == None:
nonce = initial_if_not_exists
conn.execute("INSERT INTO nonce (nonce, address_hex) VALUES ({}, '{}')".format(nonce, address))
session.execute("INSERT INTO nonce (nonce, address_hex) VALUES ({}, '{}')".format(nonce, address))
session.flush()
logg.debug('setting default nonce to {} for address {}'.format(nonce, address))
Nonce.__set(conn, address, nonce+1)
if Nonce.transactional:
conn.execute('COMMIT')
conn.close()
Nonce.__set(session, address, nonce+1)
#if Nonce.transactional:
#session.execute('COMMIT')
#session.execute('UNLOCK TABLE nonce')
#conn.close()
session.commit()
session.commit()
SessionBase.release_session(session)
return nonce

45
apps/cic-eth/cic_eth/db/models/otx.py
View File

@@ -79,6 +79,13 @@ class Otx(SessionBase):
return r
def __status_not_set(self, status):
r = not(self.status & status)
if r:
logg.warning('status bit {} not set on {}'.format(status.name, self.tx_hash))
return r
def set_block(self, block, session=None):
"""Set block number transaction was mined in.
@@ -320,6 +327,32 @@ class Otx(SessionBase):
SessionBase.release_session(session)
def dequeue(self, session=None):
"""Marks that a process to execute send attempt is underway
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_not_set(StatusBits.QUEUED):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
raise TxStateChangeError('SENDFAIL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
raise TxStateChangeError('SENDFAIL cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
self.__reset_status(StatusBits.QUEUED, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def minefail(self, block, session=None):
"""Marks that transaction was mined but code execution did not succeed.
@@ -373,18 +406,6 @@ class Otx(SessionBase):
else:
self.__set_status(StatusEnum.OBSOLETED, session)
# if confirmed:
# if self.status != StatusEnum.OBSOLETED:
# logg.warning('CANCELLED must follow OBSOLETED, but had {}'.format(StatusEnum(self.status).name))
# #raise TxStateChangeError('CANCELLED must follow OBSOLETED, but had {}'.format(StatusEnum(self.status).name))
# self.__set_status(StatusEnum.CANCELLED, session)
# elif self.status != StatusEnum.OBSOLETED:
# if self.status > StatusEnum.SENT:
# logg.warning('OBSOLETED must follow PENDING, SENDFAIL or SENT, but had {}'.format(StatusEnum(self.status).name))
# #raise TxStateChangeError('OBSOLETED must follow PENDING, SENDFAIL or SENT, but had {}'.format(StatusEnum(self.status).name))
# self.__set_status(StatusEnum.OBSOLETED, session)
if self.tracing:
self.__state_log(session=session)

11
apps/cic-eth/cic_eth/db/models/role.py
View File

@@ -40,12 +40,14 @@ class AccountRole(SessionBase):
session = SessionBase.bind_session(session)
role = AccountRole.get_role(tag, session)
role = AccountRole.__get_role(tag, session)
r = zero_address
if role != None:
r = role.address_hex
session.flush()
SessionBase.release_session(session)
return r
@@ -63,6 +65,8 @@ class AccountRole(SessionBase):
session = SessionBase.bind_session(session)
role = AccountRole.__get_role(tag, session)
session.flush()
SessionBase.release_session(session)
@@ -74,7 +78,6 @@ class AccountRole(SessionBase):
q = session.query(AccountRole)
q = q.filter(AccountRole.tag==tag)
r = q.first()
session.flush()
return r
@@ -93,10 +96,12 @@ class AccountRole(SessionBase):
"""
session = SessionBase.bind_session(session)
role = AccountRole.get_role(tag, session)
role = AccountRole.__get_role(tag, session)
if role == None:
role = AccountRole(tag)
role.address_hex = address_hex
session.flush()
SessionBase.release_session(session)

30
apps/cic-eth/cic_eth/db/models/tx.py
View File

@@ -85,26 +85,27 @@ class TxCache(SessionBase):
:param tx_hash_new: tx hash to associate the copied entry with
:type tx_hash_new: str, 0x-hex
"""
localsession = SessionBase.bind_session(session)
session = SessionBase.bind_session(session)
q = localsession.query(TxCache)
q = session.query(TxCache)
q = q.join(Otx)
q = q.filter(Otx.tx_hash==tx_hash_original)
txc = q.first()
if txc == None:
SessionBase.release_session(localsession)
SessionBase.release_session(session)
raise NotLocalTxError('original {}'.format(tx_hash_original))
if txc.block_number != None:
SessionBase.release_session(localsession)
SessionBase.release_session(session)
raise TxStateChangeError('cannot clone tx cache of confirmed tx {}'.format(tx_hash_original))
q = localsession.query(Otx)
session.flush()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash_new)
otx = q.first()
if otx == None:
SessionBase.release_session(localsession)
SessionBase.release_session(session)
raise NotLocalTxError('new {}'.format(tx_hash_new))
txc_new = TxCache(
@@ -115,18 +116,21 @@ class TxCache(SessionBase):
txc.destination_token_address,
int(txc.from_value),
int(txc.to_value),
session=session,
)
localsession.add(txc_new)
localsession.commit()
session.add(txc_new)
session.commit()
SessionBase.release_session(localsession)
SessionBase.release_session(session)
def __init__(self, tx_hash, sender, recipient, source_token_address, destination_token_address, from_value, to_value, block_number=None, tx_index=None, session=None):
localsession = SessionBase.bind_session(session)
tx = localsession.query(Otx).filter(Otx.tx_hash==tx_hash).first()
session = SessionBase.bind_session(session)
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
tx = q.first()
if tx == None:
SessionBase.release_session(localsession)
SessionBase.release_session(session)
raise FileNotFoundError('outgoing transaction record unknown {} (add a Tx first)'.format(tx_hash))
self.otx_id = tx.id
@@ -143,5 +147,5 @@ class TxCache(SessionBase):
self.date_updated = self.date_created
self.date_checked = self.date_created
SessionBase.release_session(localsession)
SessionBase.release_session(session)