kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Syncer refactor

Browse Source
This commit is contained in:
Louis Holbrook
2021-03-01 20:15:17 +00:00
parent 451079d004
commit 5001113267
56 changed files with 833 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/cic-eth/tests/unit/api/test_callback.py
View File

@@ -67,7 +67,7 @@ def test_callback_tcp(
logg.debug('recived {} '.format(data))
o = json.loads(echo)
try:
assert o == data
assert o['result'] == data
except Exception as e:
self.exception = e
@@ -130,7 +130,7 @@ def test_callback_redis(
o = json.loads(echo['data'])
logg.debug('recived {} '.format(o))
try:
assert o == data
assert o['result'] == data
except Exception as e:
self.exception = e

8
apps/cic-eth/tests/unit/db/test_role.py
View File

@@ -9,18 +9,18 @@ def test_db_role(
foo = AccountRole.set('foo', eth_empty_accounts[0])
init_database.add(foo)
init_database.commit()
assert AccountRole.get_address('foo') == eth_empty_accounts[0]
assert AccountRole.get_address('foo', init_database) == eth_empty_accounts[0]
bar = AccountRole.set('bar', eth_empty_accounts[1])
init_database.add(bar)
init_database.commit()
assert AccountRole.get_address('bar') == eth_empty_accounts[1]
assert AccountRole.get_address('bar', init_database) == eth_empty_accounts[1]
foo = AccountRole.set('foo', eth_empty_accounts[2])
init_database.add(foo)
init_database.commit()
assert AccountRole.get_address('foo') == eth_empty_accounts[2]
assert AccountRole.get_address('bar') == eth_empty_accounts[1]
assert AccountRole.get_address('foo', init_database) == eth_empty_accounts[2]
assert AccountRole.get_address('bar', init_database) == eth_empty_accounts[1]
tag = AccountRole.role_for(eth_empty_accounts[2])
assert tag == 'foo'

4
apps/cic-eth/tests/unit/db/test_tx.py
View File

@@ -26,7 +26,7 @@ def test_set(
'data': '',
'chainId': 1,
}
(tx_hash, tx_signed) = sign_tx(tx_def, 'Foo:1')
(tx_hash, tx_signed) = sign_tx(tx_def, 'foo:bar:1')
otx = Otx(
tx_def['nonce'],
tx_def['from'],
@@ -82,7 +82,7 @@ def test_clone(
'data': '',
'chainId': 1,
}
(tx_hash, tx_signed) = sign_tx(tx_def, 'Foo:1')
(tx_hash, tx_signed) = sign_tx(tx_def, 'foo:bar:1')
otx = Otx(
tx_def['nonce'],
tx_def['from'],

6
apps/cic-eth/tests/unit/eth/test_raw.py
View File

@@ -14,11 +14,11 @@ def test_unpack(
'gas': 21000,
'gasPrice': 200000000,
'data': '0x',
'chainId': 8995,
'chainId': 42,
}
(tx_hash, tx_raw) = sign_tx(tx, 'Foo:8995')
(tx_hash, tx_raw) = sign_tx(tx, 'foo:bar:42')
tx_recovered = unpack_signed_raw_tx(bytes.fromhex(tx_raw[2:]), 8995)
tx_recovered = unpack_signed_raw_tx(bytes.fromhex(tx_raw[2:]), 42)
assert tx_hash == tx_recovered['hash']

43
apps/cic-eth/tests/unit/sync/test_head.py
View File

@@ -1,43 +0,0 @@
# standard imports
import logging
# local imports
from cic_eth.sync.head import HeadSyncer
from cic_eth.sync.backend import SyncerBackend
logg = logging.getLogger()
def test_head(
init_rpc,
init_database,
init_eth_tester,
mocker,
eth_empty_accounts,
):
#backend = SyncBackend(eth_empty_accounts[0], 'foo')
block_number = init_rpc.w3.eth.blockNumber
backend = SyncerBackend.live('foo:666', block_number)
syncer = HeadSyncer(backend)
#init_eth_tester.mine_block()
nonce = init_rpc.w3.eth.getTransactionCount(init_rpc.w3.eth.accounts[0], 'pending')
logg.debug('nonce {}'.format(nonce))
tx = {
'from': init_rpc.w3.eth.accounts[0],
'to': eth_empty_accounts[0],
'value': 404,
'gas': 21000,
'gasPrice': init_rpc.w3.eth.gasPrice,
'nonce': nonce,
}
tx_hash_one = init_rpc.w3.eth.sendTransaction(tx)
block_number = init_rpc.w3.eth.blockNumber
backend.set(block_number, 0)
b = syncer.get(init_rpc.w3)
tx = init_rpc.w3.eth.getTransactionByBlock(b[0], 0)
assert tx.hash.hex() == tx_hash_one.hex()

194
apps/cic-eth/tests/unit/sync/test_history.py
View File

@@ -1,194 +0,0 @@
# standard imports
import logging
# third-party imports
import pytest
from web3.exceptions import BlockNotFound
from cic_registry import CICRegistry
# local imports
from cic_eth.sync.history import HistorySyncer
from cic_eth.sync.head import HeadSyncer
#from cic_eth.sync import Syncer
from cic_eth.db.models.otx import OtxSync
from cic_eth.db.models.base import SessionBase
from cic_eth.sync.backend import SyncerBackend
logg = logging.getLogger()
class FinishedError(Exception):
pass
class DebugFilter:
def __init__(self, address):
self.txs = []
self.monitor_to_address = address
def filter(self, w3, tx, rcpt, chain_spec):
logg.debug('sync filter {}'.format(tx['hash'].hex()))
if tx['to'] == self.monitor_to_address:
self.txs.append(tx)
# hack workaround, latest block hash not found in eth_tester for some reason
if len(self.txs) == 2:
raise FinishedError('intentionally finished on tx {}'.format(tx))
def test_history(
init_rpc,
init_database,
init_eth_tester,
#celery_session_worker,
eth_empty_accounts,
):
nonce = init_rpc.w3.eth.getTransactionCount(init_rpc.w3.eth.accounts[0], 'pending')
logg.debug('nonce {}'.format(nonce))
tx = {
'from': init_rpc.w3.eth.accounts[0],
'to': eth_empty_accounts[0],
'value': 404,
'gas': 21000,
'gasPrice': init_rpc.w3.eth.gasPrice,
'nonce': nonce,
}
tx_hash_one = init_rpc.w3.eth.sendTransaction(tx)
nonce = init_rpc.w3.eth.getTransactionCount(init_rpc.w3.eth.accounts[0], 'pending')
logg.debug('nonce {}'.format(nonce))
tx = {
'from': init_rpc.w3.eth.accounts[1],
'to': eth_empty_accounts[0],
'value': 404,
'gas': 21000,
'gasPrice': init_rpc.w3.eth.gasPrice,
'nonce': nonce,
}
tx_hash_two = init_rpc.w3.eth.sendTransaction(tx)
init_eth_tester.mine_block()
block_number = init_rpc.w3.eth.blockNumber
live_syncer = SyncerBackend.live('foo:666', 0)
HeadSyncer(live_syncer)
history_syncers = SyncerBackend.resume('foo:666', block_number)
for history_syncer in history_syncers:
logg.info('history syncer start {} target {}'.format(history_syncer.start(), history_syncer.target()))
backend = history_syncers[0]
syncer = HistorySyncer(backend)
fltr = DebugFilter(eth_empty_accounts[0])
syncer.filter.append(fltr.filter)
logg.debug('have txs {} {}'.format(tx_hash_one.hex(), tx_hash_two.hex()))
try:
syncer.loop(0.1)
except FinishedError:
pass
except BlockNotFound as e:
logg.error('the last block given in loop does not seem to exist :/ {}'.format(e))
check_hashes = []
for h in fltr.txs:
check_hashes.append(h['hash'].hex())
assert tx_hash_one.hex() in check_hashes
assert tx_hash_two.hex() in check_hashes
def test_history_multiple(
init_rpc,
init_database,
init_eth_tester,
#celery_session_worker,
eth_empty_accounts,
):
block_number = init_rpc.w3.eth.blockNumber
live_syncer = SyncerBackend.live('foo:666', block_number)
HeadSyncer(live_syncer)
nonce = init_rpc.w3.eth.getTransactionCount(init_rpc.w3.eth.accounts[0], 'pending')
logg.debug('nonce {}'.format(nonce))
tx = {
'from': init_rpc.w3.eth.accounts[0],
'to': eth_empty_accounts[0],
'value': 404,
'gas': 21000,
'gasPrice': init_rpc.w3.eth.gasPrice,
'nonce': nonce,
}
tx_hash_one = init_rpc.w3.eth.sendTransaction(tx)
init_eth_tester.mine_block()
block_number = init_rpc.w3.eth.blockNumber
history_syncers = SyncerBackend.resume('foo:666', block_number)
for history_syncer in history_syncers:
logg.info('halfway history syncer start {} target {}'.format(history_syncer.start(), history_syncer.target()))
live_syncer = SyncerBackend.live('foo:666', block_number)
HeadSyncer(live_syncer)
nonce = init_rpc.w3.eth.getTransactionCount(init_rpc.w3.eth.accounts[0], 'pending')
logg.debug('nonce {}'.format(nonce))
tx = {
'from': init_rpc.w3.eth.accounts[1],
'to': eth_empty_accounts[0],
'value': 404,
'gas': 21000,
'gasPrice': init_rpc.w3.eth.gasPrice,
'nonce': nonce,
}
tx_hash_two = init_rpc.w3.eth.sendTransaction(tx)
init_eth_tester.mine_block()
block_number = init_rpc.w3.eth.blockNumber
history_syncers = SyncerBackend.resume('foo:666', block_number)
live_syncer = SyncerBackend.live('foo:666', block_number)
HeadSyncer(live_syncer)
for history_syncer in history_syncers:
logg.info('history syncer start {} target {}'.format(history_syncer.start(), history_syncer.target()))
assert len(history_syncers) == 2
backend = history_syncers[0]
syncer = HistorySyncer(backend)
fltr = DebugFilter(eth_empty_accounts[0])
syncer.filter.append(fltr.filter)
try:
syncer.loop(0.1)
except FinishedError:
pass
except BlockNotFound as e:
logg.error('the last block given in loop does not seem to exist :/ {}'.format(e))
check_hashes = []
for h in fltr.txs:
check_hashes.append(h['hash'].hex())
assert tx_hash_one.hex() in check_hashes
backend = history_syncers[1]
syncer = HistorySyncer(backend)
fltr = DebugFilter(eth_empty_accounts[0])
syncer.filter.append(fltr.filter)
try:
syncer.loop(0.1)
except FinishedError:
pass
except BlockNotFound as e:
logg.error('the last block given in loop does not seem to exist :/ {}'.format(e))
check_hashes = []
for h in fltr.txs:
check_hashes.append(h['hash'].hex())
assert tx_hash_two.hex() in check_hashes
history_syncers = SyncerBackend.resume('foo:666', block_number)
assert len(history_syncers) == 0

79
apps/cic-eth/tests/unit/sync/test_syncer_scratch.py
View File

@@ -1,79 +0,0 @@
# third-party imports
import pytest
# local imports
from cic_eth.db.models.sync import BlockchainSync
from cic_eth.sync.backend import SyncerBackend
def test_scratch(
init_database,
):
with pytest.raises(ValueError):
s = SyncerBackend('Testchain:666', 13)
syncer = SyncerBackend.live('Testchain:666', 13)
s = SyncerBackend('Testchain:666', syncer.object_id)
def test_live(
init_database,
):
s = SyncerBackend.live('Testchain:666', 13)
s.connect()
assert s.db_object.target() == None
s.disconnect()
assert s.get() == (13, 0)
s.set(14, 1)
assert s.get() == (14, 1)
def test_resume(
init_database,
):
live = SyncerBackend.live('Testchain:666', 13)
live.set(13, 2)
resumes = SyncerBackend.resume('Testchain:666', 26)
assert len(resumes) == 1
resume = resumes[0]
assert resume.get() == (13, 2)
resume.set(13, 4)
assert resume.get() == (13, 4)
assert resume.start() == (13, 2)
assert resume.target() == 26
def test_unsynced(
init_database,
):
live = SyncerBackend.live('Testchain:666', 13)
live.set(13, 2)
resumes = SyncerBackend.resume('Testchain:666', 26)
live = SyncerBackend.live('Testchain:666', 26)
resumes[0].set(18, 12)
resumes = SyncerBackend.resume('Testchain:666', 42)
assert len(resumes) == 2
assert resumes[0].start() == (13, 2)
assert resumes[0].get() == (18, 12)
assert resumes[0].target() == 26
assert resumes[1].start() == (26, 0)
assert resumes[1].get() == (26, 0)
assert resumes[1].target() == 42

14
apps/cic-eth/tests/unit/util/test_unpack.py
View File

@@ -7,24 +7,22 @@ def test_unpack(
):
tx = {
'nonce': 42,
'nonce': 13,
'from': init_w3_conn.eth.accounts[0],
'to': init_w3_conn.eth.accounts[1],
'data': '0xdeadbeef',
'value': 1024,
'gas': 23000,
'gasPrice': 1422521,
'chainId': 1337,
'chainId': 42,
}
(tx_hash, tx_signed) = sign_tx(tx, 'Foo:1337')
(tx_hash, tx_signed) = sign_tx(tx, 'foo:bar:42')
tx_unpacked = unpack_signed_raw_tx_hex(tx_signed, 1337)
tx_unpacked = unpack_signed_raw_tx_hex(tx_signed, 42)
for k in tx.keys():
assert tx[k] == tx_unpacked[k]
tx_str = tx_hex_string(tx_signed, 1337)
assert tx_str == 'tx nonce 42 from 0x7E5F4552091A69125d5DfCb7b8C2659029395Bdf to 0x2B5AD5c4795c026514f8317c7a215E218DcCD6cF hash 0xe5aba32b1a7255d035faccb70cd8bb92c8c4a2f6bbea3f655bc5a8b802bbaa91'
tx_str = tx_hex_string(tx_signed, 42)
assert tx_str == 'tx nonce 13 from 0x7E5F4552091A69125d5DfCb7b8C2659029395Bdf to 0x2B5AD5c4795c026514f8317c7a215E218DcCD6cF hash 0x23ba3c2b400fbddcacc77d99644bfb17ac4653a69bfa46e544801fbd841b8f1e'