From 7ac6e6666c195d1d7403d624f30d09c91247b967 Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 19 Oct 2021 14:35:22 +0200 Subject: [PATCH 1/6] Uncomment tests --- tests/test_query.py | 398 ++++++++++++++++++++++---------------------- 1 file changed, 199 insertions(+), 199 deletions(-) diff --git a/tests/test_query.py b/tests/test_query.py index eb8896c..9d43b43 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -30,205 +30,205 @@ logg = logging.getLogger() class TestTxQuery(TestTxBase): -# def test_get_tx(self): -# tx = get_tx(self.chain_spec, self.tx_hash) -# expected_keys = [ -# 'otx_id', -# 'status', -# 'signed_tx', -# 'nonce', -# ] -# for k in tx.keys(): -# expected_keys.remove(k) -# -# self.assertEqual(len(expected_keys), 0) -# -# -# def test_nonce_tx(self): -# -# nonce_hashes = [self.tx_hash] -# tx_hash = add_0x(os.urandom(32).hex()) -# signed_tx = add_0x(os.urandom(128).hex()) -# create( -# self.chain_spec, -# 42, -# self.alice, -# tx_hash, -# signed_tx, -# session=self.session, -# ) -# txc = TxCache( -# tx_hash, -# self.alice, -# self.bob, -# self.foo_token, -# self.bar_token, -# self.from_value, -# self.to_value, -# session=self.session, -# ) -# self.session.add(txc) -# self.session.commit() -# -# nonce_hashes.append(tx_hash) -# -# tx_hash = add_0x(os.urandom(32).hex()) -# signed_tx = add_0x(os.urandom(128).hex()) -# create( -# self.chain_spec, -# 41, -# self.alice, -# tx_hash, -# signed_tx, -# session=self.session, -# ) -# txc = TxCache( -# tx_hash, -# self.alice, -# self.bob, -# self.foo_token, -# self.bar_token, -# self.from_value, -# self.to_value, -# session=self.session, -# ) -# self.session.add(txc) -# -# txs = get_nonce_tx_cache(self.chain_spec, 42, self.alice) -# self.assertEqual(len(txs.keys()), 2) -# -# for h in nonce_hashes: -# self.assertTrue(strip_0x(h) in txs) -# -# -# def test_paused_tx_cache(self): -# set_waitforgas(self.chain_spec, self.tx_hash) -# -# tx_hash = add_0x(os.urandom(32).hex()) -# signed_tx = add_0x(os.urandom(128).hex()) -# create( -# self.chain_spec, -# 43, -# self.alice, -# tx_hash, -# signed_tx, -# session=self.session, -# ) -# txc = TxCache( -# tx_hash, -# self.alice, -# self.bob, -# self.foo_token, -# self.bar_token, -# self.from_value, -# self.to_value, -# session=self.session, -# ) -# self.session.add(txc) -# self.session.commit() -# -# txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, sender=self.alice, session=self.session) -# self.assertEqual(len(txs.keys()), 1) -# -# txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, session=self.session) -# self.assertEqual(len(txs.keys()), 1) -# -# tx_hash = add_0x(os.urandom(32).hex()) -# signed_tx = add_0x(os.urandom(128).hex()) -# create( -# self.chain_spec, -# 42, -# self.bob, -# tx_hash, -# signed_tx, -# session=self.session, -# ) -# txc = TxCache( -# tx_hash, -# self.bob, -# self.alice, -# self.bar_token, -# self.foo_token, -# self.to_value, -# self.from_value, -# session=self.session, -# ) -# self.session.add(txc) -# self.session.commit() -# -# txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, session=self.session) -# self.assertEqual(len(txs.keys()), 1) -# -# set_waitforgas(self.chain_spec, tx_hash) -# self.session.commit() -# -# txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, session=self.session) -# self.assertEqual(len(txs.keys()), 2) -# -# txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, sender=self.bob, session=self.session) -# self.assertEqual(len(txs.keys()), 1) -# -# -# def test_count(self): -# for i in range(3): -# tx_hash = add_0x(os.urandom(32).hex()) -# signed_tx = add_0x(os.urandom(128).hex()) -# create( -# self.chain_spec, -# i, -# self.alice, -# tx_hash, -# signed_tx, -# session=self.session, -# ) -# txc = TxCache( -# tx_hash, -# self.alice, -# self.bob, -# self.foo_token, -# self.bar_token, -# self.from_value, -# self.to_value, -# session=self.session, -# ) -# self.session.add(txc) -# set_ready(self.chain_spec, tx_hash, session=self.session) -# set_reserved(self.chain_spec, tx_hash, session=self.session) -# if i > 0: -# set_sent(self.chain_spec, tx_hash, session=self.session) -# if i == 2: -# set_final(self.chain_spec, tx_hash, session=self.session) -# -# tx_hash = add_0x(os.urandom(32).hex()) -# signed_tx = add_0x(os.urandom(128).hex()) -# create( -# self.chain_spec, -# i, -# self.bob, -# tx_hash, -# signed_tx, -# session=self.session, -# ) -# txc = TxCache( -# tx_hash, -# self.bob, -# self.carol, -# self.foo_token, -# self.bar_token, -# self.from_value, -# self.to_value, -# session=self.session, -# ) -# -# self.session.add(txc) -# set_ready(self.chain_spec, tx_hash, session=self.session) -# set_reserved(self.chain_spec, tx_hash, session=self.session) -# set_sent(self.chain_spec, tx_hash, session=self.session) -# self.session.commit() -# -# self.assertEqual(count_tx(self.chain_spec, status=StatusBits.IN_NETWORK | StatusBits.FINAL, status_target=StatusBits.IN_NETWORK), 2) -# self.assertEqual(count_tx(self.chain_spec, address=self.alice, status=StatusBits.IN_NETWORK | StatusBits.FINAL, status_target=StatusBits.IN_NETWORK), 1) -# + def test_get_tx(self): + tx = get_tx(self.chain_spec, self.tx_hash) + expected_keys = [ + 'otx_id', + 'status', + 'signed_tx', + 'nonce', + ] + for k in tx.keys(): + expected_keys.remove(k) + + self.assertEqual(len(expected_keys), 0) + + + def test_nonce_tx(self): + + nonce_hashes = [self.tx_hash] + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + 42, + self.alice, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.alice, + self.bob, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + self.session.add(txc) + self.session.commit() + + nonce_hashes.append(tx_hash) + + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + 41, + self.alice, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.alice, + self.bob, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + self.session.add(txc) + + txs = get_nonce_tx_cache(self.chain_spec, 42, self.alice) + self.assertEqual(len(txs.keys()), 2) + + for h in nonce_hashes: + self.assertTrue(strip_0x(h) in txs) + + + def test_paused_tx_cache(self): + set_waitforgas(self.chain_spec, self.tx_hash) + + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + 43, + self.alice, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.alice, + self.bob, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + self.session.add(txc) + self.session.commit() + + txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, sender=self.alice, session=self.session) + self.assertEqual(len(txs.keys()), 1) + + txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, session=self.session) + self.assertEqual(len(txs.keys()), 1) + + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + 42, + self.bob, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.bob, + self.alice, + self.bar_token, + self.foo_token, + self.to_value, + self.from_value, + session=self.session, + ) + self.session.add(txc) + self.session.commit() + + txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, session=self.session) + self.assertEqual(len(txs.keys()), 1) + + set_waitforgas(self.chain_spec, tx_hash) + self.session.commit() + + txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, session=self.session) + self.assertEqual(len(txs.keys()), 2) + + txs = get_paused_tx_cache(self.chain_spec, status=StatusBits.GAS_ISSUES, sender=self.bob, session=self.session) + self.assertEqual(len(txs.keys()), 1) + + + def test_count(self): + for i in range(3): + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + i, + self.alice, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.alice, + self.bob, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + self.session.add(txc) + set_ready(self.chain_spec, tx_hash, session=self.session) + set_reserved(self.chain_spec, tx_hash, session=self.session) + if i > 0: + set_sent(self.chain_spec, tx_hash, session=self.session) + if i == 2: + set_final(self.chain_spec, tx_hash, session=self.session) + + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + i, + self.bob, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.bob, + self.carol, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + + self.session.add(txc) + set_ready(self.chain_spec, tx_hash, session=self.session) + set_reserved(self.chain_spec, tx_hash, session=self.session) + set_sent(self.chain_spec, tx_hash, session=self.session) + self.session.commit() + + self.assertEqual(count_tx(self.chain_spec, status=StatusBits.IN_NETWORK | StatusBits.FINAL, status_target=StatusBits.IN_NETWORK), 2) + self.assertEqual(count_tx(self.chain_spec, address=self.alice, status=StatusBits.IN_NETWORK | StatusBits.FINAL, status_target=StatusBits.IN_NETWORK), 1) + def test_account_tx(self): From d01ed18f12ea16a0f0e5bf6efd27f35fda6775bc Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 20 Oct 2021 09:24:08 +0200 Subject: [PATCH 2/6] Rehabilitate sql query tests --- requirements.txt | 2 +- tests/chainqueue_base.py | 21 +++++++++++---------- tests/test_query.py | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 90b0b73..37dc62a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ alembic==1.4.2 SQLAlchemy==1.3.20 confini>=0.4.1a1,<0.5.0 pyxdg~=0.27 -chainlib~=0.0.10a3 +chainlib~=0.0.10a5 diff --git a/tests/chainqueue_base.py b/tests/chainqueue_base.py index 00129e5..7a36b10 100644 --- a/tests/chainqueue_base.py +++ b/tests/chainqueue_base.py @@ -66,16 +66,17 @@ class TestTxBase(TestOtxBase): self.to_value = 13 backend = SQLBackend(self.db.dsn) - tx = { - 'hash': self.tx_hash, - 'from': self.alice, - 'to': self.bob, - 'source_token': self.foo_token, - 'destination_token': self.bar_token, - 'from_value': self.from_value, - 'to_value': self.to_value, - } - backend.cache(tx, session=self.session) + txc = TxCache( + self.tx_hash, + self.alice, + self.bob, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + self.session.add(txc) self.session.commit() otx = Otx.load(self.tx_hash) diff --git a/tests/test_query.py b/tests/test_query.py index 9d43b43..7849cc2 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -227,7 +227,7 @@ class TestTxQuery(TestTxBase): self.session.commit() self.assertEqual(count_tx(self.chain_spec, status=StatusBits.IN_NETWORK | StatusBits.FINAL, status_target=StatusBits.IN_NETWORK), 2) - self.assertEqual(count_tx(self.chain_spec, address=self.alice, status=StatusBits.IN_NETWORK | StatusBits.FINAL, status_target=StatusBits.IN_NETWORK), 1) + self.assertEqual(count_tx(self.chain_spec, sender=self.alice, status=StatusBits.IN_NETWORK | StatusBits.FINAL, status_target=StatusBits.IN_NETWORK), 1) def test_account_tx(self): From fd8f48539d18c99e70a9b60f0320dfaab447518a Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 15 Nov 2021 06:54:25 +0100 Subject: [PATCH 3/6] Fix missing normalizer symbol --- chainqueue/encode.py | 1 + chainqueue/sql/tx.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/chainqueue/encode.py b/chainqueue/encode.py index dac09e2..8c21e47 100644 --- a/chainqueue/encode.py +++ b/chainqueue/encode.py @@ -1,2 +1,3 @@ # external imports from chainlib.encode import TxHexNormalizer +TxNormalizer = TxHexNormalizer diff --git a/chainqueue/sql/tx.py b/chainqueue/sql/tx.py index c0ccabb..b090b5f 100644 --- a/chainqueue/sql/tx.py +++ b/chainqueue/sql/tx.py @@ -17,7 +17,8 @@ from chainqueue.db.enum import ( ) from chainqueue.error import TxStateChangeError -logg = logging.getLogger().getChild(__name__) +#logg = logging.getLogger(__name__) +logg = logging.getLogger() def create(chain_spec, nonce, holder_address, tx_hash, signed_tx, obsolete_predecessors=True, session=None): From 3dc9dffb26ff1eeabcbbb68a90566adc02852f79 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 15 Nov 2021 14:23:55 +0100 Subject: [PATCH 4/6] Upgrade deps, bump version --- requirements.txt | 8 ++++---- setup.cfg | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 37dc62a..e500340 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ pysha3==1.0.2 -hexathon~=0.0.1a8 -leveldir~=0.0.2 +hexathon~=0.1.0 +leveldir~=0.1.0 alembic==1.4.2 SQLAlchemy==1.3.20 -confini>=0.4.1a1,<0.5.0 +confini~=0.5.1 pyxdg~=0.27 -chainlib~=0.0.10a5 +chainlib~=0.0.12 diff --git a/setup.cfg b/setup.cfg index f096792..2353284 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = chainqueue -version = 0.0.6a1 +version = 0.0.6a3 description = Generic blockchain transaction queue control author = Louis Holbrook author_email = dev@holbrook.no From 176efd69b3795eab20874e0fd59b9f0c8796cf7a Mon Sep 17 00:00:00 2001 From: William Luke Date: Mon, 7 Feb 2022 14:55:37 +0300 Subject: [PATCH 5/6] chore: bump leveldir --- requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e500340..577683e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pysha3==1.0.2 hexathon~=0.1.0 -leveldir~=0.1.0 +leveldir~=0.3.0 alembic==1.4.2 SQLAlchemy==1.3.20 confini~=0.5.1 diff --git a/setup.cfg b/setup.cfg index 2353284..d4f3ba2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = chainqueue -version = 0.0.6a3 +version = 0.0.6a4 description = Generic blockchain transaction queue control author = Louis Holbrook author_email = dev@holbrook.no From 7a36cd1e417f4560f4fb861ffc26a09ac956409a Mon Sep 17 00:00:00 2001 From: William Luke Date: Mon, 7 Feb 2022 15:35:01 +0300 Subject: [PATCH 6/6] feat: add `get_latest_txs` query --- .gitignore | 1 + chainqueue/sql/query.py | 64 +++++++++++++++++++++++++++++++++++++++++ test_requirements.txt | 1 - tests/test_query.py | 59 +++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 1 deletion(-) delete mode 100644 test_requirements.txt diff --git a/.gitignore b/.gitignore index 399d953..a2de8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ build/ dist/ *.html *.egg-info/ +.venv diff --git a/chainqueue/sql/query.py b/chainqueue/sql/query.py index 1454a85..1cc28d5 100644 --- a/chainqueue/sql/query.py +++ b/chainqueue/sql/query.py @@ -462,6 +462,70 @@ def get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, count return txs +def get_latest_txs(chain_spec, count=10, since=None, until=None, status=None, not_status=None, status_target=None, session=None): + """Returns the lastest local queue transactions + + The since parameter effect depends on its type. Results are returned inclusive of the given parameter condition. + + :param chain_spec: Chain spec for transaction network + :type chain_spec: chainlib.chain.ChainSpec + :param status: Only include transactions where the given status bits are set + :type status: chainqueue.enum.StatusEnum + :param not_status: Only include transactions where the given status bits are not set + :type not_status: chainqueue.enum.StatusEnum + :param status_target: Only include transaction where the status argument is exact match + :type status_target: chainqueue.enum.StatusEnum + :param session: Backend state integrity session + :type session: varies + :raises ValueError: If address is set to be neither sender nor recipient + :returns: Transactions + :rtype: dict, with transaction hash as key, signed raw transaction as value + """ + txs = {} + + session = SessionBase.bind_session(session) + + try: + filter_offset = sql_range_filter(session, criteria=since) + filter_limit = sql_range_filter(session, criteria=until) + except NotLocalTxError as e: + logg.error('query build failed: {}'.format(e)) + return {} + + q = session.query(Otx) + q = q.join(TxCache) + + if filter_offset != None: + if filter_offset[0] == 'id': + q = q.filter(Otx.id>=filter_offset[1]) + elif filter_offset[0] == 'date': + q = q.filter(Otx.date_created>=filter_offset[1]) + + if filter_limit != None: + if filter_limit[0] == 'id': + q = q.filter(Otx.id<=filter_limit[1]) + elif filter_limit[0] == 'date': + q = q.filter(Otx.date_created<=filter_limit[1]) + + if status != None: + if status_target == None: + status_target = status + q = q.filter(Otx.status.op('&')(status)==status_target) + + if not_status != None: + q = q.filter(Otx.status.op('&')(not_status)==0) + + q = q.order_by(Otx.nonce.asc(), Otx.date_created.asc()).limit(count) + results = q.all() + for r in results: + if txs.get(r.tx_hash) != None: + logg.debug('tx {} already recorded'.format(r.tx_hash)) + continue + txs[r.tx_hash] = r.signed_tx + + SessionBase.release_session(session) + + return txs def count_tx(chain_spec, sender=None, status=None, status_target=None, session=None): """Count transaction records matching the given criteria. diff --git a/test_requirements.txt b/test_requirements.txt deleted file mode 100644 index 96d1a07..0000000 --- a/test_requirements.txt +++ /dev/null @@ -1 +0,0 @@ -chainlib==0.0.9a10 diff --git a/tests/test_query.py b/tests/test_query.py index 7849cc2..dd21578 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -345,6 +345,65 @@ class TestTxQuery(TestTxBase): txs = get_account_tx(self.chain_spec, self.alice, as_sender=True, as_recipient=False, not_status=StatusBits.QUEUED, status=StatusBits.IN_NETWORK, session=self.session) self.assertEqual(len(txs.keys()), 1) + def test_latest_txs(self): + + nonce_hashes = [self.tx_hash] + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + 42, + self.alice, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.alice, + self.bob, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + self.session.add(txc) + self.session.commit() + + nonce_hashes.append(tx_hash) + + time_between = datetime.datetime.utcnow() + + tx_hash = add_0x(os.urandom(32).hex()) + signed_tx = add_0x(os.urandom(128).hex()) + create( + self.chain_spec, + 41, + self.alice, + tx_hash, + signed_tx, + session=self.session, + ) + txc = TxCache( + tx_hash, + self.alice, + self.bob, + self.foo_token, + self.bar_token, + self.from_value, + self.to_value, + session=self.session, + ) + self.session.add(txc) + + nonce_hashes.append(tx_hash) + + txs = get_latest_txs(self.chain_spec, session=self.session) + self.assertEqual(len(txs.keys()), 3) + + txs = get_latest_txs(self.chain_spec, count=1, session=self.session) + self.assertEqual(len(txs.keys()), 1) if __name__ == '__main__': unittest.main()