From a9258c3085789e7223f2f94a35e08fccaea29576 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Sun, 4 Apr 2021 12:40:59 +0000 Subject: [PATCH] Use external chain queue engine --- apps/cic-eth/cic_eth/admin/nonce.py | 24 +- apps/cic-eth/cic_eth/api/api_admin.py | 59 +- apps/cic-eth/cic_eth/api/api_task.py | 26 +- apps/cic-eth/cic_eth/db/__init__.py | 4 - .../versions/0ec0d6d1e785_add_chainqueue.py | 29 + .../default/versions/1f1b3b641d08_roles.py | 29 + .../2a07b543335e_add_new_syncer_table.py | 35 - .../3b693afd526a_nonce_reservation.py | 31 - .../versions/49b348246d70_add_nonce_index.py | 29 - .../52c7c59cd0b1_add_account_roles.py | 31 - ..._debug_output.py => 5ca4b77ce205_debug.py} | 14 +- .../6ac7a1dadc46_add_otx_state_log.py | 30 - ...bd_add_attempts_and_version_log_for_otx.py | 31 - .../versions/75d4767b3031_lock.py} | 15 +- .../7cb65b893934_add_blocknumber_pointer.py | 31 - .../versions/7e8d7626e38f_add_block_sync.py | 45 - .../8593fa1ca0f4_add_transaction_queue.py | 35 - .../versions/89e1e9baa53c_add_account_lock.py | 33 - .../versions/9c420530eeb2_nonce.py} | 22 +- .../9c4bd7491015_rename_block_sync_table.py | 26 - .../9daa16518a91_add_tx_sync_state.py | 30 - ...e2aab8f331_add_date_accessed_to_txcache.py | 34 - ...rt_tx_index.py => aee12aeb47ec_convert.py} | 14 +- ...er.py => b125cbf81e32_add_chain_syncer.py} | 13 +- .../versions/df19f4e69676_add_tx_track.py | 31 - .../default/versions/e3b5330ee71c_.py | 38 - .../db/migrations/postgresql/alembic.ini | 85 -- .../cic_eth/db/migrations/postgresql/env.py | 77 -- .../db/migrations/postgresql/script.py.mako | 24 - .../2a07b543335e_add_new_syncer_table.py | 35 - .../versions/49b348246d70_add_nonce_index.py | 29 - .../52c7c59cd0b1_add_account_roles.py | 31 - .../6ac7a1dadc46_add_otx_state_log.py | 30 - ...bd_add_attempts_and_version_log_for_otx.py | 31 - .../7cb65b893934_add_blocknumber_pointer.py | 31 - .../versions/7e8d7626e38f_add_block_sync.py | 42 - .../8593fa1ca0f4_add_transaction_queue.py | 35 - .../9c4bd7491015_rename_block_sync_table.py | 26 - .../9daa16518a91_add_tx_sync_state.py | 30 - ...e2aab8f331_add_date_accessed_to_txcache.py | 33 - .../versions/cd2052be6db2_convert_tx_index.py | 34 - .../versions/df19f4e69676_add_tx_track.py | 31 - .../postgresql/versions/e3b5330ee71c_.py | 37 - .../versions/ec40ac0974c1_add_chain_syncer.py | 28 - .../versions/f738d9962fdf_debug_output.py | 32 - apps/cic-eth/cic_eth/db/models/lock.py | 10 +- apps/cic-eth/cic_eth/db/models/otx.py | 680 ---------------- apps/cic-eth/cic_eth/db/models/tx.py | 151 ---- apps/cic-eth/cic_eth/error.py | 6 - apps/cic-eth/cic_eth/eth/account.py | 10 +- apps/cic-eth/cic_eth/eth/bancor.py.bak | 6 +- apps/cic-eth/cic_eth/eth/erc20.py | 19 +- apps/cic-eth/cic_eth/eth/gas.py | 443 ++++++++-- apps/cic-eth/cic_eth/eth/nonce.py | 47 ++ apps/cic-eth/cic_eth/eth/tx.py | 425 +--------- apps/cic-eth/cic_eth/ext/tx.py | 16 +- apps/cic-eth/cic_eth/queue/__init__.py | 2 - apps/cic-eth/cic_eth/queue/balance.py | 12 +- apps/cic-eth/cic_eth/queue/lock.py | 48 ++ apps/cic-eth/cic_eth/queue/query.py | 157 ++++ apps/cic-eth/cic_eth/queue/state.py | 100 +++ apps/cic-eth/cic_eth/queue/time.py | 13 +- apps/cic-eth/cic_eth/queue/tx.py | 766 +----------------- apps/cic-eth/cic_eth/registry.py | 4 +- .../cic_eth/runnable/daemons/dispatcher.py | 23 +- .../cic_eth/runnable/daemons/filters/gas.py | 20 +- .../runnable/daemons/filters/register.py | 3 +- .../runnable/daemons/filters/transferauth.py | 4 +- .../cic_eth/runnable/daemons/filters/tx.py | 11 +- .../cic-eth/cic_eth/runnable/daemons/retry.py | 12 +- .../cic_eth/runnable/daemons/tasker.py | 43 +- apps/cic-eth/cic_eth/version.py | 6 +- apps/cic-eth/docker/Dockerfile | 2 +- apps/cic-eth/requirements.txt | 19 +- apps/cic-eth/tests/fixtures_celery.py | 3 + apps/cic-eth/tests/task/api/test_admin.py | 46 +- apps/cic-eth/tests/task/api/test_list.py | 25 +- apps/cic-eth/tests/task/test_task_account.py | 13 +- apps/cic-eth/tests/task/test_task_erc20.py | 10 +- apps/cic-eth/tests/task/test_task_tx.py | 26 +- apps/cic-eth/tests/unit/db/test_block_sync.py | 29 - .../tests/unit/db/test_db_convert_transfer.py | 2 +- apps/cic-eth/tests/unit/db/test_otx.py | 107 --- apps/cic-eth/tests/unit/db/test_status.py | 97 --- apps/cic-eth/tests/unit/db/test_tx.py | 26 +- apps/cic-eth/tests/unit/eth/test_raw.py | 5 +- apps/cic-eth/tests/unit/ext/test_address.py | 2 +- .../cic-eth/tests/unit/queue/test_balances.py | 36 +- apps/cic-eth/tests/unit/queue/test_list_tx.py | 76 -- .../tests/unit/queue/test_otx_state_log.py | 22 - apps/cic-eth/tests/unit/queue/test_query.py | 58 ++ .../tests/unit/queue/test_queue_lock.py | 10 +- .../cic-eth/tests/unit/queue/test_tx_queue.py | 577 ------------- apps/contract-migration/docker/Dockerfile | 6 +- .../scripts/import_balance.py | 40 +- .../scripts/import_sovereign_users.py | 2 +- .../scripts/requirements.txt | 8 +- apps/contract-migration/scripts/verify.py | 8 +- apps/contract-migration/seed_cic_eth.sh | 2 + 99 files changed, 1255 insertions(+), 4479 deletions(-) create mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py create mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/1f1b3b641d08_roles.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/2a07b543335e_add_new_syncer_table.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/3b693afd526a_nonce_reservation.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/49b348246d70_add_nonce_index.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/52c7c59cd0b1_add_account_roles.py rename apps/cic-eth/cic_eth/db/migrations/default/versions/{f738d9962fdf_debug_output.py => 5ca4b77ce205_debug.py} (73%) delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/6ac7a1dadc46_add_otx_state_log.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py rename apps/cic-eth/cic_eth/db/migrations/{postgresql/versions/89e1e9baa53c_add_account_lock.py => default/versions/75d4767b3031_lock.py} (74%) delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/7cb65b893934_add_blocknumber_pointer.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/7e8d7626e38f_add_block_sync.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/8593fa1ca0f4_add_transaction_queue.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/89e1e9baa53c_add_account_lock.py rename apps/cic-eth/cic_eth/db/migrations/{postgresql/versions/3b693afd526a_nonce_reservation.py => default/versions/9c420530eeb2_nonce.py} (54%) delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/9c4bd7491015_rename_block_sync_table.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/9daa16518a91_add_tx_sync_state.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/a2e2aab8f331_add_date_accessed_to_txcache.py rename apps/cic-eth/cic_eth/db/migrations/default/versions/{cd2052be6db2_convert_tx_index.py => aee12aeb47ec_convert.py} (68%) rename apps/cic-eth/cic_eth/db/migrations/default/versions/{ec40ac0974c1_add_chain_syncer.py => b125cbf81e32_add_chain_syncer.py} (72%) delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/df19f4e69676_add_tx_track.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/e3b5330ee71c_.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/alembic.ini delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/env.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/script.py.mako delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/2a07b543335e_add_new_syncer_table.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/49b348246d70_add_nonce_index.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/52c7c59cd0b1_add_account_roles.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/6ac7a1dadc46_add_otx_state_log.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7cb65b893934_add_blocknumber_pointer.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7e8d7626e38f_add_block_sync.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/8593fa1ca0f4_add_transaction_queue.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9c4bd7491015_rename_block_sync_table.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9daa16518a91_add_tx_sync_state.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/a2e2aab8f331_add_date_accessed_to_txcache.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/cd2052be6db2_convert_tx_index.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/df19f4e69676_add_tx_track.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/e3b5330ee71c_.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/ec40ac0974c1_add_chain_syncer.py delete mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py delete mode 100644 apps/cic-eth/cic_eth/db/models/otx.py delete mode 100644 apps/cic-eth/cic_eth/db/models/tx.py create mode 100644 apps/cic-eth/cic_eth/queue/lock.py create mode 100644 apps/cic-eth/cic_eth/queue/query.py create mode 100644 apps/cic-eth/cic_eth/queue/state.py delete mode 100644 apps/cic-eth/tests/unit/db/test_block_sync.py delete mode 100644 apps/cic-eth/tests/unit/db/test_otx.py delete mode 100644 apps/cic-eth/tests/unit/db/test_status.py delete mode 100644 apps/cic-eth/tests/unit/queue/test_list_tx.py delete mode 100644 apps/cic-eth/tests/unit/queue/test_otx_state_log.py create mode 100644 apps/cic-eth/tests/unit/queue/test_query.py delete mode 100644 apps/cic-eth/tests/unit/queue/test_tx_queue.py diff --git a/apps/cic-eth/cic_eth/admin/nonce.py b/apps/cic-eth/cic_eth/admin/nonce.py index 5da7e877..b150cbed 100644 --- a/apps/cic-eth/cic_eth/admin/nonce.py +++ b/apps/cic-eth/cic_eth/admin/nonce.py @@ -5,11 +5,13 @@ import logging import celery from chainlib.chain import ChainSpec from chainlib.eth.tx import unpack +from chainqueue.query import get_tx +from chainqueue.state import set_cancel +from chainqueue.db.models.otx import Otx +from chainqueue.db.models.tx import TxCache # local imports from cic_eth.db.models.base import SessionBase -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.tx import TxCache from cic_eth.db.models.nonce import Nonce from cic_eth.admin.ctrl import ( lock_send, @@ -17,14 +19,8 @@ from cic_eth.admin.ctrl import ( lock_queue, unlock_queue, ) -from cic_eth.queue.tx import ( - get_tx, - set_cancel, - ) -from cic_eth.queue.tx import create as queue_create -from cic_eth.eth.gas import ( - create_check_gas_task, - ) +from cic_eth.queue.tx import queue_create +from cic_eth.eth.gas import create_check_gas_task celery_app = celery.current_app logg = logging.getLogger() @@ -50,8 +46,8 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1): chain_spec = ChainSpec.from_chain_str(chain_str) tx_brief = get_tx(tx_hash_orig_hex) - tx_raw = bytes.fromhex(tx_brief['signed_tx'][2:]) - tx = unpack(tx_raw, chain_spec.chain_id()) + tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:])) + tx = unpack(tx_raw, chain_spec) nonce = tx_brief['nonce'] address = tx['from'] @@ -71,8 +67,8 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1): tx_hashes = [] txs = [] for otx in otxs: - tx_raw = bytes.fromhex(otx.signed_tx[2:]) - tx_new = unpack(tx_raw, chain_spec.chain_id()) + tx_raw = bytes.fromhex(strip_0x(otx.signed_tx)) + tx_new = unpack(tx_raw, chain_spec) tx_previous_hash_hex = tx_new['hash'] tx_previous_nonce = tx_new['nonce'] diff --git a/apps/cic-eth/cic_eth/api/api_admin.py b/apps/cic-eth/cic_eth/api/api_admin.py index 458b03fb..fae6e0a9 100644 --- a/apps/cic-eth/cic_eth/api/api_admin.py +++ b/apps/cic-eth/cic_eth/api/api_admin.py @@ -22,23 +22,21 @@ from hexathon import ( add_0x, ) from chainlib.eth.gas import balance - -# local imports -from cic_eth.db.models.base import SessionBase -from cic_eth.db.models.role import AccountRole -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.nonce import Nonce -from cic_eth.db.enum import ( +from chainqueue.db.enum import ( StatusEnum, StatusBits, is_alive, is_error_status, status_str, ) +from chainqueue.error import TxStateChangeError + +# local imports +from cic_eth.db.models.base import SessionBase +from cic_eth.db.models.role import AccountRole +from cic_eth.db.models.nonce import Nonce from cic_eth.error import InitializationError -from cic_eth.db.error import TxStateChangeError -from cic_eth.queue.tx import get_tx +from cic_eth.queue.query import get_tx app = celery.current_app @@ -92,7 +90,7 @@ class AdminApi: def get_lock(self): s_lock = celery.signature( - 'cic_eth.queue.tx.get_lock', + 'cic_eth.queue.lock.get_lock', [], queue=self.queue, ) @@ -138,8 +136,9 @@ class AdminApi: logg.debug('resend {}'.format(tx_hash_hex)) s_get_tx_cache = celery.signature( - 'cic_eth.queue.tx.get_tx_cache', + 'cic_eth.queue.query.get_tx_cache', [ + chain_spec.asdict(), tx_hash_hex, ], queue=self.queue, @@ -155,7 +154,7 @@ class AdminApi: raise NotImplementedError('resend as new not yet implemented') s = celery.signature( - 'cic_eth.eth.tx.resend_with_higher_gas', + 'cic_eth.eth.gas.resend_with_higher_gas', [ chain_spec.asdict(), None, @@ -165,7 +164,7 @@ class AdminApi: ) s_manual = celery.signature( - 'cic_eth.queue.tx.set_manual', + 'cic_eth.queue.state.set_manual', [ tx_hash_hex, ], @@ -188,8 +187,9 @@ class AdminApi: def check_nonce(self, address): s = celery.signature( - 'cic_eth.queue.tx.get_account_tx', + 'cic_eth.queue.query.get_account_tx', [ + chain_spec.asdict(), address, True, False, @@ -204,8 +204,9 @@ class AdminApi: last_nonce = -1 for k in txs.keys(): s_get_tx = celery.signature( - 'cic_eth.queue.tx.get_tx', + 'cic_eth.queue.query.get_tx', [ + chain_spec.asdict(), k, ], queue=self.queue, @@ -243,8 +244,9 @@ class AdminApi: def fix_nonce(self, address, nonce, chain_spec): s = celery.signature( - 'cic_eth.queue.tx.get_account_tx', + 'cic_eth.queue.query.get_account_tx', [ + chain_spec.asdict(), address, True, False, @@ -294,8 +296,9 @@ class AdminApi: """ last_nonce = -1 s = celery.signature( - 'cic_eth.queue.tx.get_account_tx', + 'cic_eth.queue.query.get_account_tx', [ + chain_spec.asdict(), address, ], queue=self.queue, @@ -306,8 +309,11 @@ class AdminApi: for tx_hash in txs.keys(): errors = [] s = celery.signature( - 'cic_eth.queue.tx.get_tx_cache', - [tx_hash], + 'cic_eth.queue.query.get_tx_cache', + [ + chain_spec.asdict(), + tx_hash, + ], queue=self.queue, ) tx_dict = s.apply_async().get() @@ -367,12 +373,16 @@ class AdminApi: #tx_hash = self.w3.keccak(hexstr=tx_raw).hex() s = celery.signature( - 'cic_eth.queue.tx.get_tx_cache', - [tx_hash], + 'cic_eth.queue.query.get_tx_cache', + [ + chain_spec.asdict(), + tx_hash, + ], queue=self.queue, ) - tx = s.apply_async().get() + t = s.apply_async() + tx = t.get() source_token = None if tx['source_token'] != ZERO_ADDRESS: @@ -524,8 +534,9 @@ class AdminApi: tx['data'] = tx_unpacked['data'] s = celery.signature( - 'cic_eth.queue.tx.get_state_log', + 'cic_eth.queue.state.get_state_log', [ + chain_spec.asdict(), tx_hash, ], queue=self.queue, diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index cbe5c391..f9c49296 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -92,8 +92,10 @@ class Api: queue=self.queue, ) s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', - [], + 'cic_eth.eth.nonce.reserve_nonce', + [ + self.chain_spec.asdict(), + ], queue=self.queue, ) s_tokens = celery.signature( @@ -154,8 +156,10 @@ class Api: queue=self.queue, ) s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', - [], + 'cic_eth.eth.nonce.reserve_nonce', + [ + self.chain_spec.asdict(), + ], queue=self.queue, ) s_tokens = celery.signature( @@ -213,8 +217,9 @@ class Api: queue=self.queue, ) s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ + self.chain_spec.asdict(), from_address, ], queue=self.queue, @@ -359,8 +364,9 @@ class Api: if register: s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ + self.chain_spec.asdict(), 'ACCOUNT_REGISTRY_WRITER', ], queue=self.queue, @@ -397,14 +403,15 @@ class Api: queue=self.queue, ) s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ + self.chain_spec.asdict(), 'GAS_GIFTER', ], queue=self.queue, ) s_refill = celery.signature( - 'cic_eth.eth.tx.refill_gas', + 'cic_eth.eth.gas.refill_gas', [ self.chain_spec.asdict(), ], @@ -439,8 +446,9 @@ class Api: """ offset = 0 s_local = celery.signature( - 'cic_eth.queue.tx.get_account_tx', + 'cic_eth.queue.query.get_account_tx', [ + self.chain_spec.asdict(), address, ], queue=self.queue, diff --git a/apps/cic-eth/cic_eth/db/__init__.py b/apps/cic-eth/cic_eth/db/__init__.py index 1a549508..62731220 100644 --- a/apps/cic-eth/cic_eth/db/__init__.py +++ b/apps/cic-eth/cic_eth/db/__init__.py @@ -11,10 +11,6 @@ logg = logging.getLogger() # an Engine, which the Session will use for connection # resources -# TODO: Remove the package exports, all models should be imported using full path -from .models.otx import Otx -from .models.convert import TxConvertTransfer - def dsn_from_config(config): """Generate a dsn string from the provided config dict. diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py new file mode 100644 index 00000000..09217067 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py @@ -0,0 +1,29 @@ +"""Add chainqueue + +Revision ID: 0ec0d6d1e785 +Revises: +Create Date: 2021-04-02 18:30:55.398388 + +""" +from alembic import op +import sqlalchemy as sa + +from chainqueue.db.migrations.sqlalchemy import ( + chainqueue_upgrade, + chainqueue_downgrade, + ) + +# revision identifiers, used by Alembic. +revision = '0ec0d6d1e785' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + chainqueue_upgrade(0, 0, 1) + + +def downgrade(): + chainqueue_downgrade(0, 0, 1) + diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/1f1b3b641d08_roles.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/1f1b3b641d08_roles.py new file mode 100644 index 00000000..35fed2a2 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/1f1b3b641d08_roles.py @@ -0,0 +1,29 @@ +"""Roles + +Revision ID: 1f1b3b641d08 +Revises: 9c420530eeb2 +Create Date: 2021-04-02 18:40:27.787631 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '1f1b3b641d08' +down_revision = '9c420530eeb2' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'account_role', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('tag', sa.Text, nullable=False, unique=True), + sa.Column('address_hex', sa.String(42), nullable=False), + ) + + +def downgrade(): + op.drop_table('account_role') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/2a07b543335e_add_new_syncer_table.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/2a07b543335e_add_new_syncer_table.py deleted file mode 100644 index 30ddb96f..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/2a07b543335e_add_new_syncer_table.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Add new syncer table - -Revision ID: 2a07b543335e -Revises: a2e2aab8f331 -Create Date: 2020-12-27 09:35:44.017981 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '2a07b543335e' -down_revision = 'a2e2aab8f331' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'blockchain_sync', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('blockchain', sa.String, nullable=False), - sa.Column('block_start', sa.Integer, nullable=False, default=0), - sa.Column('tx_start', sa.Integer, nullable=False, default=0), - sa.Column('block_cursor', sa.Integer, nullable=False, default=0), - sa.Column('tx_cursor', sa.Integer, nullable=False, default=0), - sa.Column('block_target', sa.Integer, nullable=True), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('date_updated', sa.DateTime), - ) - - -def downgrade(): - op.drop_table('blockchain_sync') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/3b693afd526a_nonce_reservation.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/3b693afd526a_nonce_reservation.py deleted file mode 100644 index 580d0345..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/3b693afd526a_nonce_reservation.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Nonce reservation - -Revision ID: 3b693afd526a -Revises: f738d9962fdf -Create Date: 2021-03-05 07:09:50.898728 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '3b693afd526a' -down_revision = 'f738d9962fdf' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'nonce_task_reservation', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('address_hex', sa.String(42), nullable=False), - sa.Column('nonce', sa.Integer, nullable=False), - sa.Column('key', sa.String, nullable=False), - sa.Column('date_created', sa.DateTime, nullable=False), - ) - - -def downgrade(): - op.drop_table('nonce_task_reservation') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/49b348246d70_add_nonce_index.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/49b348246d70_add_nonce_index.py deleted file mode 100644 index 3abee1b6..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/49b348246d70_add_nonce_index.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Add nonce index - -Revision ID: 49b348246d70 -Revises: 52c7c59cd0b1 -Create Date: 2020-12-19 09:45:36.186446 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '49b348246d70' -down_revision = '52c7c59cd0b1' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'nonce', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('address_hex', sa.String(42), nullable=False, unique=True), - sa.Column('nonce', sa.Integer, nullable=False), - ) - - -def downgrade(): - op.drop_table('nonce') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/52c7c59cd0b1_add_account_roles.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/52c7c59cd0b1_add_account_roles.py deleted file mode 100644 index 7aa74059..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/52c7c59cd0b1_add_account_roles.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add account roles - -Revision ID: 52c7c59cd0b1 -Revises: 9c4bd7491015 -Create Date: 2020-12-19 07:21:38.249237 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '52c7c59cd0b1' -down_revision = '9c4bd7491015' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'account_role', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('tag', sa.Text, nullable=False, unique=True), - sa.Column('address_hex', sa.String(42), nullable=False), - ) - pass - - -def downgrade(): - op.drop_table('account_role') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/5ca4b77ce205_debug.py similarity index 73% rename from apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py rename to apps/cic-eth/cic_eth/db/migrations/default/versions/5ca4b77ce205_debug.py index 378f2965..5014fb2d 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/5ca4b77ce205_debug.py @@ -1,8 +1,8 @@ -"""debug output +"""DEbug -Revision ID: f738d9962fdf -Revises: ec40ac0974c1 -Create Date: 2021-03-04 08:32:43.281214 +Revision ID: 5ca4b77ce205 +Revises: 75d4767b3031 +Create Date: 2021-04-02 18:42:12.257244 """ from alembic import op @@ -10,8 +10,8 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'f738d9962fdf' -down_revision = 'ec40ac0974c1' +revision = '5ca4b77ce205' +down_revision = '75d4767b3031' branch_labels = None depends_on = None @@ -24,9 +24,7 @@ def upgrade(): sa.Column('description', sa.String, nullable=False), sa.Column('date_created', sa.DateTime, nullable=False), ) - pass def downgrade(): op.drop_table('debug') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/6ac7a1dadc46_add_otx_state_log.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/6ac7a1dadc46_add_otx_state_log.py deleted file mode 100644 index f15834d9..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/6ac7a1dadc46_add_otx_state_log.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Add otx state log - -Revision ID: 6ac7a1dadc46 -Revises: 89e1e9baa53c -Create Date: 2021-01-30 13:59:49.022373 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '6ac7a1dadc46' -down_revision = '89e1e9baa53c' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'otx_state_log', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False), - sa.Column('date', sa.DateTime, nullable=False), - sa.Column('status', sa.Integer, nullable=False), - ) - - -def downgrade(): - op.drop_table('otx_state_log') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py deleted file mode 100644 index 3be687eb..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add attempts and version log for otx - -Revision ID: 71708e943dbd -Revises: 7e8d7626e38f -Create Date: 2020-09-26 14:41:19.298651 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '71708e943dbd' -down_revision = '7e8d7626e38f' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'otx_attempts', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False), - sa.Column('date', sa.DateTime, nullable=False), - ) - pass - - -def downgrade(): - op.drop_table('otx_attempts') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/89e1e9baa53c_add_account_lock.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py similarity index 74% rename from apps/cic-eth/cic_eth/db/migrations/postgresql/versions/89e1e9baa53c_add_account_lock.py rename to apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py index 4b1c4401..656fdd06 100644 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/89e1e9baa53c_add_account_lock.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py @@ -1,8 +1,8 @@ -"""Add account lock +"""Lock -Revision ID: 89e1e9baa53c -Revises: 2a07b543335e -Create Date: 2021-01-27 19:57:36.793882 +Revision ID: 75d4767b3031 +Revises: 1f1b3b641d08 +Create Date: 2021-04-02 18:41:20.864265 """ from alembic import op @@ -10,8 +10,8 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = '89e1e9baa53c' -down_revision = '2a07b543335e' +revision = '75d4767b3031' +down_revision = '1f1b3b641d08' branch_labels = None depends_on = None @@ -24,10 +24,11 @@ def upgrade(): sa.Column('blockchain', sa.String), sa.Column("flags", sa.BIGINT(), nullable=False, default=0), sa.Column("date_created", sa.DateTime, nullable=False), - sa.Column("otx_id", sa.Integer, nullable=True), + sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True), ) op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True) + def downgrade(): op.drop_index('idx_chain_address') op.drop_table('lock') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/7cb65b893934_add_blocknumber_pointer.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/7cb65b893934_add_blocknumber_pointer.py deleted file mode 100644 index 74a97f83..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/7cb65b893934_add_blocknumber_pointer.py +++ /dev/null @@ -1,31 +0,0 @@ -"""add blocknumber pointer - -Revision ID: 7cb65b893934 -Revises: 8593fa1ca0f4 -Create Date: 2020-09-24 19:29:13.543648 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '7cb65b893934' -down_revision = '8593fa1ca0f4' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'watcher_state', - sa.Column('block_number', sa.Integer) - ) - conn = op.get_bind() - conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);') - pass - - -def downgrade(): - op.drop_table('watcher_state') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/7e8d7626e38f_add_block_sync.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/7e8d7626e38f_add_block_sync.py deleted file mode 100644 index 52f29a7b..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/7e8d7626e38f_add_block_sync.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Add block sync - -Revision ID: 7e8d7626e38f -Revises: cd2052be6db2 -Create Date: 2020-09-26 11:12:27.818524 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '7e8d7626e38f' -down_revision = 'cd2052be6db2' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'block_sync', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('blockchain', sa.String, nullable=False, unique=True), - sa.Column('block_height_backlog', sa.Integer, nullable=False, default=0), - sa.Column('tx_height_backlog', sa.Integer, nullable=False, default=0), - sa.Column('block_height_session', sa.Integer, nullable=False, default=0), - sa.Column('tx_height_session', sa.Integer, nullable=False, default=0), - sa.Column('block_height_head', sa.Integer, nullable=False, default=0), - sa.Column('tx_height_head', sa.Integer, nullable=False, default=0), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('date_updated', sa.DateTime), - ) - op.drop_table('watcher_state') - pass - - -def downgrade(): - op.drop_table('block_sync') - op.create_table( - 'watcher_state', - sa.Column('block_number', sa.Integer) - ) - conn = op.get_bind() - conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/8593fa1ca0f4_add_transaction_queue.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/8593fa1ca0f4_add_transaction_queue.py deleted file mode 100644 index e9f9cf6a..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/8593fa1ca0f4_add_transaction_queue.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Add transaction queue - -Revision ID: 8593fa1ca0f4 -Revises: -Create Date: 2020-09-22 21:56:42.117047 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '8593fa1ca0f4' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'otx', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('nonce', sa.Integer, nullable=False), - sa.Column('tx_hash', sa.String(66), nullable=False), - sa.Column('signed_tx', sa.Text, nullable=False), - sa.Column('status', sa.Integer, nullable=False, default=-9), - sa.Column('block', sa.Integer), - ) - op.create_index('idx_otx_tx', 'otx', ['tx_hash'], unique=True) - - -def downgrade(): - op.drop_index('idx_otx_tx') - op.drop_table('otx') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/89e1e9baa53c_add_account_lock.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/89e1e9baa53c_add_account_lock.py deleted file mode 100644 index 4b1c4401..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/89e1e9baa53c_add_account_lock.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Add account lock - -Revision ID: 89e1e9baa53c -Revises: 2a07b543335e -Create Date: 2021-01-27 19:57:36.793882 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '89e1e9baa53c' -down_revision = '2a07b543335e' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'lock', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column("address", sa.String(42), nullable=True), - sa.Column('blockchain', sa.String), - sa.Column("flags", sa.BIGINT(), nullable=False, default=0), - sa.Column("date_created", sa.DateTime, nullable=False), - sa.Column("otx_id", sa.Integer, nullable=True), - ) - op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True) - -def downgrade(): - op.drop_index('idx_chain_address') - op.drop_table('lock') diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/3b693afd526a_nonce_reservation.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/9c420530eeb2_nonce.py similarity index 54% rename from apps/cic-eth/cic_eth/db/migrations/postgresql/versions/3b693afd526a_nonce_reservation.py rename to apps/cic-eth/cic_eth/db/migrations/default/versions/9c420530eeb2_nonce.py index 580d0345..3dffac98 100644 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/3b693afd526a_nonce_reservation.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/9c420530eeb2_nonce.py @@ -1,8 +1,8 @@ -"""Nonce reservation +"""Nonce -Revision ID: 3b693afd526a -Revises: f738d9962fdf -Create Date: 2021-03-05 07:09:50.898728 +Revision ID: 9c420530eeb2 +Revises: b125cbf81e32 +Create Date: 2021-04-02 18:38:56.459334 """ from alembic import op @@ -10,15 +10,22 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = '3b693afd526a' -down_revision = 'f738d9962fdf' +revision = '9c420530eeb2' +down_revision = 'b125cbf81e32' branch_labels = None depends_on = None def upgrade(): op.create_table( - 'nonce_task_reservation', + 'nonce', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('address_hex', sa.String(42), nullable=False, unique=True), + sa.Column('nonce', sa.Integer, nullable=False), + ) + + op.create_table( + 'nonce_task_reservation', sa.Column('id', sa.Integer, primary_key=True), sa.Column('address_hex', sa.String(42), nullable=False), sa.Column('nonce', sa.Integer, nullable=False), @@ -29,3 +36,4 @@ def upgrade(): def downgrade(): op.drop_table('nonce_task_reservation') + op.drop_table('nonce') diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/9c4bd7491015_rename_block_sync_table.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/9c4bd7491015_rename_block_sync_table.py deleted file mode 100644 index f58721b2..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/9c4bd7491015_rename_block_sync_table.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Rename block sync table - -Revision ID: 9c4bd7491015 -Revises: 9daa16518a91 -Create Date: 2020-10-15 23:45:56.306898 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '9c4bd7491015' -down_revision = '9daa16518a91' -branch_labels = None -depends_on = None - - -def upgrade(): - op.rename_table('block_sync', 'otx_sync') - pass - - -def downgrade(): - op.rename_table('otx_sync', 'block_sync') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/9daa16518a91_add_tx_sync_state.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/9daa16518a91_add_tx_sync_state.py deleted file mode 100644 index 2c6cc0fe..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/9daa16518a91_add_tx_sync_state.py +++ /dev/null @@ -1,30 +0,0 @@ -"""add tx sync state - -Revision ID: 9daa16518a91 -Revises: e3b5330ee71c -Create Date: 2020-10-10 14:43:18.699276 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '9daa16518a91' -down_revision = 'e3b5330ee71c' -branch_labels = None -depends_on = None - - -def upgrade(): -# op.create_table( -# 'tx_sync', -# sa.Column('tx', sa.String(66), nullable=False), -# ) -# op.execute("INSERT INTO tx_sync VALUES('0x0000000000000000000000000000000000000000000000000000000000000000')") - pass - - -def downgrade(): -# op.drop_table('tx_sync') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/a2e2aab8f331_add_date_accessed_to_txcache.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/a2e2aab8f331_add_date_accessed_to_txcache.py deleted file mode 100644 index f82f2615..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/a2e2aab8f331_add_date_accessed_to_txcache.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Add date accessed to txcache - -Revision ID: a2e2aab8f331 -Revises: 49b348246d70 -Create Date: 2020-12-24 18:58:06.137812 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'a2e2aab8f331' -down_revision = '49b348246d70' -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column( - 'tx_cache', - sa.Column( - 'date_checked', - sa.DateTime, - nullable=False - ) - ) - pass - - -def downgrade(): - # drop does not work withs qlite - #op.drop_column('tx_cache', 'date_checked') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/cd2052be6db2_convert_tx_index.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/aee12aeb47ec_convert.py similarity index 68% rename from apps/cic-eth/cic_eth/db/migrations/default/versions/cd2052be6db2_convert_tx_index.py rename to apps/cic-eth/cic_eth/db/migrations/default/versions/aee12aeb47ec_convert.py index 5b59d734..21aced4e 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/cd2052be6db2_convert_tx_index.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/aee12aeb47ec_convert.py @@ -1,8 +1,8 @@ -"""convert tx index +"""Convert -Revision ID: cd2052be6db2 -Revises: 7cb65b893934 -Create Date: 2020-09-24 21:20:51.580500 +Revision ID: aee12aeb47ec +Revises: 5ca4b77ce205 +Create Date: 2021-04-02 18:42:45.233356 """ from alembic import op @@ -10,8 +10,8 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'cd2052be6db2' -down_revision = '7cb65b893934' +revision = 'aee12aeb47ec' +down_revision = '5ca4b77ce205' branch_labels = None depends_on = None @@ -20,10 +20,8 @@ def upgrade(): op.create_table( 'tx_convert_transfer', sa.Column('id', sa.Integer, primary_key=True), - #sa.Column('approve_tx_hash', sa.String(66), nullable=False, unique=True), sa.Column('convert_tx_hash', sa.String(66), nullable=False, unique=True), sa.Column('transfer_tx_hash', sa.String(66), unique=True), -# sa.Column('holder_address', sa.String(42), nullable=False), sa.Column('recipient_address', sa.String(42), nullable=False), ) op.create_index('idx_tx_convert_address', 'tx_convert_transfer', ['recipient_address']) diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/ec40ac0974c1_add_chain_syncer.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py similarity index 72% rename from apps/cic-eth/cic_eth/db/migrations/default/versions/ec40ac0974c1_add_chain_syncer.py rename to apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py index a278997f..8f3be98f 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/ec40ac0974c1_add_chain_syncer.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py @@ -1,12 +1,13 @@ """Add chain syncer -Revision ID: ec40ac0974c1 -Revises: 6ac7a1dadc46 -Create Date: 2021-02-23 06:10:19.246304 +Revision ID: b125cbf81e32 +Revises: 0ec0d6d1e785 +Create Date: 2021-04-02 18:36:44.459603 """ from alembic import op import sqlalchemy as sa + from chainsyncer.db.migrations.sqlalchemy import ( chainsyncer_upgrade, chainsyncer_downgrade, @@ -14,15 +15,15 @@ from chainsyncer.db.migrations.sqlalchemy import ( # revision identifiers, used by Alembic. -revision = 'ec40ac0974c1' -down_revision = '6ac7a1dadc46' +revision = 'b125cbf81e32' +down_revision = '0ec0d6d1e785' branch_labels = None depends_on = None - def upgrade(): chainsyncer_upgrade(0, 0, 1) def downgrade(): chainsyncer_downgrade(0, 0, 1) + diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/df19f4e69676_add_tx_track.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/df19f4e69676_add_tx_track.py deleted file mode 100644 index 2c1ea138..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/df19f4e69676_add_tx_track.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add tx tracker record - -Revision ID: df19f4e69676 -Revises: 71708e943dbd -Create Date: 2020-10-09 23:31:44.563498 - -""" -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = 'df19f4e69676' -down_revision = '71708e943dbd' -branch_labels = None -depends_on = None - - -def upgrade(): -# op.create_table( -# 'tx', -# sa.Column('id', sa.Integer, primary_key=True), -# sa.Column('date_added', sa.DateTime, nullable=False), -# sa.Column('tx_hash', sa.String(66), nullable=False, unique=True), -# sa.Column('success', sa.Boolean(), nullable=False), -# ) - pass - - -def downgrade(): -# op.drop_table('tx') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/e3b5330ee71c_.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/e3b5330ee71c_.py deleted file mode 100644 index 6c9115b7..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/e3b5330ee71c_.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Add cached values for tx - -Revision ID: e3b5330ee71c -Revises: df19f4e69676 -Create Date: 2020-10-10 00:17:07.094893 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'e3b5330ee71c' -down_revision = 'df19f4e69676' -branch_labels = None -depends_on = None - -def upgrade(): - op.create_table( - 'tx_cache', - sa.Column('id', sa.Integer, primary_key=True), -# sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=True), - sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=True), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('date_updated', sa.DateTime, nullable=False), - sa.Column('source_token_address', sa.String(42), nullable=False), - sa.Column('destination_token_address', sa.String(42), nullable=False), - sa.Column('sender', sa.String(42), nullable=False), - sa.Column('recipient', sa.String(42), nullable=False), - sa.Column('from_value', sa.NUMERIC(), nullable=False), - sa.Column('to_value', sa.NUMERIC(), nullable=True), - sa.Column('block_number', sa.BIGINT(), nullable=True), - sa.Column('tx_index', sa.Integer, nullable=True), - ) - -def downgrade(): - op.drop_table('tx_cache') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/alembic.ini b/apps/cic-eth/cic_eth/db/migrations/postgresql/alembic.ini deleted file mode 100644 index 1555f68c..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/alembic.ini +++ /dev/null @@ -1,85 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = . - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# timezone to use when rendering the date -# within the migration file as well as the filename. -# string value is passed to dateutil.tz.gettz() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the -# "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; this defaults -# to migrations/versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat migrations/versions - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -#sqlalchemy.url = driver://user:pass@localhost/dbname -sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic-eth - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks=black -# black.type=console_scripts -# black.entrypoint=black -# black.options=-l 79 - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/env.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/env.py deleted file mode 100644 index 70518a2e..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/env.py +++ /dev/null @@ -1,77 +0,0 @@ -from logging.config import fileConfig - -from sqlalchemy import engine_from_config -from sqlalchemy import pool - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = None - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/script.py.mako b/apps/cic-eth/cic_eth/db/migrations/postgresql/script.py.mako deleted file mode 100644 index 2c015630..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/2a07b543335e_add_new_syncer_table.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/2a07b543335e_add_new_syncer_table.py deleted file mode 100644 index 30ddb96f..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/2a07b543335e_add_new_syncer_table.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Add new syncer table - -Revision ID: 2a07b543335e -Revises: a2e2aab8f331 -Create Date: 2020-12-27 09:35:44.017981 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '2a07b543335e' -down_revision = 'a2e2aab8f331' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'blockchain_sync', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('blockchain', sa.String, nullable=False), - sa.Column('block_start', sa.Integer, nullable=False, default=0), - sa.Column('tx_start', sa.Integer, nullable=False, default=0), - sa.Column('block_cursor', sa.Integer, nullable=False, default=0), - sa.Column('tx_cursor', sa.Integer, nullable=False, default=0), - sa.Column('block_target', sa.Integer, nullable=True), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('date_updated', sa.DateTime), - ) - - -def downgrade(): - op.drop_table('blockchain_sync') diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/49b348246d70_add_nonce_index.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/49b348246d70_add_nonce_index.py deleted file mode 100644 index 3abee1b6..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/49b348246d70_add_nonce_index.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Add nonce index - -Revision ID: 49b348246d70 -Revises: 52c7c59cd0b1 -Create Date: 2020-12-19 09:45:36.186446 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '49b348246d70' -down_revision = '52c7c59cd0b1' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'nonce', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('address_hex', sa.String(42), nullable=False, unique=True), - sa.Column('nonce', sa.Integer, nullable=False), - ) - - -def downgrade(): - op.drop_table('nonce') diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/52c7c59cd0b1_add_account_roles.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/52c7c59cd0b1_add_account_roles.py deleted file mode 100644 index 7aa74059..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/52c7c59cd0b1_add_account_roles.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add account roles - -Revision ID: 52c7c59cd0b1 -Revises: 9c4bd7491015 -Create Date: 2020-12-19 07:21:38.249237 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '52c7c59cd0b1' -down_revision = '9c4bd7491015' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'account_role', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('tag', sa.Text, nullable=False, unique=True), - sa.Column('address_hex', sa.String(42), nullable=False), - ) - pass - - -def downgrade(): - op.drop_table('account_role') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/6ac7a1dadc46_add_otx_state_log.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/6ac7a1dadc46_add_otx_state_log.py deleted file mode 100644 index f15834d9..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/6ac7a1dadc46_add_otx_state_log.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Add otx state log - -Revision ID: 6ac7a1dadc46 -Revises: 89e1e9baa53c -Create Date: 2021-01-30 13:59:49.022373 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '6ac7a1dadc46' -down_revision = '89e1e9baa53c' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'otx_state_log', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False), - sa.Column('date', sa.DateTime, nullable=False), - sa.Column('status', sa.Integer, nullable=False), - ) - - -def downgrade(): - op.drop_table('otx_state_log') diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py deleted file mode 100644 index 3be687eb..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add attempts and version log for otx - -Revision ID: 71708e943dbd -Revises: 7e8d7626e38f -Create Date: 2020-09-26 14:41:19.298651 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '71708e943dbd' -down_revision = '7e8d7626e38f' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'otx_attempts', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False), - sa.Column('date', sa.DateTime, nullable=False), - ) - pass - - -def downgrade(): - op.drop_table('otx_attempts') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7cb65b893934_add_blocknumber_pointer.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7cb65b893934_add_blocknumber_pointer.py deleted file mode 100644 index 74a97f83..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7cb65b893934_add_blocknumber_pointer.py +++ /dev/null @@ -1,31 +0,0 @@ -"""add blocknumber pointer - -Revision ID: 7cb65b893934 -Revises: 8593fa1ca0f4 -Create Date: 2020-09-24 19:29:13.543648 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '7cb65b893934' -down_revision = '8593fa1ca0f4' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'watcher_state', - sa.Column('block_number', sa.Integer) - ) - conn = op.get_bind() - conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);') - pass - - -def downgrade(): - op.drop_table('watcher_state') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7e8d7626e38f_add_block_sync.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7e8d7626e38f_add_block_sync.py deleted file mode 100644 index 6924fa26..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7e8d7626e38f_add_block_sync.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Add block sync - -Revision ID: 7e8d7626e38f -Revises: cd2052be6db2 -Create Date: 2020-09-26 11:12:27.818524 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '7e8d7626e38f' -down_revision = 'cd2052be6db2' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'block_sync', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('blockchain', sa.String, nullable=False, unique=True), - sa.Column('height_backlog', sa.Integer, nullable=False, default=0), - sa.Column('height_session', sa.Integer, nullable=False, default=0), - sa.Column('height_head', sa.Integer, nullable=False, default=0), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('date_updated', sa.DateTime), - ) - op.drop_table('watcher_state') - pass - - -def downgrade(): - op.drop_table('block_sync') - op.create_table( - 'watcher_state', - sa.Column('block_number', sa.Integer) - ) - conn = op.get_bind() - conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/8593fa1ca0f4_add_transaction_queue.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/8593fa1ca0f4_add_transaction_queue.py deleted file mode 100644 index e9f9cf6a..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/8593fa1ca0f4_add_transaction_queue.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Add transaction queue - -Revision ID: 8593fa1ca0f4 -Revises: -Create Date: 2020-09-22 21:56:42.117047 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '8593fa1ca0f4' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'otx', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('nonce', sa.Integer, nullable=False), - sa.Column('tx_hash', sa.String(66), nullable=False), - sa.Column('signed_tx', sa.Text, nullable=False), - sa.Column('status', sa.Integer, nullable=False, default=-9), - sa.Column('block', sa.Integer), - ) - op.create_index('idx_otx_tx', 'otx', ['tx_hash'], unique=True) - - -def downgrade(): - op.drop_index('idx_otx_tx') - op.drop_table('otx') diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9c4bd7491015_rename_block_sync_table.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9c4bd7491015_rename_block_sync_table.py deleted file mode 100644 index f58721b2..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9c4bd7491015_rename_block_sync_table.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Rename block sync table - -Revision ID: 9c4bd7491015 -Revises: 9daa16518a91 -Create Date: 2020-10-15 23:45:56.306898 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '9c4bd7491015' -down_revision = '9daa16518a91' -branch_labels = None -depends_on = None - - -def upgrade(): - op.rename_table('block_sync', 'otx_sync') - pass - - -def downgrade(): - op.rename_table('otx_sync', 'block_sync') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9daa16518a91_add_tx_sync_state.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9daa16518a91_add_tx_sync_state.py deleted file mode 100644 index 2c6cc0fe..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9daa16518a91_add_tx_sync_state.py +++ /dev/null @@ -1,30 +0,0 @@ -"""add tx sync state - -Revision ID: 9daa16518a91 -Revises: e3b5330ee71c -Create Date: 2020-10-10 14:43:18.699276 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '9daa16518a91' -down_revision = 'e3b5330ee71c' -branch_labels = None -depends_on = None - - -def upgrade(): -# op.create_table( -# 'tx_sync', -# sa.Column('tx', sa.String(66), nullable=False), -# ) -# op.execute("INSERT INTO tx_sync VALUES('0x0000000000000000000000000000000000000000000000000000000000000000')") - pass - - -def downgrade(): -# op.drop_table('tx_sync') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/a2e2aab8f331_add_date_accessed_to_txcache.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/a2e2aab8f331_add_date_accessed_to_txcache.py deleted file mode 100644 index 808a503c..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/a2e2aab8f331_add_date_accessed_to_txcache.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Add date accessed to txcache - -Revision ID: a2e2aab8f331 -Revises: 49b348246d70 -Create Date: 2020-12-24 18:58:06.137812 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'a2e2aab8f331' -down_revision = '49b348246d70' -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column( - 'tx_cache', - sa.Column( - 'date_checked', - sa.DateTime, - nullable=False - ) - ) - pass - - -def downgrade(): - op.drop_column('tx_cache', 'date_checked') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/cd2052be6db2_convert_tx_index.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/cd2052be6db2_convert_tx_index.py deleted file mode 100644 index 5b59d734..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/cd2052be6db2_convert_tx_index.py +++ /dev/null @@ -1,34 +0,0 @@ -"""convert tx index - -Revision ID: cd2052be6db2 -Revises: 7cb65b893934 -Create Date: 2020-09-24 21:20:51.580500 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'cd2052be6db2' -down_revision = '7cb65b893934' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'tx_convert_transfer', - sa.Column('id', sa.Integer, primary_key=True), - #sa.Column('approve_tx_hash', sa.String(66), nullable=False, unique=True), - sa.Column('convert_tx_hash', sa.String(66), nullable=False, unique=True), - sa.Column('transfer_tx_hash', sa.String(66), unique=True), -# sa.Column('holder_address', sa.String(42), nullable=False), - sa.Column('recipient_address', sa.String(42), nullable=False), - ) - op.create_index('idx_tx_convert_address', 'tx_convert_transfer', ['recipient_address']) - - -def downgrade(): - op.drop_index('idx_tx_convert_address') - op.drop_table('tx_convert_transfer') diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/df19f4e69676_add_tx_track.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/df19f4e69676_add_tx_track.py deleted file mode 100644 index 2c1ea138..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/df19f4e69676_add_tx_track.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add tx tracker record - -Revision ID: df19f4e69676 -Revises: 71708e943dbd -Create Date: 2020-10-09 23:31:44.563498 - -""" -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = 'df19f4e69676' -down_revision = '71708e943dbd' -branch_labels = None -depends_on = None - - -def upgrade(): -# op.create_table( -# 'tx', -# sa.Column('id', sa.Integer, primary_key=True), -# sa.Column('date_added', sa.DateTime, nullable=False), -# sa.Column('tx_hash', sa.String(66), nullable=False, unique=True), -# sa.Column('success', sa.Boolean(), nullable=False), -# ) - pass - - -def downgrade(): -# op.drop_table('tx') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/e3b5330ee71c_.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/e3b5330ee71c_.py deleted file mode 100644 index 3abafb73..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/e3b5330ee71c_.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Add cached values for tx - -Revision ID: e3b5330ee71c -Revises: df19f4e69676 -Create Date: 2020-10-10 00:17:07.094893 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'e3b5330ee71c' -down_revision = 'df19f4e69676' -branch_labels = None -depends_on = None - -def upgrade(): - op.create_table( - 'tx_cache', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=True), - sa.Column('date_created', sa.DateTime, nullable=False), - sa.Column('date_updated', sa.DateTime, nullable=False), - sa.Column('source_token_address', sa.String(42), nullable=False), - sa.Column('destination_token_address', sa.String(42), nullable=False), - sa.Column('sender', sa.String(42), nullable=False), - sa.Column('recipient', sa.String(42), nullable=False), - sa.Column('from_value', sa.NUMERIC(), nullable=False), - sa.Column('to_value', sa.NUMERIC(), nullable=True), - sa.Column('block_number', sa.BIGINT(), nullable=True), - sa.Column('tx_index', sa.Integer, nullable=True), - ) - -def downgrade(): - op.drop_table('tx_cache') - pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/ec40ac0974c1_add_chain_syncer.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/ec40ac0974c1_add_chain_syncer.py deleted file mode 100644 index a278997f..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/ec40ac0974c1_add_chain_syncer.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Add chain syncer - -Revision ID: ec40ac0974c1 -Revises: 6ac7a1dadc46 -Create Date: 2021-02-23 06:10:19.246304 - -""" -from alembic import op -import sqlalchemy as sa -from chainsyncer.db.migrations.sqlalchemy import ( - chainsyncer_upgrade, - chainsyncer_downgrade, - ) - - -# revision identifiers, used by Alembic. -revision = 'ec40ac0974c1' -down_revision = '6ac7a1dadc46' -branch_labels = None -depends_on = None - - -def upgrade(): - chainsyncer_upgrade(0, 0, 1) - - -def downgrade(): - chainsyncer_downgrade(0, 0, 1) diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py deleted file mode 100644 index 378f2965..00000000 --- a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py +++ /dev/null @@ -1,32 +0,0 @@ -"""debug output - -Revision ID: f738d9962fdf -Revises: ec40ac0974c1 -Create Date: 2021-03-04 08:32:43.281214 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'f738d9962fdf' -down_revision = 'ec40ac0974c1' -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'debug', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('tag', sa.String, nullable=False), - sa.Column('description', sa.String, nullable=False), - sa.Column('date_created', sa.DateTime, nullable=False), - ) - pass - - -def downgrade(): - op.drop_table('debug') - pass diff --git a/apps/cic-eth/cic_eth/db/models/lock.py b/apps/cic-eth/cic_eth/db/models/lock.py index 3737b893..d8dcb426 100644 --- a/apps/cic-eth/cic_eth/db/models/lock.py +++ b/apps/cic-eth/cic_eth/db/models/lock.py @@ -5,11 +5,11 @@ import logging # third-party imports from sqlalchemy import Column, String, Integer, DateTime, ForeignKey from chainlib.eth.constant import ZERO_ADDRESS +from chainqueue.db.models.tx import TxCache +from chainqueue.db.models.otx import Otx # local imports from cic_eth.db.models.base import SessionBase -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.otx import Otx logg = logging.getLogger() @@ -22,10 +22,12 @@ class Lock(SessionBase): __tablename__ = "lock" blockchain = Column(String) - address = Column(String, ForeignKey('tx_cache.sender')) + #address = Column(String, ForeignKey('tx_cache.sender')) + address = Column(String, ForeignKey(TxCache.sender)) flags = Column(Integer) date_created = Column(DateTime, default=datetime.datetime.utcnow) - otx_id = Column(Integer, ForeignKey('otx.id')) + otx_id = Column(Integer, ForeignKey(Otx.id)) + #otx_id = Column(Integer) def chain(self): diff --git a/apps/cic-eth/cic_eth/db/models/otx.py b/apps/cic-eth/cic_eth/db/models/otx.py deleted file mode 100644 index 6be5f53d..00000000 --- a/apps/cic-eth/cic_eth/db/models/otx.py +++ /dev/null @@ -1,680 +0,0 @@ -# standard imports -import datetime -import logging - -# external imports -from sqlalchemy import Column, Enum, String, Integer, DateTime, Text, or_, ForeignKey -from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method - -# local imports -from .base import SessionBase -from cic_eth.db.enum import ( - StatusEnum, - StatusBits, - status_str, - is_error_status, - ) -from cic_eth.db.error import TxStateChangeError - -logg = logging.getLogger() - - -class OtxStateLog(SessionBase): - - __tablename__ = 'otx_state_log' - - date = Column(DateTime, default=datetime.datetime.utcnow) - status = Column(Integer) - otx_id = Column(Integer, ForeignKey('otx.id')) - - - def __init__(self, otx): - self.otx_id = otx.id - self.status = otx.status - - -class Otx(SessionBase): - """Outgoing transactions with local origin. - - :param nonce: Transaction nonce - :type nonce: number - :param address: Ethereum address of recipient - NOT IN USE, REMOVE - :type address: str - :param tx_hash: Tranasction hash - :type tx_hash: str, 0x-hex - :param signed_tx: Signed raw transaction data - :type signed_tx: str, 0x-hex - """ - __tablename__ = 'otx' - - tracing = False - """Whether to enable queue state tracing""" - - nonce = Column(Integer) - date_created = Column(DateTime, default=datetime.datetime.utcnow) - tx_hash = Column(String(66)) - signed_tx = Column(Text) - status = Column(Integer) - block = Column(Integer) - - - def __set_status(self, status, session): - self.status |= status - session.add(self) - session.flush() - - - def __reset_status(self, status, session): - status_edit = ~status & self.status - self.status &= status_edit - session.add(self) - session.flush() - - - def __status_already_set(self, status): - r = bool(self.status & status) - if r: - logg.warning('status bit {} already set on {}'.format(status.name, self.tx_hash)) - return r - - - def __status_not_set(self, status): - r = not(self.status & status) - if r: - logg.warning('status bit {} not set on {}'.format(status.name, self.tx_hash)) - return r - - - def set_block(self, block, session=None): - """Set block number transaction was mined in. - - Only manipulates object, does not transaction or commit to backend. - - :param block: Block number - :type block: number - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - session = SessionBase.bind_session(session) - - if self.block != None: - SessionBase.release_session(session) - raise TxStateChangeError('Attempted set block {} when block was already {}'.format(block, self.block)) - self.block = block - session.add(self) - session.flush() - - SessionBase.release_session(session) - - - def waitforgas(self, session=None): - """Marks transaction as suspended pending gas funding. - - Only manipulates object, does not transaction or commit to backend. - - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - if self.__status_already_set(StatusBits.GAS_ISSUES): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if self.status & StatusBits.IN_NETWORK: - SessionBase.release_session(session) - raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.GAS_ISSUES, session) - self.__reset_status(StatusBits.QUEUED | StatusBits.DEFERRED, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def fubar(self, session=None): - """Marks transaction as "fubar." Any transaction marked this way is an anomaly and may be a symptom of a serious problem. - - Only manipulates object, does not transaction or commit to backend. - """ - if self.__status_already_set(StatusBits.UNKNOWN_ERROR): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('FUBAR cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if is_error_status(self.status): - SessionBase.release_session(session) - raise TxStateChangeError('FUBAR cannot be set on an entry with an error state already set ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.UNKNOWN_ERROR | StatusBits.FINAL, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def reject(self, session=None): - """Marks transaction as "rejected," which means the node rejected sending the transaction to the network. The nonce has not been spent, and the transaction should be replaced. - - Only manipulates object, does not transaction or commit to backend. - """ - if self.__status_already_set(StatusBits.NODE_ERROR): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('REJECTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if self.status & StatusBits.IN_NETWORK: - SessionBase.release_session(session) - raise TxStateChangeError('REJECTED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status))) - if is_error_status(self.status): - SessionBase.release_session(session) - raise TxStateChangeError('REJECTED cannot be set on an entry with an error state already set ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.NODE_ERROR | StatusBits.FINAL, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def override(self, manual=False, session=None): - """Marks transaction as manually overridden. - - Only manipulates object, does not transaction or commit to backend. - """ - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if self.status & StatusBits.IN_NETWORK: - SessionBase.release_session(session) - raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status))) - if self.status & StatusBits.OBSOLETE: - SessionBase.release_session(session) - raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already OBSOLETE ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.OBSOLETE, session) - #if manual: - # self.__set_status(StatusBits.MANUAL, session) - self.__reset_status(StatusBits.QUEUED | StatusBits.IN_NETWORK, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def manual(self, session=None): - - session = SessionBase.bind_session(session) - - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.MANUAL, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - def retry(self, session=None): - """Marks transaction as ready to retry after a timeout following a sendfail or a completed gas funding. - - Only manipulates object, does not transaction or commit to backend. - - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - if self.__status_already_set(StatusBits.QUEUED): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('RETRY cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if not is_error_status(self.status) and not StatusBits.IN_NETWORK & self.status > 0: - SessionBase.release_session(session) - raise TxStateChangeError('RETRY cannot be set on an entry that has no error ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.QUEUED, session) - self.__reset_status(StatusBits.GAS_ISSUES, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def readysend(self, session=None): - """Marks transaction as ready for initial send attempt. - - Only manipulates object, does not transaction or commit to backend. - - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - if self.__status_already_set(StatusBits.QUEUED): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('READYSEND cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if is_error_status(self.status): - SessionBase.release_session(session) - raise TxStateChangeError('READYSEND cannot be set on an errored state ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.QUEUED, session) - self.__reset_status(StatusBits.GAS_ISSUES, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def sent(self, session=None): - """Marks transaction as having been sent to network. - - Only manipulates object, does not transaction or commit to backend. - - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - if self.__status_already_set(StatusBits.IN_NETWORK): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('SENT cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.IN_NETWORK, session) - self.__reset_status(StatusBits.DEFERRED | StatusBits.QUEUED | StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def sendfail(self, session=None): - """Marks that an attempt to send the transaction to the network has failed. - - Only manipulates object, does not transaction or commit to backend. - - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - if self.__status_already_set(StatusBits.NODE_ERROR): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('SENDFAIL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if self.status & StatusBits.IN_NETWORK: - SessionBase.release_session(session) - raise TxStateChangeError('SENDFAIL cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status))) - - self.__set_status(StatusBits.LOCAL_ERROR | StatusBits.DEFERRED, session) - self.__reset_status(StatusBits.QUEUED | StatusBits.GAS_ISSUES, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def dequeue(self, session=None): - """Marks that a process to execute send attempt is underway - - Only manipulates object, does not transaction or commit to backend. - - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - if self.__status_not_set(StatusBits.QUEUED): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('QUEUED cannot be unset on an entry with FINAL state set ({})'.format(status_str(self.status))) - if self.status & StatusBits.IN_NETWORK: - SessionBase.release_session(session) - raise TxStateChangeError('QUEUED cannot be unset on an entry with IN_NETWORK state set ({})'.format(status_str(self.status))) - - self.__reset_status(StatusBits.QUEUED, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - - def minefail(self, block, session=None): - """Marks that transaction was mined but code execution did not succeed. - - Only manipulates object, does not transaction or commit to backend. - - :param block: Block number transaction was mined in. - :type block: number - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - if self.__status_already_set(StatusBits.NETWORK_ERROR): - return - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('REVERTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if not self.status & StatusBits.IN_NETWORK: - SessionBase.release_session(session) - raise TxStateChangeError('REVERTED cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status))) - - if block != None: - self.block = block - - self.__set_status(StatusBits.NETWORK_ERROR | StatusBits.FINAL, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def cancel(self, confirmed=False, session=None): - """Marks that the transaction has been succeeded by a new transaction with same nonce. - - If set to confirmed, the previous state must be OBSOLETED, and will transition to CANCELLED - a finalized state. Otherwise, the state must follow a non-finalized state, and will be set to OBSOLETED. - - Only manipulates object, does not transaction or commit to backend. - - :param confirmed: Whether transition is to a final state. - :type confirmed: bool - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('CANCEL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - - if confirmed: - if self.status > 0 and not self.status & StatusBits.OBSOLETE: - SessionBase.release_session(session) - raise TxStateChangeError('CANCEL can only be set on an entry marked OBSOLETE ({})'.format(status_str(self.status))) - self.__set_status(StatusEnum.CANCELLED, session) - else: - self.__set_status(StatusEnum.OBSOLETED, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - def success(self, block, session=None): - """Marks that transaction was successfully mined. - - Only manipulates object, does not transaction or commit to backend. - - :param block: Block number transaction was mined in. - :type block: number - :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist. - """ - - session = SessionBase.bind_session(session) - - if self.status & StatusBits.FINAL: - SessionBase.release_session(session) - raise TxStateChangeError('SUCCESS cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) - if not self.status & StatusBits.IN_NETWORK: - SessionBase.release_session(session) - raise TxStateChangeError('SUCCESS cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status))) - if is_error_status(self.status): - SessionBase.release_session(session) - raise TxStateChangeError('SUCCESS cannot be set on an entry with error state set ({})'.format(status_str(self.status))) - - if block != None: - self.block = block - self.__set_status(StatusEnum.SUCCESS, session) - - if self.tracing: - self.__state_log(session=session) - - SessionBase.release_session(session) - - - @staticmethod - def get(status=0, limit=4096, status_exact=True, session=None): - """Returns outgoing transaction lists by status. - - Status may either be matched exactly, or be an upper bound of the integer value of the status enum. - - :param status: Status value to use in query - :type status: cic_eth.db.enum.StatusEnum - :param limit: Max results to return - :type limit: number - :param status_exact: Whether or not to perform exact status match - :type bool: - :returns: List of transaction hashes - :rtype: tuple, where first element is transaction hash - """ - e = None - - session = SessionBase.bind_session(session) - - if status_exact: - e = session.query(Otx.tx_hash).filter(Otx.status==status).order_by(Otx.date_created.asc()).limit(limit).all() - else: - e = session.query(Otx.tx_hash).filter(Otx.status<=status).order_by(Otx.date_created.asc()).limit(limit).all() - - SessionBase.release_session(session) - return e - - - @staticmethod - def load(tx_hash, session=None): - """Retrieves the outgoing transaction record by transaction hash. - - :param tx_hash: Transaction hash - :type tx_hash: str, 0x-hex - """ - session = SessionBase.bind_session(session) - - q = session.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - - SessionBase.release_session(session) - - return q.first() - - - @staticmethod - def account(account_address): - """Retrieves all transaction hashes for which the given Ethereum address is sender or recipient. - - :param account_address: Ethereum address to use in query. - :type account_address: str, 0x-hex - :returns: Outgoing transactions - :rtype: tuple, where first element is transaction hash - """ - session = Otx.create_session() - q = session.query(Otx.tx_hash) - q = q.join(TxCache) - q = q.filter(or_(TxCache.sender==account_address, TxCache.recipient==account_address)) - txs = q.all() - session.close() - return list(txs) - - - def __state_log(self, session): - l = OtxStateLog(self) - session.add(l) - - - # TODO: it is not safe to return otx here unless session has been passed in - @staticmethod - def add(nonce, address, tx_hash, signed_tx, session=None): - external_session = session != None - - session = SessionBase.bind_session(session) - - otx = Otx(nonce, address, tx_hash, signed_tx) - session.add(otx) - session.flush() - if otx.tracing: - otx.__state_log(session=session) - session.flush() - - SessionBase.release_session(session) - - if not external_session: - return None - - return otx - - - def __init__(self, nonce, address, tx_hash, signed_tx): - self.nonce = nonce - self.tx_hash = tx_hash - self.signed_tx = signed_tx - self.status = StatusEnum.PENDING - signed_tx_bytes = bytes.fromhex(signed_tx[2:]) - - # sender_address = address_hex_from_signed_tx(signed_tx_bytes) - # logg.debug('decoded tx {}'.format(sender_address)) - - - -# TODO: Most of the methods on this object are obsolete, but it contains a static function for retrieving "expired" outgoing transactions that should be moved to Otx instead. -class OtxSync(SessionBase): - """Obsolete - """ - __tablename__ = 'otx_sync' - - blockchain = Column(String) - block_height_backlog = Column(Integer) - tx_height_backlog = Column(Integer) - block_height_session = Column(Integer) - tx_height_session = Column(Integer) - block_height_head = Column(Integer) - tx_height_head = Column(Integer) - date_created = Column(DateTime, default=datetime.datetime.utcnow) - date_updated = Column(DateTime) - - - def backlog(self, block_height=None, tx_height=None): - #session = OtxSync.create_session() - if block_height != None: - if tx_height == None: - raise ValueError('tx height missing') - self.block_height_backlog = block_height - self.tx_height_backlog = tx_height - #session.add(self) - self.date_updated = datetime.datetime.utcnow() - #session.commit() - block_height = self.block_height_backlog - tx_height = self.tx_height_backlog - #session.close() - return (block_height, tx_height) - - - def session(self, block_height=None, tx_height=None): - #session = OtxSync.create_session() - if block_height != None: - if tx_height == None: - raise ValueError('tx height missing') - self.block_height_session = block_height - self.tx_height_session = tx_height - #session.add(self) - self.date_updated = datetime.datetime.utcnow() - #session.commit() - block_height = self.block_height_session - tx_height = self.tx_height_session - #session.close() - return (block_height, tx_height) - - - def head(self, block_height=None, tx_height=None): - #session = OtxSync.create_session() - if block_height != None: - if tx_height == None: - raise ValueError('tx height missing') - self.block_height_head = block_height - self.tx_height_head = tx_height - #session.add(self) - self.date_updated = datetime.datetime.utcnow() - #session.commit() - block_height = self.block_height_head - tx_height = self.tx_height_head - #session.close() - return (block_height, tx_height) - - - @hybrid_property - def synced(self): - #return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.block_height_backlog - return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.tx_height_backlog - - - @staticmethod - def load(blockchain_string, session): - q = session.query(OtxSync) - q = q.filter(OtxSync.blockchain==blockchain_string) - return q.first() - - - @staticmethod - def latest(nonce): - session = SessionBase.create_session() - otx = session.query(Otx).filter(Otx.nonce==nonce).order_by(Otx.created.desc()).first() - session.close() - return otx - - - @staticmethod - def get_expired(datetime_threshold): - session = SessionBase.create_session() - q = session.query(Otx) - q = q.filter(Otx.date_created gas_balance: + s_nonce = celery.signature( + 'cic_eth.eth.nonce.reserve_nonce', + [ + address, + chain_spec_dict, + gas_provider, + ], + queue=queue, + ) + s_refill_gas = celery.signature( + 'cic_eth.eth.gas.refill_gas', + [ + chain_spec_dict, + ], + queue=queue, + ) + s_nonce.link(s_refill_gas) + s_nonce.apply_async() + wait_tasks = [] + for tx_hash in tx_hashes: + s = celery.signature( + 'cic_eth.queue.state.set_waitforgas', + [ + chain_spec_dict, + tx_hash, + ], + queue=queue, + ) + wait_tasks.append(s) + celery.group(wait_tasks)() + raise OutOfGasError('need to fill gas, required {}, had {}'.format(gas_required, gas_balance)) + + safe_gas = self.safe_gas_threshold_amount + if gas_balance < safe_gas: + s_nonce = celery.signature( + 'cic_eth.eth.nonce.reserve_nonce', + [ + address, + chain_spec_dict, + gas_provider, + ], + queue=queue, + ) + s_refill_gas = celery.signature( + 'cic_eth.eth.gas.refill_gas', + [ + chain_spec_dict, + ], + queue=queue, + ) + s_nonce.link(s_refill_gas) + s_nonce.apply_async() + logg.debug('requested refill from {} to {}'.format(gas_provider, address)) + ready_tasks = [] + for tx_hash in tx_hashes: + s = celery.signature( + 'cic_eth.queue.state.set_ready', + [ + chain_spec_dict, + tx_hash, + ], + queue=queue, + ) + ready_tasks.append(s) + celery.group(ready_tasks)() + + return txs + + +# TODO: if this method fails the nonce will be out of sequence. session needs to be extended to include the queue create, so that nonce is rolled back if the second sql query fails. Better yet, split each state change into separate tasks. +# TODO: method is too long, factor out code for clarity +@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3AndSignerTask) +def refill_gas(self, recipient_address, chain_spec_dict): + """Executes a native token transaction to fund the recipient's gas expenditures. + + :param recipient_address: Recipient in need of gas + :type recipient_address: str, 0x-hex + :param chain_str: Chain spec, string representation + :type chain_str: str + :raises AlreadyFillingGasError: A gas refill transaction for this address is already executing + :returns: Transaction hash. + :rtype: str, 0x-hex + """ + # essentials + chain_spec = ChainSpec.from_dict(chain_spec_dict) + queue = self.request.delivery_info.get('routing_key') + + # Determine value of gas tokens to send + # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce + # however, we will perform a 0-value transaction instead + zero_amount = False + session = SessionBase.create_session() + status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR + q = session.query(Otx.tx_hash) + q = q.join(TxCache) + q = q.filter(Otx.status.op('&')(StatusBits.FINAL.value)==0) + q = q.filter(TxCache.from_value!=0) + q = q.filter(TxCache.recipient==recipient_address) + c = q.count() + if c > 0: + logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address)))) + zero_amount = True + session.flush() + + # finally determine the value to send + refill_amount = 0 + if not zero_amount: + refill_amount = self.safe_gas_refill_amount + + # determine sender + gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) + session.flush() + + # set up evm RPC connection + rpc = RPCConnection.connect(chain_spec, 'default') + + # set up transaction builder + nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session) + gas_oracle = self.create_gas_oracle(rpc) + rpc_signer = RPCConnection.connect(chain_spec, 'signer') + c = Gas(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + + # build and add transaction + logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address)) + (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED) + logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex)) + cache_task = 'cic_eth.eth.gas.cache_gas_data' + register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session) + + # add transaction to send queue + s_status = celery.signature( + 'cic_eth.queue.state.set_ready', + [ + chain_spec.asdict(), + tx_hash_hex, + ], + queue=queue, + ) + t = s_status.apply_async() + + return tx_signed_raw_hex + + +@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask) +def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, default_factor=1.1): + """Create a new transaction from an existing one with same nonce and higher gas price. + + :param txold_hash_hex: Transaction to re-create + :type txold_hash_hex: str, 0x-hex + :param chain_str: Chain spec, string representation + :type chain_str: str + :param gas: Explicitly use the specified gas amount + :type gas: number + :param default_factor: Default factor by which to increment the gas price by + :type default_factor: float + :raises NotLocalTxError: Transaction does not exist in the local queue + :returns: Transaction hash + :rtype: str, 0x-hex + """ + session = SessionBase.create_session() + + otx = Otx.load(txold_hash_hex, session) + if otx == None: + session.close() + raise NotLocalTxError(txold_hash_hex) + + chain_spec = ChainSpec.from_dict(chain_spec_dict) + + tx_signed_raw_bytes = bytes.fromhex(otx.signed_tx) + tx = unpack(tx_signed_raw_bytes, chain_spec) + logg.debug('resend otx {} {}'.format(tx, otx.signed_tx)) + + queue = self.request.delivery_info.get('routing_key') + + logg.debug('before {}'.format(tx)) + + rpc = RPCConnection.connect(chain_spec, 'default') + new_gas_price = gas + if new_gas_price == None: + o = price() + r = rpc.do(o) + current_gas_price = int(r, 16) + if tx['gasPrice'] > current_gas_price: + logg.info('Network gas price {} is lower than overdue tx gas price {}'.format(curent_gas_price, tx['gasPrice'])) + #tx['gasPrice'] = int(tx['gasPrice'] * default_factor) + new_gas_price = tx['gasPrice'] + 1 + else: + new_gas_price = int(tx['gasPrice'] * default_factor) + #if gas_price > new_gas_price: + # tx['gasPrice'] = gas_price + #else: + # tx['gasPrice'] = new_gas_price + + + rpc_signer = RPCConnection.connect(chain_spec, 'signer') + gas_oracle = OverrideGasOracle(price=new_gas_price, conn=rpc) + + c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle) + logg.debug('change gas price from old {} to new {} for tx {}'.format(tx['gasPrice'], new_gas_price, tx)) + tx['gasPrice'] = new_gas_price + (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx) + queue_create( + chain_spec, + tx['nonce'], + tx['from'], + tx_hash_hex, + tx_signed_raw_hex, + session=session, + ) + TxCache.clone(txold_hash_hex, tx_hash_hex, session=session) + session.close() + + s = create_check_gas_task( + [tx_signed_raw_hex], + chain_spec, + tx['from'], + tx['gasPrice'] * tx['gas'], + [tx_hash_hex], + queue=queue, + ) + s.apply_async() + + return tx_hash_hex + + diff --git a/apps/cic-eth/cic_eth/eth/nonce.py b/apps/cic-eth/cic_eth/eth/nonce.py index fa15ad79..412e18ee 100644 --- a/apps/cic-eth/cic_eth/eth/nonce.py +++ b/apps/cic-eth/cic_eth/eth/nonce.py @@ -1,9 +1,23 @@ +# standard imports +import logging + +# external imports +import celery +from chainlib.eth.address import is_checksum_address + # local imports +from cic_eth.db.models.role import AccountRole +from cic_eth.db.models.base import SessionBase +from cic_eth.task import CriticalSQLAlchemyTask from cic_eth.db.models.nonce import ( Nonce, NonceReservation, ) +celery_app = celery.current_app +logg = logging.getLogger() + + class CustodialTaskNonceOracle(): """Ensures atomic nonce increments for all transactions across all tasks and threads. @@ -30,3 +44,36 @@ class CustodialTaskNonceOracle(): """ r = NonceReservation.release(self.address, self.uuid, session=self.session) return r[1] + + +@celery_app.task(bind=True, base=CriticalSQLAlchemyTask) +def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None): + + self.log_banner() + + session = SessionBase.create_session() + + address = None + if signer_address == None: + address = chained_input + logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input)) + else: + if is_checksum_address(signer_address): + address = signer_address + logg.debug('explicit address for reserve nonce {}'.format(signer_address)) + else: + address = AccountRole.get_address(signer_address, session=session) + logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address)) + + if not is_checksum_address(address): + raise ValueError('invalid result when resolving address for nonce {}'.format(address)) + + root_id = self.request.root_id + r = NonceReservation.next(address, root_id, session=session) + logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0])) + + session.commit() + + session.close() + + return chained_input diff --git a/apps/cic-eth/cic_eth/eth/tx.py b/apps/cic-eth/cic_eth/eth/tx.py index a930b847..05bef05d 100644 --- a/apps/cic-eth/cic_eth/eth/tx.py +++ b/apps/cic-eth/cic_eth/eth/tx.py @@ -1,76 +1,34 @@ # standard imports import logging -# third-party imports +# external imports import celery -import requests -from chainlib.eth.constant import ZERO_ADDRESS from chainlib.chain import ChainSpec from chainlib.eth.address import is_checksum_address -from chainlib.eth.gas import ( - balance, - price, - ) -from chainlib.eth.error import ( - EthException, - NotFoundEthException, - ) +from chainlib.eth.error import NotFoundEthException from chainlib.eth.tx import ( transaction, receipt, raw, - TxFormat, - TxFactory, - unpack, ) from chainlib.connection import RPCConnection from chainlib.hash import keccak256_hex_to_hex -from chainlib.eth.gas import ( - Gas, - OverrideGasOracle, - ) -from chainlib.eth.contract import ( - abi_decode_single, - ABIContractType, - ) from hexathon import ( add_0x, strip_0x, ) +from chainqueue.db.models.tx import Otx +from chainqueue.db.models.tx import TxCache +from chainqueue.db.enum import StatusBits +from chainqueue.error import NotLocalTxError # local imports -from cic_eth.db import ( - Otx, - SessionBase, - ) -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.nonce import NonceReservation -from cic_eth.db.models.lock import Lock -from cic_eth.db.models.role import AccountRole -from cic_eth.db.enum import ( - LockEnum, - StatusBits, - ) -from cic_eth.error import PermanentTxError -from cic_eth.error import TemporaryTxError -from cic_eth.error import NotLocalTxError -#from cic_eth.queue.tx import create as queue_create -from cic_eth.queue.tx import ( - get_tx, - register_tx, - get_nonce_tx, - create as queue_create, - ) -from cic_eth.error import OutOfGasError -from cic_eth.error import LockedError -from cic_eth.eth.gas import ( - create_check_gas_task, - ) -from cic_eth.eth.nonce import CustodialTaskNonceOracle +from cic_eth.db import SessionBase from cic_eth.error import ( - AlreadyFillingGasError, - EthError, + PermanentTxError, + TemporaryTxError, ) +from cic_eth.eth.gas import create_check_gas_task from cic_eth.admin.ctrl import lock_send from cic_eth.task import ( CriticalSQLAlchemyTask, @@ -86,127 +44,6 @@ logg = logging.getLogger() MAX_NONCE_ATTEMPTS = 3 -# TODO this function is too long -@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task) -def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None): - """Check the gas level of the sender address of a transaction. - - If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser. - - If account balance is sufficient, but level of gas before spend is below "safe" threshold, gas refill is requested, and execution continues normally. - - :param tx_hashes: Transaction hashes due to be submitted - :type tx_hashes: list of str, 0x-hex - :param chain_spec_dict: Chain spec dict representation - :type chain_spec_dict: dict - :param txs: Signed raw transaction data, corresponding to tx_hashes - :type txs: list of str, 0x-hex - :param address: Sender address - :type address: str, 0x-hex - :param gas_required: Gas limit * gas price for transaction, (optional, if not set will be retrived from transaction data) - :type gas_required: int - :return: Signed raw transaction data list - :rtype: param txs, unchanged - """ - if len(txs) == 0: - for i in range(len(tx_hashes)): - o = get_tx(tx_hashes[i]) - txs.append(o['signed_tx']) - if address == None: - address = o['address'] - - #if not web3.Web3.isChecksumAddress(address): - if not is_checksum_address(address): - raise ValueError('invalid address {}'.format(address)) - - chain_spec = ChainSpec.from_dict(chain_spec_dict) - - queue = self.request.delivery_info.get('routing_key') - - conn = RPCConnection.connect(chain_spec) - - # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx - gas_balance = 0 - try: - o = balance(address) - r = conn.do(o) - conn.disconnect() - gas_balance = abi_decode_single(ABIContractType.UINT256, r) - except EthException as e: - conn.disconnect() - raise EthError('gas_balance call for {}: {}'.format(address, e)) - - logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required)) - session = SessionBase.create_session() - gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) - session.close() - - if gas_required > gas_balance: - s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', - [ - address, - gas_provider, - ], - queue=queue, - ) - s_refill_gas = celery.signature( - 'cic_eth.eth.tx.refill_gas', - [ - chain_spec_dict, - ], - queue=queue, - ) - s_nonce.link(s_refill_gas) - s_nonce.apply_async() - wait_tasks = [] - for tx_hash in tx_hashes: - s = celery.signature( - 'cic_eth.queue.tx.set_waitforgas', - [ - tx_hash, - ], - queue=queue, - ) - wait_tasks.append(s) - celery.group(wait_tasks)() - raise OutOfGasError('need to fill gas, required {}, had {}'.format(gas_required, gas_balance)) - - safe_gas = self.safe_gas_threshold_amount - if gas_balance < safe_gas: - s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', - [ - address, - gas_provider, - ], - queue=queue, - ) - s_refill_gas = celery.signature( - 'cic_eth.eth.tx.refill_gas', - [ - chain_spec_dict, - ], - queue=queue, - ) - s_nonce.link(s_refill_gas) - s_nonce.apply_async() - logg.debug('requested refill from {} to {}'.format(gas_provider, address)) - ready_tasks = [] - for tx_hash in tx_hashes: - s = celery.signature( - 'cic_eth.queue.tx.set_ready', - [ - tx_hash, - ], - queue=queue, - ) - ready_tasks.append(s) - celery.group(ready_tasks)() - - return txs - - # TODO: chain chainable transactions that use hashes as inputs may be chained to this function to output signed txs instead. @celery_app.task(bind=True, base=CriticalSQLAlchemyTask) def hashes_to_txs(self, tx_hashes): @@ -268,7 +105,7 @@ def send(self, txs, chain_spec_dict): chain_spec = ChainSpec.from_dict(chain_spec_dict) - tx_hex = txs[0] + tx_hex = add_0x(txs[0]) tx_hash_hex = add_0x(keccak256_hex_to_hex(tx_hex)) @@ -278,8 +115,9 @@ def send(self, txs, chain_spec_dict): r = None s_set_sent = celery.signature( - 'cic_eth.queue.tx.set_sent_status', + 'cic_eth.queue.state.set_sent', [ + chain_spec_dict, tx_hash_hex, False ], @@ -296,7 +134,10 @@ def send(self, txs, chain_spec_dict): if len(tx_tail) > 0: s = celery.signature( 'cic_eth.eth.tx.send', - [tx_tail], + [ + tx_tail, + chain_spec_dict, + ], queue=queue, ) s.apply_async() @@ -304,196 +145,6 @@ def send(self, txs, chain_spec_dict): return tx_hash_hex -# TODO: if this method fails the nonce will be out of sequence. session needs to be extended to include the queue create, so that nonce is rolled back if the second sql query fails. Better yet, split each state change into separate tasks. -# TODO: method is too long, factor out code for clarity -@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3AndSignerTask) -def refill_gas(self, recipient_address, chain_spec_dict): - """Executes a native token transaction to fund the recipient's gas expenditures. - - :param recipient_address: Recipient in need of gas - :type recipient_address: str, 0x-hex - :param chain_str: Chain spec, string representation - :type chain_str: str - :raises AlreadyFillingGasError: A gas refill transaction for this address is already executing - :returns: Transaction hash. - :rtype: str, 0x-hex - """ - # essentials - chain_spec = ChainSpec.from_dict(chain_spec_dict) - queue = self.request.delivery_info.get('routing_key') - - # Determine value of gas tokens to send - # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce - # however, we will perform a 0-value transaction instead - zero_amount = False - session = SessionBase.create_session() - status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR - q = session.query(Otx.tx_hash) - q = q.join(TxCache) - q = q.filter(Otx.status.op('&')(StatusBits.FINAL.value)==0) - q = q.filter(TxCache.from_value!=0) - q = q.filter(TxCache.recipient==recipient_address) - c = q.count() - if c > 0: - logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address)))) - zero_amount = True - session.flush() - - # finally determine the value to send - refill_amount = 0 - if not zero_amount: - refill_amount = self.safe_gas_refill_amount - - # determine sender - gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) - session.flush() - - # set up evm RPC connection - rpc = RPCConnection.connect(chain_spec, 'default') - - # set up transaction builder - nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session) - gas_oracle = self.create_gas_oracle(rpc) - rpc_signer = RPCConnection.connect(chain_spec, 'signer') - c = Gas(signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id()) - - # build and add transaction - logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address)) - (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED) - logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex)) - cache_task = 'cic_eth.eth.tx.cache_gas_data' - register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session) - - # add transaction to send queue - s_status = celery.signature( - 'cic_eth.queue.tx.set_ready', - [ - tx_hash_hex, - ], - queue=queue, - ) - t = s_status.apply_async() - - return tx_signed_raw_hex - - -@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask) -def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, default_factor=1.1): - """Create a new transaction from an existing one with same nonce and higher gas price. - - :param txold_hash_hex: Transaction to re-create - :type txold_hash_hex: str, 0x-hex - :param chain_str: Chain spec, string representation - :type chain_str: str - :param gas: Explicitly use the specified gas amount - :type gas: number - :param default_factor: Default factor by which to increment the gas price by - :type default_factor: float - :raises NotLocalTxError: Transaction does not exist in the local queue - :returns: Transaction hash - :rtype: str, 0x-hex - """ - session = SessionBase.create_session() - - q = session.query(Otx) - q = q.filter(Otx.tx_hash==txold_hash_hex) - otx = q.first() - if otx == None: - session.close() - raise NotLocalTxError(txold_hash_hex) - - chain_spec = ChainSpec.from_dict(chain_spec_dict) - - tx_signed_raw_bytes = bytes.fromhex(otx.signed_tx[2:]) - tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id()) - logg.debug('resend otx {} {}'.format(tx, otx.signed_tx)) - - queue = self.request.delivery_info.get('routing_key') - - logg.debug('before {}'.format(tx)) - - rpc = RPCConnection.connect(chain_spec, 'default') - new_gas_price = gas - if new_gas_price == None: - o = price() - r = rpc.do(o) - current_gas_price = int(r, 16) - if tx['gasPrice'] > current_gas_price: - logg.info('Network gas price {} is lower than overdue tx gas price {}'.format(curent_gas_price, tx['gasPrice'])) - #tx['gasPrice'] = int(tx['gasPrice'] * default_factor) - new_gas_price = tx['gasPrice'] + 1 - else: - new_gas_price = int(tx['gasPrice'] * default_factor) - #if gas_price > new_gas_price: - # tx['gasPrice'] = gas_price - #else: - # tx['gasPrice'] = new_gas_price - - - rpc_signer = RPCConnection.connect(chain_spec, 'signer') - gas_oracle = OverrideGasOracle(price=new_gas_price, conn=rpc) - - c = TxFactory(signer=rpc_signer, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id()) - logg.debug('change gas price from old {} to new {} for tx {}'.format(tx['gasPrice'], new_gas_price, tx)) - tx['gasPrice'] = new_gas_price - (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx) - queue_create( - tx['nonce'], - tx['from'], - tx_hash_hex, - tx_signed_raw_hex, - chain_spec, - session=session, - ) - TxCache.clone(txold_hash_hex, tx_hash_hex, session=session) - session.close() - - s = create_check_gas_task( - [tx_signed_raw_hex], - chain_spec, - tx['from'], - tx['gasPrice'] * tx['gas'], - [tx_hash_hex], - queue=queue, - ) - s.apply_async() - - return tx_hash_hex - - -@celery_app.task(bind=True, base=CriticalSQLAlchemyTask) -def reserve_nonce(self, chained_input, signer_address=None): - - self.log_banner() - - session = SessionBase.create_session() - - address = None - if signer_address == None: - address = chained_input - logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input)) - else: - #if web3.Web3.isChecksumAddress(signer_address): - if is_checksum_address(signer_address): - address = signer_address - logg.debug('explicit address for reserve nonce {}'.format(signer_address)) - else: - address = AccountRole.get_address(signer_address, session=session) - logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address)) - - if not is_checksum_address(address): - raise ValueError('invalid result when resolving address for nonce {}'.format(address)) - - root_id = self.request.root_id - r = NonceReservation.next(address, root_id) - logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0])) - - session.commit() - - session.close() - - return chained_input - @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task) def sync_tx(self, tx_hash_hex, chain_spec_dict): @@ -525,7 +176,7 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict): logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success)) s = celery.signature( - 'cic_eth.queue.tx.set_final_status', + 'cic_eth.queue.state.set_final', [ tx_hash_hex, rcpt['blockNumber'], @@ -537,7 +188,7 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict): logg.debug('sync tx {} mempool'.format(tx_hash_hex)) s = celery.signature( - 'cic_eth.queue.tx.set_sent_status', + 'cic_eth.queue.state.set_sent', [ tx_hash_hex, ], @@ -573,7 +224,7 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict): # # tx_signed_raw_hex = r[0] # tx_signed_bytes = bytes.fromhex(tx_signed_raw_hex[2:]) -# tx = unpack_signed_raw_tx(tx_signed_bytes, chain_spec.chain_id()) +# tx = unpack(tx_signed_bytes, chain_spec) # # queue = self.request.delivery_info['routing_key'] # @@ -589,39 +240,3 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict): # return txpending_hash_hex -# TODO: Move to cic_eth.eth.gas -@celery_app.task(base=CriticalSQLAlchemyTask) -def cache_gas_data( - tx_hash_hex, - tx_signed_raw_hex, - chain_spec_dict, - ): - """Helper function for otx_cache_parse_tx - - :param tx_hash_hex: Transaction hash - :type tx_hash_hex: str, 0x-hex - :param tx: Signed raw transaction - :type tx: str, 0x-hex - :returns: Transaction hash and id of cache element in storage backend, respectively - :rtype: tuple - """ - chain_spec = ChainSpec.from_dict(chain_spec_dict) - tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) - tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id()) - - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - tx['to'], - ZERO_ADDRESS, - ZERO_ADDRESS, - tx['value'], - tx['value'], - ) - - session = SessionBase.create_session() - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id - session.close() - return (tx_hash_hex, cache_id) diff --git a/apps/cic-eth/cic_eth/ext/tx.py b/apps/cic-eth/cic_eth/ext/tx.py index 1dba80eb..d635fd54 100644 --- a/apps/cic-eth/cic_eth/ext/tx.py +++ b/apps/cic-eth/cic_eth/ext/tx.py @@ -18,13 +18,14 @@ from chainlib.eth.erc20 import ERC20 from hexathon import strip_0x from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token +from chainqueue.db.models.otx import Otx +from chainqueue.db.enum import StatusEnum +from chainqueue.query import get_tx_cache # local imports -from cic_eth.db.models.otx import Otx -from cic_eth.db.enum import StatusEnum -from cic_eth.queue.tx import get_tx_cache from cic_eth.queue.time import tx_times from cic_eth.task import BaseTask +from cic_eth.db.models.base import SessionBase celery_app = celery.current_app logg = logging.getLogger() @@ -168,14 +169,16 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True): if isinstance(tx_batches, dict): tx_batches = [tx_batches] + session = SessionBase.create_session() + for b in tx_batches: for v in b.values(): tx = None k = None try: hx = strip_0x(v) - tx = unpack(bytes.fromhex(hx), chain_spec.chain_id()) - txc = get_tx_cache(tx['hash']) + tx = unpack(bytes.fromhex(hx), chain_spec) + txc = get_tx_cache(chain_spec, tx['hash'], session) txc['timestamp'] = int(txc['date_created'].timestamp()) txc['hash'] = txc['tx_hash'] tx = txc @@ -185,6 +188,8 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True): k = '{}.{}.{}'.format(tx['timestamp'], tx['sender'], tx['nonce']) txs_by_block[k] = tx + session.close() + txs = [] ks = list(txs_by_block.keys()) ks.sort() @@ -192,4 +197,5 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True): ks.reverse() for k in ks: txs.append(txs_by_block[k]) + return txs diff --git a/apps/cic-eth/cic_eth/queue/__init__.py b/apps/cic-eth/cic_eth/queue/__init__.py index 8acee1b9..d53e2500 100644 --- a/apps/cic-eth/cic_eth/queue/__init__.py +++ b/apps/cic-eth/cic_eth/queue/__init__.py @@ -1,4 +1,2 @@ # third-party imports import celery - -from .tx import get_tx diff --git a/apps/cic-eth/cic_eth/queue/balance.py b/apps/cic-eth/cic_eth/queue/balance.py index 1a5cf2db..a927a17a 100644 --- a/apps/cic-eth/cic_eth/queue/balance.py +++ b/apps/cic-eth/cic_eth/queue/balance.py @@ -5,15 +5,15 @@ import logging import celery from chainlib.chain import ChainSpec from hexathon import strip_0x - -# local imports -from cic_eth.db import SessionBase -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.tx import TxCache -from cic_eth.db.enum import ( +from chainqueue.db.models.otx import Otx +from chainqueue.db.models.tx import TxCache +from chainqueue.db.enum import ( StatusBits, dead, ) + +# local imports +from cic_eth.db import SessionBase from cic_eth.task import CriticalSQLAlchemyTask celery_app = celery.current_app diff --git a/apps/cic-eth/cic_eth/queue/lock.py b/apps/cic-eth/cic_eth/queue/lock.py new file mode 100644 index 00000000..baa5accb --- /dev/null +++ b/apps/cic-eth/cic_eth/queue/lock.py @@ -0,0 +1,48 @@ +# external imports +from chainqueue.db.models.otx import Otx +import celery + +# local imports +from cic_eth.task import CriticalSQLAlchemyTask +from cic_eth.db import SessionBase +from cic_eth.db.models.lock import Lock + +celery_app = celery.current_app + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def get_lock(address=None): + """Retrieve all active locks + + If address is set, the query will look up the lock for the specified address only. A list of zero or one elements is returned, depending on whether a lock is set or not. + + :param address: Get lock for only the specified address + :type address: str, 0x-hex + :returns: List of locks + :rtype: list of dicts + """ + session = SessionBase.create_session() + q = session.query( + Lock.date_created, + Lock.address, + Lock.flags, + Otx.tx_hash, + ) + q = q.join(Otx, isouter=True) + if address != None: + q = q.filter(Lock.address==address) + else: + q = q.order_by(Lock.date_created.asc()) + + locks = [] + for lock in q.all(): + o = { + 'date': lock[0], + 'address': lock[1], + 'tx_hash': lock[3], + 'flags': lock[2], + } + locks.append(o) + session.close() + + return locks diff --git a/apps/cic-eth/cic_eth/queue/query.py b/apps/cic-eth/cic_eth/queue/query.py new file mode 100644 index 00000000..1572289b --- /dev/null +++ b/apps/cic-eth/cic_eth/queue/query.py @@ -0,0 +1,157 @@ +# standard imports +import datetime + +# external imports +import celery +from chainlib.chain import ChainSpec +from chainlib.eth.tx import unpack +import chainqueue.query +from chainqueue.db.enum import ( + StatusEnum, + is_alive, + ) +from sqlalchemy import func +from sqlalchemy import or_ +from chainqueue.db.models.tx import TxCache +from chainqueue.db.models.otx import Otx + +# local imports +from cic_eth.db.enum import LockEnum +from cic_eth.task import CriticalSQLAlchemyTask +from cic_eth.db.models.lock import Lock +from cic_eth.db.models.base import SessionBase + +celery_app = celery.current_app + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def get_tx_cache(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.query.get_tx_cache(chain_spec, tx_hash, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def get_tx(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.query.get_tx(chain_spec, tx_hash) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack) + session.close() + return r + + +def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None): + return chainqueue.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack) + + +def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None): + return chainqueue.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack) + + +def get_nonce_tx(chain_spec, nonce, sender): + return get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack) + + +def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None): + """Returns the next pending transaction, specifically the transaction with the lowest nonce, for every recipient that has pending transactions. + + Will omit addresses that have the LockEnum.SEND bit in Lock set. + + (TODO) Will not return any rows if LockEnum.SEND bit in Lock is set for zero address. + + :param status: Defines the status used to filter as upcoming. + :type status: cic_eth.db.enum.StatusEnum + :param recipient: Ethereum address of recipient to return transaction for + :type recipient: str, 0x-hex + :param before: Only return transactions if their modification date is older than the given timestamp + :type before: datetime.datetime + :param chain_id: Chain id to use to parse signed transaction data + :type chain_id: number + :raises ValueError: Status is finalized, sent or never attempted sent + :returns: Transactions + :rtype: dict, with transaction hash as key, signed raw transaction as value + """ + session = SessionBase.bind_session(session) + q_outer = session.query( + TxCache.sender, + func.min(Otx.nonce).label('nonce'), + ) + q_outer = q_outer.join(TxCache) + q_outer = q_outer.join(Lock, isouter=True) + q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0)) + + if not is_alive(status): + SessionBase.release_session(session) + raise ValueError('not a valid non-final tx value: {}'.format(status)) + if status == StatusEnum.PENDING: + q_outer = q_outer.filter(Otx.status==status.value) + else: + q_outer = q_outer.filter(Otx.status.op('&')(status)==status) + + if not_status != None: + q_outer = q_outer.filter(Otx.status.op('&')(not_status)==0) + + if recipient != None: + q_outer = q_outer.filter(TxCache.recipient==recipient) + + q_outer = q_outer.group_by(TxCache.sender) + + txs = {} + + i = 0 + for r in q_outer.all(): + q = session.query(Otx) + q = q.join(TxCache) + q = q.filter(TxCache.sender==r.sender) + q = q.filter(Otx.nonce==r.nonce) + + if before != None: + q = q.filter(TxCache.date_checked 0 and limit == i: + break + + SessionBase.release_session(session) + + return txs + diff --git a/apps/cic-eth/cic_eth/queue/state.py b/apps/cic-eth/cic_eth/queue/state.py new file mode 100644 index 00000000..4eca85cf --- /dev/null +++ b/apps/cic-eth/cic_eth/queue/state.py @@ -0,0 +1,100 @@ +# external imports +from chainlib.chain import ChainSpec +import chainqueue.state + +# local imports +import celery +from cic_eth.task import CriticalSQLAlchemyTask +from cic_eth.db.models.base import SessionBase + +celery_app = celery.current_app + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_sent(chain_spec_dict, tx_hash, fail=False): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_sent(chain_spec, tx_hash, fail, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_final(chain_spec_dict, tx_hash, block=None, fail=False): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_final(chain_spec, tx_hash, block, fail, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_cancel(chain_spec_dict, tx_hash, manual=False): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_cancel(chain_spec, tx_hash, manual, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_rejected(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_rejected(chain_spec, tx_hash, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_fubar(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_fubar(chain_spec, tx_hash, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_manual(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_manual(chain_spec, tx_hash, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_ready(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_ready(chain_spec, tx_hash, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_reserved(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_reserved(chain_spec, tx_hash, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def set_waitforgas(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.set_waitforgas(chain_spec, tx_hash, session=session) + session.close() + return r + + +@celery_app.task(base=CriticalSQLAlchemyTask) +def get_state_log(chain_spec_dict, tx_hash): + chain_spec = ChainSpec.from_dict(chain_spec_dict) + session = SessionBase.create_session() + r = chainqueue.state.get_state_log(chain_spec, tx_hash, session=session) + session.close() + return r diff --git a/apps/cic-eth/cic_eth/queue/time.py b/apps/cic-eth/cic_eth/queue/time.py index f85e55c0..b3c74ece 100644 --- a/apps/cic-eth/cic_eth/queue/time.py +++ b/apps/cic-eth/cic_eth/queue/time.py @@ -7,10 +7,10 @@ from chainlib.chain import ChainSpec from chainlib.connection import RPCConnection from chainlib.eth.block import block_by_hash from chainlib.eth.tx import receipt +from chainqueue.db.models.otx import Otx +from chainqueue.error import NotLocalTxError # local imports -from cic_eth.db.models.otx import Otx -from cic_eth.error import NotLocalTxError from cic_eth.task import CriticalSQLAlchemyAndWeb3Task celery_app = celery.current_app @@ -18,7 +18,10 @@ celery_app = celery.current_app logg = logging.getLogger() -def tx_times(tx_hash, chain_spec): +def tx_times(tx_hash, chain_spec, session=None): + + session = SessionBase.bind_session(session) + rpc = RPCConnection.connect(chain_spec, 'default') time_pair = { 'network': None, @@ -35,8 +38,10 @@ def tx_times(tx_hash, chain_spec): logg.debug('error with getting timestamp details for {}: {}'.format(tx_hash, e)) pass - otx = Otx.load(tx_hash) + otx = Otx.load(tx_hash, session=session) if otx != None: time_pair['queue'] = int(otx['date_created'].timestamp()) + SessionBase.release_session(session) + return time_pair diff --git a/apps/cic-eth/cic_eth/queue/tx.py b/apps/cic-eth/cic_eth/queue/tx.py index 66b4e3a2..841f45b5 100644 --- a/apps/cic-eth/cic_eth/queue/tx.py +++ b/apps/cic-eth/cic_eth/queue/tx.py @@ -5,92 +5,50 @@ import datetime # external imports import celery +from chainqueue.db.models.otx import Otx +from chainqueue.db.models.otx import OtxStateLog +from chainqueue.db.models.tx import TxCache from hexathon import strip_0x from sqlalchemy import or_ from sqlalchemy import not_ from sqlalchemy import tuple_ from sqlalchemy import func +from chainlib.chain import ChainSpec from chainlib.eth.tx import unpack - -# local imports -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.otx import OtxStateLog -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.lock import Lock -from cic_eth.db import SessionBase -from cic_eth.db.enum import ( +import chainqueue.state +from chainqueue.db.enum import ( StatusEnum, - LockEnum, StatusBits, is_alive, dead, ) +from chainqueue.tx import create +from chainqueue.error import NotLocalTxError +from chainqueue.db.enum import status_str + +# local imports +from cic_eth.db.models.lock import Lock +from cic_eth.db import SessionBase +from cic_eth.db.enum import LockEnum from cic_eth.task import CriticalSQLAlchemyTask -from cic_eth.error import NotLocalTxError from cic_eth.error import LockedError -from cic_eth.db.enum import status_str celery_app = celery.current_app -#logg = celery_app.log.get_default_logger() logg = logging.getLogger() -def create(nonce, holder_address, tx_hash, signed_tx, chain_spec, obsolete_predecessors=True, session=None): - """Create a new transaction queue record. - - :param nonce: Transaction nonce - :type nonce: int - :param holder_address: Sender address - :type holder_address: str, 0x-hex - :param tx_hash: Transaction hash - :type tx_hash: str, 0x-hex - :param signed_tx: Signed raw transaction - :type signed_tx: str, 0x-hex - :param chain_spec: Chain spec to create transaction for - :type chain_spec: ChainSpec - :returns: transaction hash - :rtype: str, 0x-hash - """ +def queue_create(chain_spec, nonce, holder_address, tx_hash, signed_tx, session=None): session = SessionBase.bind_session(session) + lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session) if lock > 0: SessionBase.release_session(session) raise LockedError(lock) - o = Otx.add( - nonce=nonce, - address=holder_address, - tx_hash=tx_hash, - signed_tx=signed_tx, - session=session, - ) - session.flush() - - if obsolete_predecessors: - q = session.query(Otx) - q = q.join(TxCache) - q = q.filter(Otx.nonce==nonce) - q = q.filter(TxCache.sender==holder_address) - q = q.filter(Otx.tx_hash!=tx_hash) - q = q.filter(Otx.status.op('&')(StatusBits.FINAL)==0) - - for otx in q.all(): - logg.info('otx {} obsoleted by {}'.format(otx.tx_hash, tx_hash)) - try: - otx.cancel(confirmed=False, session=session) - except TxStateChangeError as e: - logg.exception('obsolete fail: {}'.format(e)) - session.close() - raise(e) - except Exception as e: - logg.exception('obsolete UNEXPECTED fail: {}'.format(e)) - session.close() - raise(e) - - - session.commit() + tx_hash = create(chain_spec, nonce, holder_address, tx_hash, signed_tx, chain_spec, session=session) + SessionBase.release_session(session) - logg.debug('queue created nonce {} from {} hash {}'.format(nonce, holder_address, tx_hash)) + return tx_hash @@ -111,14 +69,14 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No """ logg.debug('adding queue tx {}:{} -> {}'.format(chain_spec, tx_hash_hex, tx_signed_raw_hex)) tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex)) - tx = unpack(tx_signed_raw, chain_id=chain_spec.chain_id()) + tx = unpack(tx_signed_raw, chain_spec) - create( + tx_hash = queue_create( + chain_spec, tx['nonce'], tx['from'], tx_hash_hex, tx_signed_raw_hex, - chain_spec, session=session, ) @@ -134,683 +92,3 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No queue=queue, ) s_cache.apply_async() - - return (tx_hash_hex, tx_signed_raw_hex,) - - -# TODO: Replace set_* with single task for set status -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_sent_status(tx_hash, fail=False): - """Used to set the status after a send attempt - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :param fail: if True, will set a SENDFAIL status, otherwise a SENT status. (Default: False) - :type fail: boolean - :raises NotLocalTxError: If transaction not found in queue. - :returns: True if tx is known, False otherwise - :rtype: boolean - """ - session = SessionBase.create_session() - q = session.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - o = q.first() - if o == None: - logg.warning('not local tx, skipping {}'.format(tx_hash)) - session.close() - return False - - try: - if fail: - o.sendfail(session=session) - else: - o.sent(session=session) - except TxStateChangeError as e: - logg.exception('set sent fail: {}'.format(e)) - session.close() - raise(e) - except Exception as e: - logg.exception('set sent UNEXPECED fail: {}'.format(e)) - session.close() - raise(e) - - - session.commit() - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_final_status(tx_hash, block=None, fail=False): - """Used to set the status of an incoming transaction result. - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :param block: Block number if final status represents a confirmation on the network - :type block: number - :param fail: if True, will set a SUCCESS status, otherwise a REVERTED status. (Default: False) - :type fail: boolean - :raises NotLocalTxError: If transaction not found in queue. - """ - session = SessionBase.create_session() - q = session.query( - Otx.nonce.label('nonce'), - TxCache.sender.label('sender'), - Otx.id.label('otxid'), - ) - q = q.join(TxCache) - q = q.filter(Otx.tx_hash==tx_hash) - o = q.first() - - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - session.flush() - - nonce = o.nonce - sender = o.sender - otxid = o.otxid - - q = session.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - o = q.first() - - try: - if fail: - o.minefail(block, session=session) - else: - o.success(block, session=session) - session.commit() - except TxStateChangeError as e: - logg.exception('set final fail: {}'.format(e)) - session.close() - raise(e) - except Exception as e: - logg.exception('set final UNEXPECED fail: {}'.format(e)) - session.close() - raise(e) - - q = session.query(Otx) - q = q.join(TxCache) - q = q.filter(Otx.nonce==nonce) - q = q.filter(TxCache.sender==sender) - q = q.filter(Otx.tx_hash!=tx_hash) - - for otwo in q.all(): - try: - otwo.cancel(True, session=session) - except TxStateChangeError as e: - logg.exception('cancel non-final fail: {}'.format(e)) - session.close() - raise(e) - except Exception as e: - logg.exception('cancel non-final UNEXPECTED fail: {}'.format(e)) - session.close() - raise(e) - session.commit() - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_cancel(tx_hash, manual=False): - """Used to set the status when a transaction is cancelled. - - Will set the state to CANCELLED or OVERRIDDEN - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :param manual: If set, status will be OVERRIDDEN. Otherwise CANCELLED. - :type manual: boolean - :raises NotLocalTxError: If transaction not found in queue. - """ - - session = SessionBase.create_session() - o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first() - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - session.flush() - - try: - if manual: - o.override(session=session) - else: - o.cancel(session=session) - session.commit() - except TxStateChangeError as e: - logg.exception('set cancel fail: {}'.format(e)) - except Exception as e: - logg.exception('set cancel UNEXPECTED fail: {}'.format(e)) - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_rejected(tx_hash): - """Used to set the status when the node rejects sending a transaction to network - - Will set the state to REJECTED - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :raises NotLocalTxError: If transaction not found in queue. - """ - - session = SessionBase.create_session() - o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first() - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - session.flush() - - o.reject(session=session) - session.commit() - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_fubar(tx_hash): - """Used to set the status when an unexpected error occurs. - - Will set the state to FUBAR - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :raises NotLocalTxError: If transaction not found in queue. - """ - - session = SessionBase.create_session() - o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first() - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - session.flush() - - o.fubar(session=session) - session.commit() - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_manual(tx_hash): - """Used to set the status when queue is manually changed - - Will set the state to MANUAL - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :raises NotLocalTxError: If transaction not found in queue. - """ - - session = SessionBase.create_session() - o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first() - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - session.flush() - - o.manual(session=session) - session.commit() - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_ready(tx_hash): - """Used to mark a transaction as ready to be sent to network - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :raises NotLocalTxError: If transaction not found in queue. - """ - session = SessionBase.create_session() - o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first() - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - session.flush() - - if o.status & StatusBits.GAS_ISSUES or o.status == StatusEnum.PENDING: - o.readysend(session=session) - else: - o.retry(session=session) - - session.commit() - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_dequeue(tx_hash): - session = SessionBase.create_session() - q = session.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - o = q.first() - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - session.flush() - - o.dequeue(session=session) - session.commit() - session.close() - - return tx_hash - - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def set_waitforgas(tx_hash): - """Used to set the status when a transaction must be deferred due to gas refill - - Will set the state to WAITFORGAS - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :raises NotLocalTxError: If transaction not found in queue. - """ - - session = SessionBase.create_session() - o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first() - if o == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - session.flush() - - o.waitforgas(session=session) - session.commit() - session.close() - - return tx_hash - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def get_state_log(tx_hash): - - logs = [] - - session = SessionBase.create_session() - - q = session.query(OtxStateLog) - q = q.join(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - q = q.order_by(OtxStateLog.date.asc()) - for l in q.all(): - logs.append((l.date, l.status,)) - - session.close() - - return logs - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def get_tx_cache(tx_hash): - """Returns an aggregate dictionary of outgoing transaction data and metadata - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :raises NotLocalTxError: If transaction not found in queue. - :returns: Transaction data - :rtype: dict - """ - session = SessionBase.create_session() - q = session.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - otx = q.first() - - if otx == None: - session.close() - raise NotLocalTxError(tx_hash) - - session.flush() - - q = session.query(TxCache) - q = q.filter(TxCache.otx_id==otx.id) - txc = q.first() - - session.close() - - tx = { - 'tx_hash': otx.tx_hash, - 'signed_tx': otx.signed_tx, - 'nonce': otx.nonce, - 'status': status_str(otx.status), - 'status_code': otx.status, - 'source_token': txc.source_token_address, - 'destination_token': txc.destination_token_address, - 'block_number': txc.block_number, - 'tx_index': txc.tx_index, - 'sender': txc.sender, - 'recipient': txc.recipient, - 'from_value': int(txc.from_value), - 'to_value': int(txc.to_value), - 'date_created': txc.date_created, - 'date_updated': txc.date_updated, - 'date_checked': txc.date_checked, - } - - return tx - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def get_lock(address=None): - """Retrieve all active locks - - If address is set, the query will look up the lock for the specified address only. A list of zero or one elements is returned, depending on whether a lock is set or not. - - :param address: Get lock for only the specified address - :type address: str, 0x-hex - :returns: List of locks - :rtype: list of dicts - """ - session = SessionBase.create_session() - q = session.query( - Lock.date_created, - Lock.address, - Lock.flags, - Otx.tx_hash, - ) - q = q.join(Otx, isouter=True) - if address != None: - q = q.filter(Lock.address==address) - else: - q = q.order_by(Lock.date_created.asc()) - - locks = [] - for lock in q.all(): - o = { - 'date': lock[0], - 'address': lock[1], - 'tx_hash': lock[3], - 'flags': lock[2], - } - locks.append(o) - session.close() - - return locks - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def get_tx(tx_hash): - """Retrieve a transaction queue record by transaction hash - - :param tx_hash: Transaction hash of record to modify - :type tx_hash: str, 0x-hex - :raises NotLocalTxError: If transaction not found in queue. - :returns: nonce, address and signed_tx (raw signed transaction) - :rtype: dict - """ - session = SessionBase.create_session() - q = session.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - tx = q.first() - if tx == None: - session.close() - raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash)) - - o = { - 'otx_id': tx.id, - 'nonce': tx.nonce, - 'signed_tx': tx.signed_tx, - 'status': tx.status, - } - logg.debug('get tx {}'.format(o)) - session.close() - return o - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def get_nonce_tx(nonce, sender, chain_id): - """Retrieve all transactions for address with specified nonce - - :param nonce: Nonce - :type nonce: number - :param address: Ethereum address - :type address: str, 0x-hex - :returns: Transactions - :rtype: dict, with transaction hash as key, signed raw transaction as value - """ - session = SessionBase.create_session() - q = session.query(Otx) - q = q.join(TxCache) - q = q.filter(TxCache.sender==sender) - q = q.filter(Otx.nonce==nonce) - - txs = {} - for r in q.all(): - tx_signed_bytes = bytes.fromhex(r.signed_tx[2:]) - tx = unpack(tx_signed_bytes, chain_id) - if sender == None or tx['from'] == sender: - txs[r.tx_hash] = r.signed_tx - - session.close() - - return txs - - - -# TODO: pass chain spec instead of chain id -def get_paused_txs(status=None, sender=None, chain_id=0, session=None): - """Returns not finalized transactions that have been attempted sent without success. - - :param status: If set, will return transactions with this local queue status only - :type status: cic_eth.db.enum.StatusEnum - :param recipient: Recipient address to return transactions for - :type recipient: str, 0x-hex - :param chain_id: Numeric chain id to use to parse signed transaction data - :type chain_id: number - :raises ValueError: Status is finalized, sent or never attempted sent - :returns: Transactions - :rtype: dict, with transaction hash as key, signed raw transaction as value - """ - session = SessionBase.bind_session(session) - q = session.query(Otx) - - if status != None: - #if status == StatusEnum.PENDING or status >= StatusEnum.SENT: - if status == StatusEnum.PENDING or status & StatusBits.IN_NETWORK or not is_alive(status): - SessionBase.release_session(session) - raise ValueError('not a valid paused tx value: {}'.format(status)) - q = q.filter(Otx.status.op('&')(status.value)==status.value) - q = q.join(TxCache) - else: - q = q.filter(Otx.status>StatusEnum.PENDING.value) - q = q.filter(not_(Otx.status.op('&')(StatusBits.IN_NETWORK.value)>0)) - - if sender != None: - q = q.filter(TxCache.sender==sender) - - txs = {} - - for r in q.all(): - tx_signed_bytes = bytes.fromhex(r.signed_tx[2:]) - tx = unpack(tx_signed_bytes, chain_id) - if sender == None or tx['from'] == sender: - #gas += tx['gas'] * tx['gasPrice'] - txs[r.tx_hash] = r.signed_tx - - SessionBase.release_session(session) - - return txs - - -def get_status_tx(status, not_status=None, before=None, exact=False, limit=0, session=None): - """Retrieve transaction with a specific queue status. - - :param status: Status to match transactions with - :type status: str - :param before: If set, return only transactions older than the timestamp - :type status: datetime.dateTime - :param limit: Limit amount of returned transactions - :type limit: number - :returns: Transactions - :rtype: list of cic_eth.db.models.otx.Otx - """ - txs = {} - session = SessionBase.bind_session(session) - q = session.query(Otx) - q = q.join(TxCache) - # before = datetime.datetime.utcnow() - if before != None: - q = q.filter(TxCache.date_updated0) - if not_status != None: - q = q.filter(Otx.status.op('&')(not_status)==0) - q = q.order_by(Otx.nonce.asc(), Otx.date_created.asc()) - i = 0 - for o in q.all(): - if limit > 0 and i == limit: - break - txs[o.tx_hash] = o.signed_tx - i += 1 - SessionBase.release_session(session) - return txs - - -# TODO: move query to model -def get_upcoming_tx(status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, chain_id=0, session=None): - """Returns the next pending transaction, specifically the transaction with the lowest nonce, for every recipient that has pending transactions. - - Will omit addresses that have the LockEnum.SEND bit in Lock set. - - (TODO) Will not return any rows if LockEnum.SEND bit in Lock is set for zero address. - - :param status: Defines the status used to filter as upcoming. - :type status: cic_eth.db.enum.StatusEnum - :param recipient: Ethereum address of recipient to return transaction for - :type recipient: str, 0x-hex - :param before: Only return transactions if their modification date is older than the given timestamp - :type before: datetime.datetime - :param chain_id: Chain id to use to parse signed transaction data - :type chain_id: number - :raises ValueError: Status is finalized, sent or never attempted sent - :returns: Transactions - :rtype: dict, with transaction hash as key, signed raw transaction as value - """ - session = SessionBase.bind_session(session) - q_outer = session.query( - TxCache.sender, - func.min(Otx.nonce).label('nonce'), - ) - q_outer = q_outer.join(TxCache) - q_outer = q_outer.join(Lock, isouter=True) - q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0)) - - if not is_alive(status): - SessionBase.release_session(session) - raise ValueError('not a valid non-final tx value: {}'.format(status)) - if status == StatusEnum.PENDING: - q_outer = q_outer.filter(Otx.status==status.value) - else: - q_outer = q_outer.filter(Otx.status.op('&')(status)==status) - - if not_status != None: - q_outer = q_outer.filter(Otx.status.op('&')(not_status)==0) - - if recipient != None: - q_outer = q_outer.filter(TxCache.recipient==recipient) - - q_outer = q_outer.group_by(TxCache.sender) - - txs = {} - - i = 0 - for r in q_outer.all(): - q = session.query(Otx) - q = q.join(TxCache) - q = q.filter(TxCache.sender==r.sender) - q = q.filter(Otx.nonce==r.nonce) - - if before != None: - q = q.filter(TxCache.date_checked 0 and limit == i: - break - - SessionBase.release_session(session) - - return txs - - -@celery_app.task(base=CriticalSQLAlchemyTask) -def get_account_tx(address, as_sender=True, as_recipient=True, counterpart=None): - """Returns all local queue transactions for a given Ethereum address - - :param address: Ethereum address - :type address: str, 0x-hex - :param as_sender: If False, will omit transactions where address is sender - :type as_sender: bool - :param as_sender: If False, will omit transactions where address is recipient - :type as_sender: bool - :param counterpart: Only return transactions where this Ethereum address is the other end of the transaction (not in use) - :type counterpart: str, 0x-hex - :raises ValueError: If address is set to be neither sender nor recipient - :returns: Transactions - :rtype: dict, with transaction hash as key, signed raw transaction as value - """ - if not as_sender and not as_recipient: - raise ValueError('at least one of as_sender and as_recipient must be True') - - txs = {} - - session = SessionBase.create_session() - q = session.query(Otx) - q = q.join(TxCache) - if as_sender and as_recipient: - q = q.filter(or_(TxCache.sender==address, TxCache.recipient==address)) - elif as_sender: - q = q.filter(TxCache.sender==address) - else: - q = q.filter(TxCache.recipient==address) - q = q.order_by(Otx.nonce.asc(), Otx.date_created.asc()) - - results = q.all() - for r in results: - if txs.get(r.tx_hash) != None: - logg.debug('tx {} already recorded'.format(r.tx_hash)) - continue - txs[r.tx_hash] = r.signed_tx - session.close() - - return txs - diff --git a/apps/cic-eth/cic_eth/registry.py b/apps/cic-eth/cic_eth/registry.py index 247bf994..e1e481ef 100644 --- a/apps/cic-eth/cic_eth/registry.py +++ b/apps/cic-eth/cic_eth/registry.py @@ -13,7 +13,7 @@ def connect_token_registry(rpc, chain_spec): registry = CICRegistry(chain_spec, rpc) token_registry_address = registry.by_name('TokenRegistry') logg.debug('using token registry address {}'.format(token_registry_address)) - lookup = TokenIndexLookup(token_registry_address) + lookup = TokenIndexLookup(chain_spec, token_registry_address) CICRegistry.add_lookup(lookup) @@ -21,7 +21,7 @@ def connect_declarator(rpc, chain_spec, trusted_addresses): registry = CICRegistry(chain_spec, rpc) declarator_address = registry.by_name('AddressDeclarator') logg.debug('using declarator address {}'.format(declarator_address)) - lookup = AddressDeclaratorLookup(declarator_address, trusted_addresses) + lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses) CICRegistry.add_lookup(lookup) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py index 9f4ef332..b93f1184 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py @@ -17,26 +17,25 @@ from chainlib.eth.tx import unpack from chainlib.connection import RPCConnection from chainsyncer.error import SyncDone from hexathon import strip_0x +from chainqueue.db.enum import ( + StatusEnum, + StatusBits, + ) +from chainqueue.error import NotLocalTxError +from chainqueue.state import set_reserved # local imports import cic_eth from cic_eth.db import SessionBase -from cic_eth.db.enum import StatusEnum -from cic_eth.db.enum import StatusBits from cic_eth.db.enum import LockEnum from cic_eth.db import dsn_from_config -from cic_eth.queue.tx import ( - get_upcoming_tx, - set_dequeue, - ) +from cic_eth.queue.query import get_upcoming_tx from cic_eth.admin.ctrl import lock_send from cic_eth.eth.tx import send as task_tx_send from cic_eth.error import ( PermanentTxError, TemporaryTxError, - NotLocalTxError, ) -#from cic_eth.eth.util import unpack_signed_raw_tx_hex logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -92,7 +91,6 @@ class DispatchSyncer: def __init__(self, chain_spec): self.chain_spec = chain_spec - self.chain_id = chain_spec.chain_id() def chain(self): @@ -103,13 +101,14 @@ class DispatchSyncer: c = len(txs.keys()) logg.debug('processing {} txs {}'.format(c, list(txs.keys()))) chain_str = str(self.chain_spec) + session = SessionBase.create_session() for k in txs.keys(): tx_raw = txs[k] tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw)) - tx = unpack(tx_raw_bytes, self.chain_spec.chain_id()) + tx = unpack(tx_raw_bytes, self.chain_spec) try: - set_dequeue(tx['hash']) + set_reserved(self.chain_spec, tx['hash'], session=session) except NotLocalTxError as e: logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash'])) continue @@ -140,7 +139,7 @@ class DispatchSyncer: while run: txs = {} typ = StatusBits.QUEUED - utxs = get_upcoming_tx(typ, chain_id=self.chain_id) + utxs = get_upcoming_tx(self.chain_spec, typ) for k in utxs.keys(): txs[k] = utxs[k] self.process(w3, txs) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py index 458e2851..b0219093 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py @@ -2,14 +2,18 @@ import logging # external imports -from hexathon import add_0x +from hexathon import ( + add_0x, + strip_0x, + ) +from chainlib.eth.tx import unpack +from chainqueue.db.enum import StatusBits +from chainqueue.db.models.tx import TxCache +from chainqueue.db.models.otx import Otx +from chainqueue.query import get_paused_tx_cache as get_paused_tx # local imports -from cic_eth.db.enum import StatusBits from cic_eth.db.models.base import SessionBase -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.otx import Otx -from cic_eth.queue.tx import get_paused_txs from cic_eth.eth.gas import create_check_gas_task from .base import SyncFilter @@ -24,13 +28,13 @@ class GasFilter(SyncFilter): def filter(self, conn, block, tx, session): - tx_hash_hex = add_0x(tx.hash) if tx.value > 0: + tx_hash_hex = add_0x(tx.hash) logg.debug('gas refill tx {}'.format(tx_hash_hex)) session = SessionBase.bind_session(session) q = session.query(TxCache.recipient) q = q.join(Otx) - q = q.filter(Otx.tx_hash==tx_hash_hex) + q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex)) r = q.first() if r == None: @@ -38,7 +42,7 @@ class GasFilter(SyncFilter): SessionBase.release_session(session) return - txs = get_paused_txs(StatusBits.GAS_ISSUES, r[0], self.chain_spec.chain_id(), session=session) + txs = get_paused_tx(self.chain_spec, status=StatusBits.GAS_ISSUES, sender=r[0], session=session, decoder=unpack) SessionBase.release_session(session) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py index 40eef8ad..841e7d75 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py @@ -35,9 +35,10 @@ class RegistrationFilter(SyncFilter): address = to_checksum_address(add_0x(address_hex)) logg.info('request token gift to {}'.format(address)) s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ address, + self.chain_spec.asdict(), ], queue=self.queue, ) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py index 9c09bafe..d6646b6c 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py @@ -61,7 +61,7 @@ class TransferAuthFilter(SyncFilter): } s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ [token_data], sender, @@ -69,7 +69,7 @@ class TransferAuthFilter(SyncFilter): queue=self.queue, ) s_approve = celery.signature( - 'cic_eth.eth.token.approve', + 'cic_eth.eth.erc20.approve', [ sender, recipient, diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py index 52ec2532..4c3a1ae0 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py @@ -1,16 +1,16 @@ # standard imports import logging -# third-party imports +# external imports import celery from hexathon import ( add_0x, ) +from chainsyncer.db.models.base import SessionBase +from chainqueue.db.models.otx import Otx +from chainlib.status import Status # local imports -from cic_eth.db.models.otx import Otx -from chainsyncer.db.models.base import SessionBase -from chainlib.status import Status from .base import SyncFilter logg = logging.getLogger().getChild(__name__) @@ -34,8 +34,9 @@ class TxFilter(SyncFilter): db_session.flush() SessionBase.release_session(db_session) s = celery.signature( - 'cic_eth.queue.tx.set_final_status', + 'cic_eth.queue.state.set_final', [ + self.chain_spec.asdict(), add_0x(tx_hash_hex), tx.block.number, tx.status == Status.ERROR, diff --git a/apps/cic-eth/cic_eth/runnable/daemons/retry.py b/apps/cic-eth/cic_eth/runnable/daemons/retry.py index b7889a32..5d546819 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/retry.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/retry.py @@ -92,7 +92,7 @@ straggler_delay = int(config.get('CIC_TX_RETRY_DELAY')) # TODO: we already have the signed raw tx in get, so its a waste of cycles to get_tx here def sendfail_filter(w3, tx_hash, rcpt, chain_spec): tx_dict = get_tx(tx_hash) - tx = unpack_signed_raw_tx_hex(tx_dict['signed_tx'], chain_spec.chain_id()) + tx = unpack(tx_dict['signed_tx'], chain_spec) logg.debug('submitting tx {} for retry'.format(tx_hash)) s_check = celery.signature( 'cic_eth.admin.ctrl.check_lock', @@ -113,12 +113,12 @@ def sendfail_filter(w3, tx_hash, rcpt, chain_spec): # ) # s_retry_status = celery.signature( -# 'cic_eth.queue.tx.set_ready', +# 'cic_eth.queue.state.set_ready', # [], # queue=queue, # ) s_resend = celery.signature( - 'cic_eth.eth.tx.resend_with_higher_gas', + 'cic_eth.eth.gas.resend_with_higher_gas', [ chain_str, ], @@ -143,7 +143,7 @@ def dispatch(conn, chain_spec): for k in txs.keys(): #tx_cache = get_tx_cache(k) tx_raw = txs[k] - tx = unpack_signed_raw_tx_hex(tx_raw, chain_spec.chain_id()) + tx = unpack(tx_raw, chain_spec) s_check = celery.signature( 'cic_eth.admin.ctrl.check_lock', @@ -184,7 +184,7 @@ def dispatch(conn, chain_spec): # txs = list(txs.keys()) # logg.debug('straggler txs {} chain {}'.format(signed_txs, chain_str)) # s_send = celery.signature( -# 'cic_eth.eth.resend_with_higher_gas', +# 'cic_eth.eth.gas.resend_with_higher_gas', # [ # txs, # chain_str, @@ -204,7 +204,7 @@ class StragglerFilter: def filter(self, conn, block, tx, db_session=None): logg.debug('tx {}'.format(tx)) s_send = celery.signature( - 'cic_eth.eth.tx.resend_with_higher_gas', + 'cic_eth.eth.gas.resend_with_higher_gas', [ tx, self.chain_spec.asdict(), diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index fd316be0..ef7c122b 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -14,21 +14,35 @@ import confini from chainlib.connection import RPCConnection from chainlib.eth.connection import EthUnixSignerConnection from chainlib.chain import ChainSpec +from chainqueue.db.models.otx import Otx # local imports -from cic_eth.eth import erc20 -from cic_eth.eth import tx -from cic_eth.eth import account -from cic_eth.admin import debug -from cic_eth.admin import ctrl -from cic_eth.queue import tx -from cic_eth.queue import balance -from cic_eth.callbacks import Callback -from cic_eth.callbacks import http -from cic_eth.callbacks import tcp -from cic_eth.callbacks import redis +from cic_eth.eth import ( + erc20, + tx, + account, + nonce, + gas, + ) +from cic_eth.admin import ( + debug, + ctrl, + ) +from cic_eth.queue import ( + query, + balance, + state, + tx, + lock, + time, + ) +from cic_eth.callbacks import ( + Callback, + http, + #tcp, + redis, + ) from cic_eth.db.models.base import SessionBase -from cic_eth.db.models.otx import Otx from cic_eth.db import dsn_from_config from cic_eth.ext import tx from cic_eth.registry import ( @@ -162,5 +176,10 @@ def main(): current_app.worker_main(argv) +@celery.signals.eventlet_pool_postshutdown.connect +def shutdown(sender=None, headers=None, body=None, **kwargs): + logg.warning('in shudown event hook') + + if __name__ == '__main__': main() diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 0245ce93..b81b08b0 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -8,9 +8,9 @@ import semver version = ( 0, - 10, - 1, - 'beta.2', + 11, + 0, + 'alpha.3', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 86bde4c5..b5c4b35c 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip # python merge_requirements.py | tee merged_requirements.txt #RUN cd cic-base && \ # pip install $pip_extra_index_url_flag -r ./merged_requirements.txt -RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a44 +RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a57 COPY cic-eth/scripts/ scripts/ COPY cic-eth/setup.cfg cic-eth/setup.py ./ diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index e382cd39..727ee618 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,24 +1,25 @@ -cic-base~=0.1.2a53 +cic-base~=0.1.2a57 celery==4.4.7 -crypto-dev-signer~=0.4.14a16 +crypto-dev-signer~=0.4.14a17 confini~=0.3.6rc3 -cic-eth-registry~=0.5.4a9 +cic-eth-registry~=0.5.4a11 #cic-bancor~=0.0.6 redis==3.5.3 alembic==1.4.2 websockets==8.1 requests~=2.24.0 -eth_accounts_index~=0.0.11a6 -erc20-transfer-authorization~=0.3.1a2 +eth_accounts_index~=0.0.11a7 +erc20-transfer-authorization~=0.3.1a3 #simple-rlp==0.1.2 uWSGI==2.0.19.1 semver==2.13.0 websocket-client==0.57.0 moolb~=0.1.1b2 -eth-address-index~=0.1.1a5 -chainlib~=0.0.1a43 +eth-address-index~=0.1.1a7 +chainlib~=0.0.2a1 hexathon~=0.0.1a7 -chainsyncer~=0.0.1a20 +chainsyncer~=0.0.1a21 +chainqueue~=0.0.1a3 pysha3==1.0.2 coincurve==15.0.0 -sarafu-faucet~=0.0.2a15 +sarafu-faucet~=0.0.2a16 diff --git a/apps/cic-eth/tests/fixtures_celery.py b/apps/cic-eth/tests/fixtures_celery.py index 47de8801..33cadc31 100644 --- a/apps/cic-eth/tests/fixtures_celery.py +++ b/apps/cic-eth/tests/fixtures_celery.py @@ -27,6 +27,9 @@ def celery_includes(): 'cic_eth.eth.tx', 'cic_eth.ext.tx', 'cic_eth.queue.tx', + 'cic_eth.queue.lock', + 'cic_eth.queue.query', + 'cic_eth.queue.state', 'cic_eth.queue.balance', 'cic_eth.admin.ctrl', 'cic_eth.admin.nonce', diff --git a/apps/cic-eth/tests/task/api/test_admin.py b/apps/cic-eth/tests/task/api/test_admin.py index 411eabcb..ca50f107 100644 --- a/apps/cic-eth/tests/task/api/test_admin.py +++ b/apps/cic-eth/tests/task/api/test_admin.py @@ -16,27 +16,22 @@ from hexathon import ( strip_0x, add_0x, ) +from chainqueue.db.models.otx import Otx +from chainqueue.db.models.tx import TxCache +from chainqueue.db.enum import ( + StatusEnum, + StatusBits, + status_str, + ) +from chainqueue.query import get_tx # local imports from cic_eth.api import AdminApi from cic_eth.db.models.role import AccountRole -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.tx import TxCache -from cic_eth.db.enum import ( - StatusEnum, - StatusBits, - status_str, - LockEnum, - ) +from cic_eth.db.enum import LockEnum from cic_eth.error import InitializationError -from cic_eth.eth.tx import ( - cache_gas_data, - ) -#from cic_eth.eth.gas import cache_gas_tx -from cic_eth.queue.tx import ( - create as queue_create, - get_tx, - ) +from cic_eth.eth.gas import cache_gas_data +from cic_eth.queue.tx import queue_create logg = logging.getLogger() @@ -56,7 +51,7 @@ logg = logging.getLogger() # gas_provider = c.gas_provider() # # s_nonce = celery.signature( -# 'cic_eth.eth.tx.reserve_nonce', +# 'cic_eth.eth.nonce.reserve_nonce', # [ # init_w3.eth.accounts[0], # gas_provider, @@ -64,7 +59,7 @@ logg = logging.getLogger() # queue=None, # ) # s_refill = celery.signature( -# 'cic_eth.eth.tx.refill_gas', +# 'cic_eth.eth.gas.refill_gas', # [ # chain_str, # ], @@ -83,7 +78,7 @@ logg = logging.getLogger() # o = q.first() # tx_raw = o.signed_tx # -# tx_dict = unpack_signed_raw_tx(bytes.fromhex(tx_raw[2:]), default_chain_spec.chain_id()) +# tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec) # gas_price_before = tx_dict['gasPrice'] # # s = celery.signature( @@ -109,7 +104,7 @@ logg = logging.getLogger() # # tx_raw_new = get_tx(tx_hash_new_hex) # logg.debug('get {}'.format(tx_raw_new)) -# tx_dict_new = unpack_signed_raw_tx(bytes.fromhex(tx_raw_new['signed_tx'][2:]), default_chain_spec.chain_id()) +# tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec) # assert tx_hash_new_hex != tx_dict['hash'] # assert tx_dict_new['gasPrice'] > gas_price_before # @@ -133,7 +128,7 @@ logg = logging.getLogger() # sigs = [] # for i in range(5): # s = celery.signature( -# 'cic_eth.eth.tx.refill_gas', +# 'cic_eth.eth.gas.refill_gas', # [ # eth_empty_accounts[i], # chain_str, @@ -278,15 +273,14 @@ def test_tx( eth_signer, agent_roles, contract_roles, - celery_session_worker, + celery_worker, ): - chain_id = default_chain_spec.chain_id() nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED) - tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id) - queue_create(tx['nonce'], agent_roles['ALICE'], tx_hash_hex, tx_signed_raw_hex, default_chain_spec, session=init_database) + tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) + queue_create(default_chain_spec, tx['nonce'], agent_roles['ALICE'], tx_hash_hex, tx_signed_raw_hex) cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT']) diff --git a/apps/cic-eth/tests/task/api/test_list.py b/apps/cic-eth/tests/task/api/test_list.py index 27f0bdf2..8f513ff4 100644 --- a/apps/cic-eth/tests/task/api/test_list.py +++ b/apps/cic-eth/tests/task/api/test_list.py @@ -1,23 +1,29 @@ # standard imports import logging -# local imports +# external imports +import pytest from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.erc20 import ERC20 from chainlib.eth.tx import receipt + +# local imports from cic_eth.api.api_task import Api -from tests.mock.filter import ( - block_filter, - tx_filter, - ) from cic_eth.db.models.nonce import ( Nonce, NonceReservation, ) +# test imports +from tests.mock.filter import ( + block_filter, + tx_filter, + ) + logg = logging.getLogger() +@pytest.mark.xfail() def test_list_tx( default_chain_spec, init_database, @@ -29,11 +35,9 @@ def test_list_tx( foo_token, register_tokens, init_eth_tester, - celery_session_worker, + celery_worker, ): - chain_id = default_chain_spec.chain_id() - tx_hashes = [] # external tx @@ -53,7 +57,7 @@ def test_list_tx( init_database.commit() init_eth_tester.mine_blocks(13) - c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) (tx_hash_hex, o) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024) eth_rpc.do(o) o = receipt(tx_hash_hex) @@ -73,7 +77,7 @@ def test_list_tx( init_eth_tester.mine_blocks(13) nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) - c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) (tx_hash_hex, o) = c.transfer(foo_token, agent_roles['ALICE'], agent_roles['BOB'], 256) eth_rpc.do(o) o = receipt(tx_hash_hex) @@ -110,7 +114,6 @@ def test_list_tx( r = t.get_leaf() assert t.successful() - assert len(r) == 3 logg.debug('rrrr {}'.format(r)) diff --git a/apps/cic-eth/tests/task/test_task_account.py b/apps/cic-eth/tests/task/test_task_account.py index d4e0ee6f..412e1f07 100644 --- a/apps/cic-eth/tests/task/test_task_account.py +++ b/apps/cic-eth/tests/task/test_task_account.py @@ -11,13 +11,12 @@ from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.tx import receipt from eth_accounts_index import AccountRegistry from hexathon import strip_0x +from chainqueue.db.enum import StatusEnum +from chainqueue.db.models.otx import Otx # local imports from cic_eth.error import OutOfGasError -from cic_eth.db.models.otx import Otx from cic_eth.db.models.base import SessionBase -from cic_eth.db.enum import StatusEnum -from cic_eth.db.enum import StatusEnum from cic_eth.db.models.nonce import Nonce from cic_eth.db.models.role import AccountRole @@ -74,9 +73,10 @@ def test_register_account( ): s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ eth_empty_accounts[0], + default_chain_spec.asdict(), custodial_roles['ACCOUNT_REGISTRY_WRITER'], ], queue=None, @@ -160,7 +160,7 @@ def test_gift( ): nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], eth_rpc) - c = AccountRegistry(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id()) + c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE']) eth_rpc.do(o) o = receipt(tx_hash_hex) @@ -168,9 +168,10 @@ def test_gift( assert r['status'] == 1 s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ agent_roles['ALICE'], + default_chain_spec.asdict(), ], queue=None, ) diff --git a/apps/cic-eth/tests/task/test_task_erc20.py b/apps/cic-eth/tests/task/test_task_erc20.py index 048ed71b..c202bded 100644 --- a/apps/cic-eth/tests/task/test_task_erc20.py +++ b/apps/cic-eth/tests/task/test_task_erc20.py @@ -28,7 +28,7 @@ def test_otx_cache_transfer( celery_session_worker, ): nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc) - c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id()) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) transfer_value = 100 * (10**6) (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value, tx_format=TxFormat.RLP_SIGNED) register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) @@ -59,7 +59,7 @@ def test_erc20_balance_task( ): nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc) - c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id()) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) transfer_value = 100 * (10**6) (tx_hash_hex, o) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value) eth_rpc.do(o) @@ -102,9 +102,10 @@ def test_erc20_transfer_task( transfer_value = 100 * (10 ** 6) s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ [token_object], + default_chain_spec.asdict(), custodial_roles['FOO_TOKEN_GIFTER'], ], queue=None, @@ -143,9 +144,10 @@ def test_erc20_approve_task( transfer_value = 100 * (10 ** 6) s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce', + 'cic_eth.eth.nonce.reserve_nonce', [ [token_object], + default_chain_spec.asdict(), custodial_roles['FOO_TOKEN_GIFTER'], ], queue=None, diff --git a/apps/cic-eth/tests/task/test_task_tx.py b/apps/cic-eth/tests/task/test_task_tx.py index 99d1a3d0..939b800b 100644 --- a/apps/cic-eth/tests/task/test_task_tx.py +++ b/apps/cic-eth/tests/task/test_task_tx.py @@ -13,16 +13,15 @@ from chainlib.eth.tx import ( receipt, ) from hexathon import strip_0x +from chainqueue.db.models.otx import Otx # local imports from cic_eth.queue.tx import register_tx -from cic_eth.eth.tx import cache_gas_data -from cic_eth.db.models.otx import Otx +from cic_eth.eth.gas import cache_gas_data logg = logging.getLogger() -@pytest.mark.skip() def test_tx_send( init_database, default_chain_spec, @@ -33,11 +32,9 @@ def test_tx_send( celery_session_worker, ): - chain_id = default_chain_spec.chain_id() nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED) - #unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id) register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) @@ -62,12 +59,11 @@ def test_tx_send( assert rcpt['status'] == 1 -@pytest.mark.skip() def test_sync_tx( default_chain_spec, eth_rpc, eth_signer, - celery_worker, + celery_session_worker, ): pass @@ -78,20 +74,18 @@ def test_resend_with_higher_gas( eth_rpc, eth_signer, agent_roles, - celery_worker, + celery_session_worker, ): - chain_id = default_chain_spec.chain_id() nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED) - #unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id) register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec.chain_id()) + tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) s = celery.signature( - 'cic_eth.eth.tx.resend_with_higher_gas', + 'cic_eth.eth.gas.resend_with_higher_gas', [ tx_hash_hex, default_chain_spec.asdict(), @@ -102,12 +96,12 @@ def test_resend_with_higher_gas( r = t.get_leaf() q = init_database.query(Otx) - q = q.filter(Otx.tx_hash==r) + q = q.filter(Otx.tx_hash==strip_0x(r)) otx = q.first() if otx == None: raise NotLocalTxError(r) - tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec.chain_id()) + tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec) logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice'])) assert tx_after['gasPrice'] > tx_before['gasPrice'] diff --git a/apps/cic-eth/tests/unit/db/test_block_sync.py b/apps/cic-eth/tests/unit/db/test_block_sync.py deleted file mode 100644 index 87fdee4f..00000000 --- a/apps/cic-eth/tests/unit/db/test_block_sync.py +++ /dev/null @@ -1,29 +0,0 @@ -# standard imports -import logging - -# local imports -from cic_eth.db.models.otx import OtxSync - -logg = logging.getLogger() - - -def test_db_block_sync( - init_database, - ): - - s = OtxSync('eip155:8995:bloxberg') - - s.head(666, 12) - assert s.head() == (666, 12) - - s.session(42, 13) - assert s.session() == (42, 13) - - s.backlog(13, 2) - assert s.backlog() == (13, 2) - - assert not s.synced - - s.backlog(42, 13) - assert s.backlog() == (42, 13) - assert s.synced diff --git a/apps/cic-eth/tests/unit/db/test_db_convert_transfer.py b/apps/cic-eth/tests/unit/db/test_db_convert_transfer.py index 15e766db..bc3b6e5a 100644 --- a/apps/cic-eth/tests/unit/db/test_db_convert_transfer.py +++ b/apps/cic-eth/tests/unit/db/test_db_convert_transfer.py @@ -3,7 +3,7 @@ import logging import pytest -from cic_eth.db import TxConvertTransfer +from cic_eth.db.models.convert import TxConvertTransfer from cic_eth.db.error import UnknownConvertError logg = logging.getLogger() diff --git a/apps/cic-eth/tests/unit/db/test_otx.py b/apps/cic-eth/tests/unit/db/test_otx.py deleted file mode 100644 index 00057a5e..00000000 --- a/apps/cic-eth/tests/unit/db/test_otx.py +++ /dev/null @@ -1,107 +0,0 @@ -# standard imports -import os -import logging - -# third-party imports -import pytest - -# local imports -from cic_eth.db.models.base import SessionBase -from cic_eth.db.models.otx import OtxStateLog -from cic_eth.db.models.otx import Otx -from cic_eth.db.enum import ( - StatusEnum, - StatusBits, - is_alive, - ) - -logg = logging.getLogger() - - -#def test_get( -# rpc_eth, -# rpc_signer, -# agent_roles, -# init_database, -# ): -# -# rpc = RPCConnection.connect(default_chain_spec, 'default') -# nonce_oracle = RPCNonceOracle(agent_roles['ALICE']) -# gas_oracle = RPCGasOracle(eth_rpc) -# c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) -# -# for i in range(10): -# -# (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)), -# -# tx_def = { -# 'from': init_w3.eth.accounts[0], -# 'to': init_w3.eth.accounts[1], -# 'nonce': 0, -# 'value': 101, -# 'gasPrice': 2000000000, -# 'gas': 21000, -# 'data': '', -# 'chainId': 1, -# } -# -# session = init_database -# txs = [] -# for i in range(10): -# nonce = init_w3.eth.getTransactionCount(init_w3.eth.accounts[0], 'pending') -# tx_def['nonce'] = nonce -# tx = init_w3.eth.sign_transaction(tx_def) -# tx_hash = init_w3.eth.send_raw_transaction(tx['raw']) -# logg.debug('tx {}'.format(tx)) -# -# address = init_w3.eth.accounts[i%3] -# otx = Otx(int((i/3)+1), address, '0x'+tx_hash.hex(), tx['raw']) -# txs.append(otx) -# session.add(otx) -# session.flush() -# -# logg.debug(txs) -# session.commit() -# -# txs[0].status = 0 -# session.add(txs[0]) -# session.commit() -# session.close() -# -# get_txs = Otx.get() -# logg.debug(get_txs) - - -def test_state_log( - init_database, - ): - - Otx.tracing = True - - address = '0x' + os.urandom(20).hex() - tx_hash = '0x' + os.urandom(32).hex() - signed_tx = '0x' + os.urandom(128).hex() - otx = Otx.add(0, address, tx_hash, signed_tx, session=init_database) - - otx.waitforgas(session=init_database) - init_database.commit() - - otx.readysend(session=init_database) - init_database.commit() - - otx.sent(session=init_database) - init_database.commit() - - otx.success(1024, session=init_database) - init_database.commit() - - q = init_database.query(OtxStateLog) - q = q.filter(OtxStateLog.otx_id==otx.id) - q = q.order_by(OtxStateLog.date.asc()) - logs = q.all() - - assert logs[0].status == StatusEnum.PENDING - assert logs[1].status == StatusEnum.WAITFORGAS - assert logs[2].status & StatusBits.QUEUED - assert logs[3].status & StatusBits.IN_NETWORK - assert not is_alive(logs[4].status) diff --git a/apps/cic-eth/tests/unit/db/test_status.py b/apps/cic-eth/tests/unit/db/test_status.py deleted file mode 100644 index f40cd6d8..00000000 --- a/apps/cic-eth/tests/unit/db/test_status.py +++ /dev/null @@ -1,97 +0,0 @@ -# standard imports -import os - -# third-party imports -import pytest - -# local imports -from cic_eth.db.models.otx import Otx -from cic_eth.db.enum import ( - StatusEnum, - StatusBits, - is_alive, - ) - - -@pytest.fixture(scope='function') -def otx( - init_database, - ): - - bogus_hash = '0x' + os.urandom(32).hex() - bogus_address = '0x' + os.urandom(20).hex() - bogus_tx_raw = '0x' + os.urandom(128).hex() - return Otx(0, bogus_address, bogus_hash, bogus_tx_raw) - - -def test_status_chain_gas( - init_database, - otx, - ): - - otx.waitforgas(init_database) - otx.readysend(init_database) - otx.sent(init_database) - otx.success(1024, init_database) - assert not is_alive(otx.status) - - -def test_status_chain_straight_success( - init_database, - otx, - ): - - otx.readysend(init_database) - otx.sent(init_database) - otx.success(1024, init_database) - assert not is_alive(otx.status) - - -def test_status_chain_straight_revert( - init_database, - otx, - ): - - otx.readysend(init_database) - otx.sent(init_database) - otx.minefail(1024, init_database) - assert not is_alive(otx.status) - - -def test_status_chain_nodeerror( - init_database, - otx, - ): - - otx.readysend(init_database) - otx.sendfail(init_database) - otx.retry(init_database) - otx.sent(init_database) - otx.success(1024, init_database) - assert not is_alive(otx.status) - - - -def test_status_chain_nodeerror_multiple( - init_database, - otx, - ): - - otx.readysend(init_database) - otx.sendfail(init_database) - otx.retry(init_database) - otx.sendfail(init_database) - otx.retry(init_database) - otx.sent(init_database) - otx.success(1024, init_database) - assert not is_alive(otx.status) - - -def test_status_chain_nodeerror( - init_database, - otx, - ): - - otx.readysend(init_database) - otx.reject(init_database) - assert not is_alive(otx.status) diff --git a/apps/cic-eth/tests/unit/db/test_tx.py b/apps/cic-eth/tests/unit/db/test_tx.py index c616bf77..2cc54e73 100644 --- a/apps/cic-eth/tests/unit/db/test_tx.py +++ b/apps/cic-eth/tests/unit/db/test_tx.py @@ -18,10 +18,8 @@ from hexathon import ( add_0x, strip_0x, ) - -# local imports -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.otx import Otx +from chainqueue.db.models.tx import TxCache +from chainqueue.db.models.otx import Otx # test imports from tests.util.gas import StaticGasOracle @@ -35,18 +33,16 @@ def test_set( agent_roles, ): - chain_id = default_chain_spec.chain_id() rpc = RPCConnection.connect(default_chain_spec, 'default') nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_id) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) - tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id) + tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) otx = Otx( tx['nonce'], - tx['from'], tx_hash_hex, tx_signed_raw_hex, ) @@ -66,6 +62,7 @@ def test_set( to_value, 666, 13, + session=init_database, ) init_database.add(txc) init_database.commit() @@ -89,18 +86,17 @@ def test_clone( agent_roles, ): - chain_id = default_chain_spec.chain_id() rpc = RPCConnection.connect(default_chain_spec, 'default') nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) gas_oracle = StaticGasOracle(2 * (10 ** 9), 21000) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_id) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) txs_rpc = [ c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED), ] gas_oracle = StaticGasOracle(4 * (10 ** 9), 21000) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_id) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) txs_rpc += [ c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED), ] @@ -109,10 +105,9 @@ def test_clone( for tx_rpc in txs_rpc: tx_hash_hex = tx_rpc[0] tx_signed_raw_hex = tx_rpc[1] - tx_dict = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id) + tx_dict = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) otx = Otx( tx_dict['nonce'], - tx_dict['from'], tx_hash_hex, tx_signed_raw_hex, ) @@ -130,15 +125,16 @@ def test_clone( ZERO_ADDRESS, txs[0]['value'], txs[0]['value'], + session=init_database, ) init_database.add(txc) init_database.commit() - TxCache.clone(txs[0]['hash'], txs[1]['hash']) + TxCache.clone(txs[0]['hash'], txs[1]['hash'], session=init_database) q = init_database.query(TxCache) q = q.join(Otx) - q = q.filter(Otx.tx_hash==txs[1]['hash']) + q = q.filter(Otx.tx_hash==strip_0x(txs[1]['hash'])) txc_clone = q.first() assert txc_clone != None diff --git a/apps/cic-eth/tests/unit/eth/test_raw.py b/apps/cic-eth/tests/unit/eth/test_raw.py index ab7f65d2..4230b7f3 100644 --- a/apps/cic-eth/tests/unit/eth/test_raw.py +++ b/apps/cic-eth/tests/unit/eth/test_raw.py @@ -18,13 +18,12 @@ def test_unpack( agent_roles, ): - chain_id = default_chain_spec.chain_id() rpc = RPCConnection.connect(default_chain_spec, 'default') nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) - tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id=chain_id) + tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) assert tx_hash_hex == tx['hash'] diff --git a/apps/cic-eth/tests/unit/ext/test_address.py b/apps/cic-eth/tests/unit/ext/test_address.py index ecfde959..332af060 100644 --- a/apps/cic-eth/tests/unit/ext/test_address.py +++ b/apps/cic-eth/tests/unit/ext/test_address.py @@ -23,7 +23,7 @@ def test_translate( nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc) - c = AddressDeclarator(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id()) + c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) description = 'alice'.encode('utf-8').ljust(32, b'\x00').hex() (tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], add_0x(description)) diff --git a/apps/cic-eth/tests/unit/queue/test_balances.py b/apps/cic-eth/tests/unit/queue/test_balances.py index 168401c9..2a372314 100644 --- a/apps/cic-eth/tests/unit/queue/test_balances.py +++ b/apps/cic-eth/tests/unit/queue/test_balances.py @@ -2,12 +2,12 @@ import os import logging -# third-party imports +# external imports import pytest +from chainqueue.db.models.otx import Otx +from chainqueue.db.models.tx import TxCache # local imports -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.tx import TxCache from cic_eth.queue.balance import ( balance_outgoing, balance_incoming, @@ -59,17 +59,15 @@ def test_assemble(): assert r[1].get('balance_xyzzy') != None -@pytest.mark.skip() def test_outgoing_balance( default_chain_spec, init_database, ): - chain_str = str(default_chain_spec) recipient = '0x' + os.urandom(20).hex() tx_hash = '0x' + os.urandom(32).hex() signed_tx = '0x' + os.urandom(128).hex() - otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database) + otx = Otx.add(0, tx_hash, signed_tx, session=init_database) init_database.add(otx) init_database.commit() @@ -83,6 +81,7 @@ def test_outgoing_balance( token_address, 1000, 1000, + session=init_database, ) init_database.add(txc) init_database.commit() @@ -91,33 +90,35 @@ def test_outgoing_balance( 'address': token_address, 'converters': [], } - b = balance_outgoing([token_data], sender, chain_str) + b = balance_outgoing([token_data], sender, default_chain_spec.asdict()) assert b[0]['balance_outgoing'] == 1000 + otx.readysend(session=init_database) + init_database.flush() + otx.reserve(session=init_database) + init_database.flush() otx.sent(session=init_database) init_database.commit() - b = balance_outgoing([token_data], sender, chain_str) + b = balance_outgoing([token_data], sender, default_chain_spec.asdict()) assert b[0]['balance_outgoing'] == 1000 otx.success(block=1024, session=init_database) init_database.commit() - b = balance_outgoing([token_data], sender, chain_str) + b = balance_outgoing([token_data], sender, default_chain_spec.asdict()) assert b[0]['balance_outgoing'] == 0 -@pytest.mark.skip() def test_incoming_balance( default_chain_spec, init_database, ): - chain_str = str(default_chain_spec) recipient = '0x' + os.urandom(20).hex() tx_hash = '0x' + os.urandom(32).hex() signed_tx = '0x' + os.urandom(128).hex() - otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database) + otx = Otx.add(0, tx_hash, signed_tx, session=init_database) init_database.add(otx) init_database.commit() @@ -131,6 +132,7 @@ def test_incoming_balance( token_address, 1000, 1000, + session=init_database, ) init_database.add(txc) init_database.commit() @@ -139,19 +141,23 @@ def test_incoming_balance( 'address': token_address, 'converters': [], } - b = balance_incoming([token_data], recipient, chain_str) + b = balance_incoming([token_data], recipient, default_chain_spec.asdict()) assert b[0]['balance_incoming'] == 0 + otx.readysend(session=init_database) + init_database.flush() + otx.reserve(session=init_database) + init_database.flush() otx.sent(session=init_database) init_database.commit() - b = balance_incoming([token_data], recipient, chain_str) + b = balance_incoming([token_data], recipient, default_chain_spec.asdict()) assert b[0]['balance_incoming'] == 1000 otx.success(block=1024, session=init_database) init_database.commit() - b = balance_incoming([token_data], recipient, chain_str) + b = balance_incoming([token_data], recipient, default_chain_spec.asdict()) assert b[0]['balance_incoming'] == 0 diff --git a/apps/cic-eth/tests/unit/queue/test_list_tx.py b/apps/cic-eth/tests/unit/queue/test_list_tx.py deleted file mode 100644 index 397ed07e..00000000 --- a/apps/cic-eth/tests/unit/queue/test_list_tx.py +++ /dev/null @@ -1,76 +0,0 @@ -# standard imports -import logging - -# external imports -from chainlib.connection import RPCConnection -from chainlib.eth.gas import RPCGasOracle -from chainlib.eth.nonce import RPCNonceOracle -from chainlib.eth.gas import Gas - -# local imports -from cic_eth.queue.tx import get_status_tx -from cic_eth.db.enum import ( - StatusEnum, - StatusBits, - ) -from cic_eth.queue.tx import create as queue_create -from cic_eth.eth.tx import cache_gas_data -from cic_eth.queue.tx import register_tx -from cic_eth.db.models.otx import Otx - -logg = logging.getLogger() - - -def test_status_tx_list( - default_chain_spec, - init_database, - eth_rpc, - eth_signer, - agent_roles, - ): - - rpc = RPCConnection.connect(default_chain_spec, 'default') - - nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - (tx_hash_hex, o) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024) - r = rpc.do(o) - - tx_signed_raw_hex = o['params'][0] - #queue_create(tx['nonce'], tx['from'], tx_hash.hex(), tx_signed['raw'], str(default_chain_spec)) - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - q = init_database.query(Otx) - otx = q.get(1) - otx.sendfail(session=init_database) - init_database.add(otx) - init_database.commit() - init_database.refresh(otx) - - txs = get_status_tx(StatusBits.LOCAL_ERROR, session=init_database) - assert len(txs) == 1 - - otx.sendfail(session=init_database) - otx.retry(session=init_database) - init_database.add(otx) - init_database.commit() - init_database.refresh(otx) - - txs = get_status_tx(StatusBits.LOCAL_ERROR, session=init_database) - assert len(txs) == 1 - - txs = get_status_tx(StatusBits.QUEUED, session=init_database) - assert len(txs) == 1 - - txs = get_status_tx(StatusBits.QUEUED, not_status=StatusBits.LOCAL_ERROR, session=init_database) - assert len(txs) == 0 - - txs = get_status_tx(StatusBits.QUEUED, not_status=StatusBits.IN_NETWORK, session=init_database) - assert len(txs) == 1 - - txs = get_status_tx(StatusBits.IN_NETWORK, session=init_database) - assert len(txs) == 0 - diff --git a/apps/cic-eth/tests/unit/queue/test_otx_state_log.py b/apps/cic-eth/tests/unit/queue/test_otx_state_log.py deleted file mode 100644 index 9c5194da..00000000 --- a/apps/cic-eth/tests/unit/queue/test_otx_state_log.py +++ /dev/null @@ -1,22 +0,0 @@ -# standard imports -import os - -# local imports -from cic_eth.db.models.otx import Otx -from cic_eth.queue.tx import get_state_log - - -def test_otx_state_log( - init_database, - ): - - Otx.tracing = True - - address = '0x' + os.urandom(20).hex() - tx_hash = '0x' + os.urandom(32).hex() - signed_tx = '0x' + os.urandom(128).hex() - otx = Otx.add(0, address, tx_hash, signed_tx, session=init_database) - init_database.commit() - - log = get_state_log(tx_hash) - assert len(log) == 1 diff --git a/apps/cic-eth/tests/unit/queue/test_query.py b/apps/cic-eth/tests/unit/queue/test_query.py new file mode 100644 index 00000000..b82fbd4e --- /dev/null +++ b/apps/cic-eth/tests/unit/queue/test_query.py @@ -0,0 +1,58 @@ +# external imports +from chainqueue.db.enum import ( + StatusEnum, + StatusBits, + ) +from chainlib.connection import RPCConnection +from chainlib.eth.gas import ( + RPCGasOracle, + Gas, + ) +from chainlib.chain import ChainSpec + +# local imports +from cic_eth.db.enum import LockEnum +from cic_eth.db.models.lock import Lock +from cic_eth.queue.query import get_upcoming_tx +from cic_eth.queue.tx import register_tx +from cic_eth.eth.gas import cache_gas_data + +# test imports +from tests.util.nonce import StaticNonceOracle + + +def test_upcoming_with_lock( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = StaticNonceOracle(42) + gas_oracle = RPCGasOracle(eth_rpc) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + + (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)) + tx_signed_raw_hex = tx_rpc['params'][0] + + register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) + cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) + + txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING) + assert len(txs.keys()) == 1 + + Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE']) + + txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING) + assert len(txs.keys()) == 0 + + (tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6)) + tx_signed_raw_hex = tx_rpc['params'][0] + + register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) + cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) + + txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING) + assert len(txs.keys()) == 1 diff --git a/apps/cic-eth/tests/unit/queue/test_queue_lock.py b/apps/cic-eth/tests/unit/queue/test_queue_lock.py index fe18fa16..7b0a6e5b 100644 --- a/apps/cic-eth/tests/unit/queue/test_queue_lock.py +++ b/apps/cic-eth/tests/unit/queue/test_queue_lock.py @@ -5,10 +5,10 @@ import os import pytest # local imports -from cic_eth.queue.tx import create as queue_create from cic_eth.db.models.lock import Lock from cic_eth.db.enum import LockEnum from cic_eth.error import LockedError +from cic_eth.queue.tx import queue_create def test_queue_lock( @@ -25,41 +25,41 @@ def test_queue_lock( Lock.set(chain_str, LockEnum.QUEUE) with pytest.raises(LockedError): queue_create( + default_chain_spec, 0, address, tx_hash, tx_raw, - chain_str ) Lock.set(chain_str, LockEnum.QUEUE, address=address) with pytest.raises(LockedError): queue_create( + default_chain_spec, 0, address, tx_hash, tx_raw, - chain_str ) Lock.reset(chain_str, LockEnum.QUEUE) with pytest.raises(LockedError): queue_create( + default_chain_spec, 0, address, tx_hash, tx_raw, - chain_str ) Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash) with pytest.raises(LockedError): queue_create( + default_chain_spec, 0, address, tx_hash, tx_raw, - chain_str ) diff --git a/apps/cic-eth/tests/unit/queue/test_tx_queue.py b/apps/cic-eth/tests/unit/queue/test_tx_queue.py deleted file mode 100644 index 385b3fb5..00000000 --- a/apps/cic-eth/tests/unit/queue/test_tx_queue.py +++ /dev/null @@ -1,577 +0,0 @@ -# standard imports import logging -import datetime -import os -import logging - -# external imports -import pytest -from sqlalchemy import DateTime -from chainlib.connection import RPCConnection -from chainlib.eth.nonce import OverrideNonceOracle -from chainlib.eth.tx import unpack -from chainlib.eth.gas import ( - RPCGasOracle, - Gas, - ) -from chainlib.eth.constant import ZERO_ADDRESS -from hexathon import strip_0x - -# local imports -from cic_eth.eth.tx import cache_gas_data -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.otx import OtxSync -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.lock import Lock -from cic_eth.db.models.base import SessionBase -from cic_eth.db.enum import ( - StatusEnum, - LockEnum, - StatusBits, - is_alive, - is_error_status, - status_str, - ) -from cic_eth.queue.tx import create as queue_create -from cic_eth.queue.tx import set_final_status -from cic_eth.queue.tx import set_sent_status -from cic_eth.queue.tx import set_waitforgas -from cic_eth.queue.tx import set_ready -from cic_eth.queue.tx import get_paused_txs -from cic_eth.queue.tx import get_upcoming_tx -from cic_eth.queue.tx import get_account_tx -from cic_eth.queue.tx import get_tx -from cic_eth.db.error import TxStateChangeError -from cic_eth.queue.tx import register_tx - -# test imports -from tests.util.nonce import StaticNonceOracle - -logg = logging.getLogger() - - -def test_finalize( - default_chain_spec, - eth_rpc, - eth_signer, - init_database, - agent_roles, - ): - - rpc = RPCConnection.connect(default_chain_spec, 'default') - nonce_oracle = StaticNonceOracle(0) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - txs_rpc = [ - c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['BOB'], 200 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['BOB'], 300 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['BOB'], 400 * (10 ** 6)), - ] - - nonce_oracle = StaticNonceOracle(1) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - txs_rpc.append(c.create(agent_roles['ALICE'], agent_roles['BOB'], 500 * (10 ** 6))) - - tx_hashes = [] - i = 0 - for entry in txs_rpc: - tx_hash_hex = entry[0] - tx_rpc = entry[1] - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - tx_hashes.append(tx_hash_hex) - - if i < 3: - set_sent_status(tx_hash_hex) - - i += 1 - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[0]).first() - assert otx.status & StatusBits.OBSOLETE - assert not is_alive(otx.status) - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[1]).first() - assert otx.status & StatusBits.OBSOLETE - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[2]).first() - assert otx.status & StatusBits.OBSOLETE - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[3]).first() - assert otx.status == StatusEnum.PENDING - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[4]).first() - assert otx.status == StatusEnum.PENDING - - set_sent_status(tx_hashes[3], False) - set_sent_status(tx_hashes[4], False) - set_final_status(tx_hashes[3], 1024) - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[0]).first() - assert otx.status & (StatusBits.OBSOLETE | StatusBits.FINAL) - assert not is_alive(otx.status) - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[1]).first() - assert otx.status & (StatusBits.OBSOLETE | StatusBits.FINAL) - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[2]).first() - assert otx.status & (StatusBits.OBSOLETE | StatusBits.FINAL) - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[3]).first() - assert otx.status & (StatusBits.IN_NETWORK | StatusBits.FINAL) - assert not is_error_status(otx.status) - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[4]).first() - assert otx.status & (StatusBits.IN_NETWORK | StatusBits.FINAL) - assert not is_error_status(otx.status) - - -def test_expired( - default_chain_spec, - init_database, - eth_rpc, - eth_signer, - agent_roles, - ): - - rpc = RPCConnection.connect(default_chain_spec, 'default') - nonce_oracle = StaticNonceOracle(42) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - txs_rpc = [ - c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['BOB'], 200 * (10 ** 6)), - ] - - nonce_oracle = StaticNonceOracle(43) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - txs_rpc += [ - c.create(agent_roles['ALICE'], agent_roles['BOB'], 300 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['BOB'], 400 * (10 ** 6)), - ] - - nonce_oracle = StaticNonceOracle(44) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - txs_rpc.append(c.create(agent_roles['ALICE'], agent_roles['BOB'], 500 * (10 ** 6))) - - tx_hashes = [] - - i = 0 - for entry in txs_rpc: - tx_hash_hex = entry[0] - tx_rpc = entry[1] - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - tx_hashes.append(tx_hash_hex) - - set_sent_status(tx_hash_hex, False) - - otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hash_hex).first() - fake_created = datetime.datetime.utcnow() - datetime.timedelta(seconds=40*i) - otx.date_created = fake_created - init_database.add(otx) - init_database.commit() - init_database.refresh(otx) - - i += 1 - - now = datetime.datetime.utcnow() - delta = datetime.timedelta(seconds=61) - then = now - delta - - otxs = OtxSync.get_expired(then) - nonce_acc = 0 - for otx in otxs: - nonce_acc += otx.nonce - - assert nonce_acc == (43 + 44) - - -def test_get_paused( - init_database, - default_chain_spec, - eth_rpc, - eth_signer, - agent_roles, - ): - - chain_id = default_chain_spec.chain_id() - rpc = RPCConnection.connect(default_chain_spec, 'default') - nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - txs_rpc = [ - c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['BOB'], 200 * (10 ** 6)), - ] - - tx_hashes = [] - for entry in txs_rpc: - tx_hash_hex = entry[0] - tx_rpc = entry[1] - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - tx_hashes.append(tx_hash_hex) - - txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id) - assert len(txs.keys()) == 0 - - q = init_database.query(Otx) - q = q.filter(Otx.tx_hash==tx_hashes[0]) - r = q.first() - r.waitforgas(session=init_database) - init_database.add(r) - init_database.commit() - - chain_id = default_chain_spec.chain_id() - txs = get_paused_txs(chain_id=chain_id) - assert len(txs.keys()) == 1 - - txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id) # init_w3.eth.accounts[0]) - assert len(txs.keys()) == 1 - - txs = get_paused_txs(status=StatusBits.GAS_ISSUES) - assert len(txs.keys()) == 1 - - txs = get_paused_txs(sender=agent_roles['ALICE'], status=StatusBits.GAS_ISSUES, chain_id=chain_id) - assert len(txs.keys()) == 1 - - - q = init_database.query(Otx) - q = q.filter(Otx.tx_hash==tx_hashes[1]) - o = q.first() - o.waitforgas(session=init_database) - init_database.add(o) - init_database.commit() - - txs = get_paused_txs() - assert len(txs.keys()) == 2 - - txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id) # init_w3.eth.accounts[0]) - assert len(txs.keys()) == 2 - - txs = get_paused_txs(status=StatusBits.GAS_ISSUES, chain_id=chain_id) - assert len(txs.keys()) == 2 - - txs = get_paused_txs(sender=agent_roles['ALICE'], status=StatusBits.GAS_ISSUES, chain_id=chain_id) # init_w3.eth.accounts[0]) - assert len(txs.keys()) == 2 - - q = init_database.query(Otx) - q = q.filter(Otx.tx_hash==tx_hashes[1]) - o = q.first() - o.sendfail(session=init_database) - init_database.add(o) - init_database.commit() - - txs = get_paused_txs() - assert len(txs.keys()) == 2 - - txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id) # init_w3.eth.accounts[0]) - assert len(txs.keys()) == 2 - - txs = get_paused_txs(status=StatusBits.GAS_ISSUES, chain_id=chain_id) - txs = get_paused_txs(status=StatusEnum.WAITFORGAS, chain_id=chain_id) - assert len(txs.keys()) == 1 - - txs = get_paused_txs(sender=agent_roles['ALICE'], status=StatusBits.GAS_ISSUES, chain_id=chain_id) # init_w3.eth.accounts[0]) - assert len(txs.keys()) == 1 - - -def test_get_upcoming( - default_chain_spec, - eth_rpc, - eth_signer, - init_database, - agent_roles, - ): - - chain_id = default_chain_spec.chain_id() - rpc = RPCConnection.connect(default_chain_spec, 'default') - nonce_oracle = StaticNonceOracle(42) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - txs_rpc = [ - c.create(agent_roles['ALICE'], agent_roles['DAVE'], 100 * (10 ** 6)), - c.create(agent_roles['BOB'], agent_roles['DAVE'], 200 * (10 ** 6)), - c.create(agent_roles['CAROL'], agent_roles['DAVE'], 300 * (10 ** 6)), - ] - - nonce_oracle = StaticNonceOracle(43) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - txs_rpc += [ - c.create(agent_roles['ALICE'], agent_roles['DAVE'], 400 * (10 ** 6)), - c.create(agent_roles['BOB'], agent_roles['DAVE'], 500 * (10 ** 6)), - c.create(agent_roles['CAROL'], agent_roles['DAVE'], 600 * (10 ** 6)), - ] - - nonce_oracle = StaticNonceOracle(44) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - txs_rpc += [ - c.create(agent_roles['ALICE'], agent_roles['DAVE'], 700 * (10 ** 6)), - ] - - tx_hashes = [] - for entry in txs_rpc: - tx_hash_hex = entry[0] - tx_rpc = entry[1] - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - tx_hashes.append(tx_hash_hex) - - set_ready(tx_hash_hex) - - txs = get_upcoming_tx(StatusBits.QUEUED, chain_id=chain_id) - assert len(txs.keys()) == 3 - - tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[0]])), chain_id) - assert tx['nonce'] == 42 - - tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[1]])), chain_id) - assert tx['nonce'] == 42 - - tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[2]])), chain_id) - assert tx['nonce'] == 42 - - q = init_database.query(TxCache) - q = q.filter(TxCache.sender==agent_roles['ALICE']) - for o in q.all(): - o.date_checked -= datetime.timedelta(seconds=30) - init_database.add(o) - init_database.commit() - - before = datetime.datetime.now() - datetime.timedelta(seconds=20) - logg.debug('before {}'.format(before)) - txs = get_upcoming_tx(StatusBits.QUEUED, before=before) - logg.debug('txs {} {}'.format(txs.keys(), txs.values())) - assert len(txs.keys()) == 1 - - # Now date checked has been set to current time, and the check returns no results - txs = get_upcoming_tx(StatusBits.QUEUED, before=before) - logg.debug('txs {} {}'.format(txs.keys(), txs.values())) - assert len(txs.keys()) == 0 - - set_sent_status(tx_hashes[0]) - - txs = get_upcoming_tx(StatusBits.QUEUED) - assert len(txs.keys()) == 3 - with pytest.raises(KeyError): - tx = txs[tx_hashes[0]] - - tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[3]])), chain_id) - assert tx['nonce'] == 43 - - set_waitforgas(tx_hashes[1]) - txs = get_upcoming_tx(StatusBits.QUEUED) - assert len(txs.keys()) == 3 - with pytest.raises(KeyError): - tx = txs[tx_hashes[1]] - - tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[3]])), chain_id) - assert tx['nonce'] == 43 - - txs = get_upcoming_tx(StatusBits.GAS_ISSUES) - assert len(txs.keys()) == 1 - - -def test_upcoming_with_lock( - default_chain_spec, - init_database, - eth_rpc, - eth_signer, - agent_roles, - ): - - chain_id = int(default_chain_spec.chain_id()) - - rpc = RPCConnection.connect(default_chain_spec, 'default') - nonce_oracle = StaticNonceOracle(42) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)) - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - txs = get_upcoming_tx(StatusEnum.PENDING, chain_id=chain_id) - assert len(txs.keys()) == 1 - - Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE']) - - txs = get_upcoming_tx(StatusEnum.PENDING, chain_id=chain_id) - assert len(txs.keys()) == 0 - - (tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6)) - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - txs = get_upcoming_tx(StatusEnum.PENDING, chain_id=chain_id) - assert len(txs.keys()) == 1 - - -def test_obsoletion( - default_chain_spec, - init_database, - eth_rpc, - eth_signer, - agent_roles, - ): - - chain_id = default_chain_spec.chain_id() - rpc = RPCConnection.connect(default_chain_spec, 'default') - nonce_oracle = StaticNonceOracle(42) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - txs_rpc = [ - c.create(agent_roles['ALICE'], agent_roles['DAVE'], 100 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['DAVE'], 200 * (10 ** 6)), - c.create(agent_roles['BOB'], agent_roles['DAVE'], 300 * (10 ** 6)), - ] - - nonce_oracle = StaticNonceOracle(43) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - txs_rpc += [ - c.create(agent_roles['BOB'], agent_roles['DAVE'], 400 * (10 ** 6)), - ] - - tx_hashes = [] - i = 0 - for entry in txs_rpc: - tx_hash_hex = entry[0] - tx_rpc = entry[1] - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - tx_hashes.append(tx_hash_hex) - - if i < 2: - set_sent_status(tx_hash_hex) - - i += 1 - - session = SessionBase.create_session() - q = session.query(Otx) - q = q.filter(Otx.status.op('&')(StatusEnum.OBSOLETED.value)==StatusEnum.OBSOLETED.value) - z = 0 - for o in q.all(): - z += o.nonce - - session.close() - assert z == 42 - - set_final_status(tx_hashes[1], 1023, True) - - session = SessionBase.create_session() - q = session.query(Otx) - q = q.filter(Otx.status.op('&')(StatusEnum.CANCELLED.value)==StatusEnum.OBSOLETED.value) - zo = 0 - for o in q.all(): - zo += o.nonce - - q = session.query(Otx) - q = q.filter(Otx.status.op('&')(StatusEnum.CANCELLED.value)==StatusEnum.CANCELLED.value) - zc = 0 - for o in q.all(): - zc += o.nonce - - session.close() - assert zo == 0 - assert zc == 42 - - -def test_retry( - init_database, - ): - - address = '0x' + os.urandom(20).hex() - tx_hash = '0x' + os.urandom(32).hex() - signed_tx = '0x' + os.urandom(128).hex() - otx = Otx(0, address, tx_hash, signed_tx) - init_database.add(otx) - init_database.commit() - - set_sent_status(tx_hash, True) - set_ready(tx_hash) - - q = init_database.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - otx = q.first() - - assert (otx.status & StatusEnum.RETRY.value) == StatusEnum.RETRY.value - assert is_error_status(otx.status) - - set_sent_status(tx_hash, False) - set_ready(tx_hash) - - init_database.commit() - - q = init_database.query(Otx) - q = q.filter(Otx.tx_hash==tx_hash) - otx = q.first() - - assert (otx.status & StatusEnum.RETRY.value) == StatusBits.QUEUED.value - assert not is_error_status(otx.status) - - -def test_get_account_tx( - default_chain_spec, - init_database, - eth_rpc, - eth_signer, - agent_roles, - ): - - chain_id = default_chain_spec.chain_id() - rpc = RPCConnection.connect(default_chain_spec, 'default') - nonce_oracle = OverrideNonceOracle(ZERO_ADDRESS, 42) - gas_oracle = RPCGasOracle(eth_rpc) - c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id()) - - txs_rpc = [ - c.create(agent_roles['ALICE'], agent_roles['DAVE'], 100 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6)), - c.create(agent_roles['ALICE'], agent_roles['BOB'], 300 * (10 ** 6)), - c.create(agent_roles['BOB'], agent_roles['ALICE'], 300 * (10 ** 6)), - ] - - tx_hashes = [] - for entry in txs_rpc: - tx_hash_hex = entry[0] - tx_rpc = entry[1] - tx_signed_raw_hex = tx_rpc['params'][0] - - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - - tx_hashes.append(tx_hash_hex) - - txs = get_account_tx(agent_roles['ALICE']) - logg.debug('tx {} tx {}'.format(list(txs.keys()), tx_hashes)) - assert list(txs.keys()) == tx_hashes - - txs = get_account_tx(agent_roles['ALICE'], as_recipient=False) - assert list(txs.keys()) == tx_hashes[:3] - - txs = get_account_tx(agent_roles['ALICE'], as_sender=False) - assert list(txs.keys()) == tx_hashes[3:] diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 940457cb..d72b387f 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -57,9 +57,9 @@ WORKDIR /home/grassroots USER grassroots ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 -ARG cic_base_version=0.1.2a49 -ARG cic_eth_version=0.10.1b1 -ARG sarafu_faucet_version=0.0.2a13 +ARG cic_base_version=0.1.2a57 +ARG cic_eth_version=0.11.0a3 +ARG sarafu_faucet_version=0.0.2a16 ARG cic_contracts_version=0.0.2a2 RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \ cic-eth==$cic_eth_version \ diff --git a/apps/contract-migration/scripts/import_balance.py b/apps/contract-migration/scripts/import_balance.py index 244b8202..061e5348 100644 --- a/apps/contract-migration/scripts/import_balance.py +++ b/apps/contract-migration/scripts/import_balance.py @@ -174,25 +174,25 @@ class Handler: # logg.error('key record not found in imports: {}'.format(e).ljust(200)) -class BlockGetter: - - def __init__(self, conn, gas_oracle, nonce_oracle, chain_id): - self.conn = conn - self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id) - - - def get(self, n): - o = block_by_number(n) - r = self.conn.do(o) - b = None - try: - b = Block(r) - except TypeError as e: - if r == None: - logg.debug('block not found {}'.format(n)) - else: - logg.error('block retrieve error {}'.format(e)) - return b +#class BlockGetter: +# +# def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec): +# self.conn = conn +# self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id) +# +# +# def get(self, n): +# o = block_by_number(n) +# r = self.conn.do(o) +# b = None +# try: +# b = Block(r) +# except TypeError as e: +# if r == None: +# logg.debug('block not found {}'.format(n)) +# else: +# logg.error('block retrieve error {}'.format(e)) +# return b def progress_callback(block_number, tx_index, s): @@ -208,7 +208,7 @@ def main(): nonce_oracle = RPCNonceOracle(signer_address, conn) # Get Token registry address - txf = TxFactory(signer=signer, gas_oracle=gas_oracle, nonce_oracle=None, chain_id=chain_spec.network_id()) + txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None) tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS')) registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8] diff --git a/apps/contract-migration/scripts/import_sovereign_users.py b/apps/contract-migration/scripts/import_sovereign_users.py index 30e3f5db..bf7eff60 100644 --- a/apps/contract-migration/scripts/import_sovereign_users.py +++ b/apps/contract-migration/scripts/import_sovereign_users.py @@ -103,7 +103,7 @@ def register_eth(i, u): address = add_0x(to_checksum_address(address_hex)) gas_oracle = RPCGasOracle(rpc, code_callback=AccountRegistry.gas) - c = AccountRegistry(signer=signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id()) + c = AccountRegistry(chain_spec, signer=signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) (tx_hash_hex, o) = c.add(account_registry_address, signer_address, address) logg.debug('o {}'.format(o)) rpc.do(o) diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index 9955cfde..c7611236 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,5 +1,5 @@ -cic-base[full_graph]==0.1.2a52 -sarafu-faucet==0.0.2a15 -cic-eth==0.10.1b2+build.9c750e70 -cic-types==0.1.0a8 +cic-base[full_graph]==0.1.2a57 +sarafu-faucet==0.0.2a16 +cic-eth==0.11.0a3 +cic-types==0.1.0a10 crypto-dev-signer==0.4.14a17 diff --git a/apps/contract-migration/scripts/verify.py b/apps/contract-migration/scripts/verify.py index 08b791a7..d18435b9 100644 --- a/apps/contract-migration/scripts/verify.py +++ b/apps/contract-migration/scripts/verify.py @@ -203,12 +203,12 @@ class Verifier: self.index_address = index_address self.token_address = token_address self.faucet_address = faucet_address - self.erc20_tx_factory = ERC20(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle) - self.tx_factory = TxFactory(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle) + self.erc20_tx_factory = ERC20(chain_spec, gas_oracle=gas_oracle) + self.tx_factory = TxFactory(chain_spec, gas_oracle=gas_oracle) self.api = cic_eth_api self.data_dir = data_dir self.exit_on_error = exit_on_error - self.faucet_tx_factory = SingleShotFaucet(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle) + self.faucet_tx_factory = SingleShotFaucet(chain_spec, gas_oracle=gas_oracle) verifymethods = [] for k in dir(self): @@ -326,7 +326,7 @@ def main(): gas_oracle = OverrideGasOracle(conn=conn, limit=8000000) # Get Token registry address - txf = TxFactory(signer=None, gas_oracle=gas_oracle, nonce_oracle=None, chain_id=chain_spec.chain_id()) + txf = TxFactory(chain_spec, signer=None, gas_oracle=gas_oracle, nonce_oracle=None) tx = txf.template(ZERO_ADDRESS, config.get('CIC_REGISTRY_ADDRESS')) # TODO: replace with cic-eth-registry diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index b937cf2f..5d6da696 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -30,6 +30,8 @@ truncate $env_out_file -s 0 set -e set -a +#pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL eth-address-index==0.1.1a7 + # get required addresses from registries DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry` DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`