kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'lash/external-chain-queue' into 'master'

Browse Source 
Use external chain queue engine

See merge request grassrootseconomics/cic-internal-integration!84
This commit is contained in:
Louis Holbrook 2021-04-04 12:41:00 +00:00
commit 1f2fc3e952
99 changed files with 1255 additions and 4479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
apps/cic-eth/cic_eth/admin/nonce.py
View File

@ -5,11 +5,13 @@ import logging
import celery
from chainlib.chain import ChainSpec
from chainlib.eth.tx import unpack
from chainqueue.query import get_tx
from chainqueue.state import set_cancel
from chainqueue.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import Nonce
from cic_eth.admin.ctrl import (
lock_send,
@ -17,14 +19,8 @@ from cic_eth.admin.ctrl import (
lock_queue,
unlock_queue,
)
from cic_eth.queue.tx import (
get_tx,
set_cancel,
)
from cic_eth.queue.tx import create as queue_create
from cic_eth.eth.gas import (
create_check_gas_task,
)
from cic_eth.queue.tx import queue_create
from cic_eth.eth.gas import create_check_gas_task
celery_app = celery.current_app
logg = logging.getLogger()
@ -50,8 +46,8 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
chain_spec = ChainSpec.from_chain_str(chain_str)
tx_brief = get_tx(tx_hash_orig_hex)
tx_raw = bytes.fromhex(tx_brief['signed_tx'][2:])
tx = unpack(tx_raw, chain_spec.chain_id())
tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:]))
tx = unpack(tx_raw, chain_spec)
nonce = tx_brief['nonce']
address = tx['from']
@ -71,8 +67,8 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
tx_hashes = []
txs = []
for otx in otxs:
tx_raw = bytes.fromhex(otx.signed_tx[2:])
tx_new = unpack(tx_raw, chain_spec.chain_id())
tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
tx_new = unpack(tx_raw, chain_spec)
tx_previous_hash_hex = tx_new['hash']
tx_previous_nonce = tx_new['nonce']

59
apps/cic-eth/cic_eth/api/api_admin.py
View File

@ -22,23 +22,21 @@ from hexathon import (
add_0x,
)
from chainlib.eth.gas import balance
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import Nonce
from cic_eth.db.enum import (
from chainqueue.db.enum import (
StatusEnum,
StatusBits,
is_alive,
is_error_status,
status_str,
)
from chainqueue.error import TxStateChangeError
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.nonce import Nonce
from cic_eth.error import InitializationError
from cic_eth.db.error import TxStateChangeError
from cic_eth.queue.tx import get_tx
from cic_eth.queue.query import get_tx
app = celery.current_app
@ -92,7 +90,7 @@ class AdminApi:
def get_lock(self):
s_lock = celery.signature(
'cic_eth.queue.tx.get_lock',
'cic_eth.queue.lock.get_lock',
[],
queue=self.queue,
)
@ -138,8 +136,9 @@ class AdminApi:
logg.debug('resend {}'.format(tx_hash_hex))
s_get_tx_cache = celery.signature(
'cic_eth.queue.tx.get_tx_cache',
'cic_eth.queue.query.get_tx_cache',
[
chain_spec.asdict(),
tx_hash_hex,
],
queue=self.queue,
@ -155,7 +154,7 @@ class AdminApi:
raise NotImplementedError('resend as new not yet implemented')
s = celery.signature(
'cic_eth.eth.tx.resend_with_higher_gas',
'cic_eth.eth.gas.resend_with_higher_gas',
[
chain_spec.asdict(),
None,
@ -165,7 +164,7 @@ class AdminApi:
)
s_manual = celery.signature(
'cic_eth.queue.tx.set_manual',
'cic_eth.queue.state.set_manual',
[
tx_hash_hex,
],
@ -188,8 +187,9 @@ class AdminApi:
def check_nonce(self, address):
s = celery.signature(
'cic_eth.queue.tx.get_account_tx',
'cic_eth.queue.query.get_account_tx',
[
chain_spec.asdict(),
address,
True,
False,
@ -204,8 +204,9 @@ class AdminApi:
last_nonce = -1
for k in txs.keys():
s_get_tx = celery.signature(
'cic_eth.queue.tx.get_tx',
'cic_eth.queue.query.get_tx',
[
chain_spec.asdict(),
k,
],
queue=self.queue,
@ -243,8 +244,9 @@ class AdminApi:
def fix_nonce(self, address, nonce, chain_spec):
s = celery.signature(
'cic_eth.queue.tx.get_account_tx',
'cic_eth.queue.query.get_account_tx',
[
chain_spec.asdict(),
address,
True,
False,
@ -294,8 +296,9 @@ class AdminApi:
"""
last_nonce = -1
s = celery.signature(
'cic_eth.queue.tx.get_account_tx',
'cic_eth.queue.query.get_account_tx',
[
chain_spec.asdict(),
address,
],
queue=self.queue,
@ -306,8 +309,11 @@ class AdminApi:
for tx_hash in txs.keys():
errors = []
s = celery.signature(
'cic_eth.queue.tx.get_tx_cache',
[tx_hash],
'cic_eth.queue.query.get_tx_cache',
[
chain_spec.asdict(),
tx_hash,
],
queue=self.queue,
)
tx_dict = s.apply_async().get()
@ -367,12 +373,16 @@ class AdminApi:
#tx_hash = self.w3.keccak(hexstr=tx_raw).hex()
s = celery.signature(
'cic_eth.queue.tx.get_tx_cache',
[tx_hash],
'cic_eth.queue.query.get_tx_cache',
[
chain_spec.asdict(),
tx_hash,
],
queue=self.queue,
)
tx = s.apply_async().get()
t = s.apply_async()
tx = t.get()
source_token = None
if tx['source_token'] != ZERO_ADDRESS:
@ -524,8 +534,9 @@ class AdminApi:
tx['data'] = tx_unpacked['data']
s = celery.signature(
'cic_eth.queue.tx.get_state_log',
'cic_eth.queue.state.get_state_log',
[
chain_spec.asdict(),
tx_hash,
],
queue=self.queue,

26
apps/cic-eth/cic_eth/api/api_task.py
View File

@ -92,8 +92,10 @@ class Api:
queue=self.queue,
)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[],
'cic_eth.eth.nonce.reserve_nonce',
[
self.chain_spec.asdict(),
],
queue=self.queue,
)
s_tokens = celery.signature(
@ -154,8 +156,10 @@ class Api:
queue=self.queue,
)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[],
'cic_eth.eth.nonce.reserve_nonce',
[
self.chain_spec.asdict(),
],
queue=self.queue,
)
s_tokens = celery.signature(
@ -213,8 +217,9 @@ class Api:
queue=self.queue,
)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
self.chain_spec.asdict(),
from_address,
],
queue=self.queue,
@ -359,8 +364,9 @@ class Api:
if register:
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
self.chain_spec.asdict(),
'ACCOUNT_REGISTRY_WRITER',
],
queue=self.queue,
@ -397,14 +403,15 @@ class Api:
queue=self.queue,
)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
self.chain_spec.asdict(),
'GAS_GIFTER',
],
queue=self.queue,
)
s_refill = celery.signature(
'cic_eth.eth.tx.refill_gas',
'cic_eth.eth.gas.refill_gas',
[
self.chain_spec.asdict(),
],
@ -439,8 +446,9 @@ class Api:
"""
offset = 0
s_local = celery.signature(
'cic_eth.queue.tx.get_account_tx',
'cic_eth.queue.query.get_account_tx',
[
self.chain_spec.asdict(),
address,
],
queue=self.queue,

4
apps/cic-eth/cic_eth/db/__init__.py
View File

@ -11,10 +11,6 @@ logg = logging.getLogger()
# an Engine, which the Session will use for connection
# resources
# TODO: Remove the package exports, all models should be imported using full path
from .models.otx import Otx
from .models.convert import TxConvertTransfer
def dsn_from_config(config):
"""Generate a dsn string from the provided config dict.

29
apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py Normal file
View File

@ -0,0 +1,29 @@
"""Add chainqueue
Revision ID: 0ec0d6d1e785
Revises:
Create Date: 2021-04-02 18:30:55.398388
"""
from alembic import op
import sqlalchemy as sa
from chainqueue.db.migrations.sqlalchemy import (
chainqueue_upgrade,
chainqueue_downgrade,
)
# revision identifiers, used by Alembic.
revision = '0ec0d6d1e785'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
chainqueue_upgrade(0, 0, 1)
def downgrade():
chainqueue_downgrade(0, 0, 1)

29
apps/cic-eth/cic_eth/db/migrations/default/versions/1f1b3b641d08_roles.py Normal file
View File

@ -0,0 +1,29 @@
"""Roles
Revision ID: 1f1b3b641d08
Revises: 9c420530eeb2
Create Date: 2021-04-02 18:40:27.787631
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '1f1b3b641d08'
down_revision = '9c420530eeb2'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'account_role',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('tag', sa.Text, nullable=False, unique=True),
sa.Column('address_hex', sa.String(42), nullable=False),
)
def downgrade():
op.drop_table('account_role')

35
apps/cic-eth/cic_eth/db/migrations/default/versions/2a07b543335e_add_new_syncer_table.py
View File

@ -1,35 +0,0 @@
"""Add new syncer table
Revision ID: 2a07b543335e
Revises: a2e2aab8f331
Create Date: 2020-12-27 09:35:44.017981
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2a07b543335e'
down_revision = 'a2e2aab8f331'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'blockchain_sync',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('blockchain', sa.String, nullable=False),
sa.Column('block_start', sa.Integer, nullable=False, default=0),
sa.Column('tx_start', sa.Integer, nullable=False, default=0),
sa.Column('block_cursor', sa.Integer, nullable=False, default=0),
sa.Column('tx_cursor', sa.Integer, nullable=False, default=0),
sa.Column('block_target', sa.Integer, nullable=True),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('date_updated', sa.DateTime),
)
def downgrade():
op.drop_table('blockchain_sync')

31
apps/cic-eth/cic_eth/db/migrations/default/versions/3b693afd526a_nonce_reservation.py
View File

@ -1,31 +0,0 @@
"""Nonce reservation
Revision ID: 3b693afd526a
Revises: f738d9962fdf
Create Date: 2021-03-05 07:09:50.898728
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3b693afd526a'
down_revision = 'f738d9962fdf'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'nonce_task_reservation',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('address_hex', sa.String(42), nullable=False),
sa.Column('nonce', sa.Integer, nullable=False),
sa.Column('key', sa.String, nullable=False),
sa.Column('date_created', sa.DateTime, nullable=False),
)
def downgrade():
op.drop_table('nonce_task_reservation')

29
apps/cic-eth/cic_eth/db/migrations/default/versions/49b348246d70_add_nonce_index.py
View File

@ -1,29 +0,0 @@
"""Add nonce index
Revision ID: 49b348246d70
Revises: 52c7c59cd0b1
Create Date: 2020-12-19 09:45:36.186446
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '49b348246d70'
down_revision = '52c7c59cd0b1'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'nonce',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('address_hex', sa.String(42), nullable=False, unique=True),
sa.Column('nonce', sa.Integer, nullable=False),
)
def downgrade():
op.drop_table('nonce')

31
apps/cic-eth/cic_eth/db/migrations/default/versions/52c7c59cd0b1_add_account_roles.py
View File

@ -1,31 +0,0 @@
"""Add account roles
Revision ID: 52c7c59cd0b1
Revises: 9c4bd7491015
Create Date: 2020-12-19 07:21:38.249237
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '52c7c59cd0b1'
down_revision = '9c4bd7491015'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'account_role',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('tag', sa.Text, nullable=False, unique=True),
sa.Column('address_hex', sa.String(42), nullable=False),
)
pass
def downgrade():
op.drop_table('account_role')
pass

14
apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py → apps/cic-eth/cic_eth/db/migrations/default/versions/5ca4b77ce205_debug.py
View File

@ -1,8 +1,8 @@
"""debug output
"""DEbug
Revision ID: f738d9962fdf
Revises: ec40ac0974c1
Create Date: 2021-03-04 08:32:43.281214
Revision ID: 5ca4b77ce205
Revises: 75d4767b3031
Create Date: 2021-04-02 18:42:12.257244
"""
from alembic import op
@ -10,8 +10,8 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'f738d9962fdf'
down_revision = 'ec40ac0974c1'
revision = '5ca4b77ce205'
down_revision = '75d4767b3031'
branch_labels = None
depends_on = None
@ -24,9 +24,7 @@ def upgrade():
sa.Column('description', sa.String, nullable=False),
sa.Column('date_created', sa.DateTime, nullable=False),
)
pass
def downgrade():
op.drop_table('debug')
pass

30
apps/cic-eth/cic_eth/db/migrations/default/versions/6ac7a1dadc46_add_otx_state_log.py
View File

@ -1,30 +0,0 @@
"""Add otx state log
Revision ID: 6ac7a1dadc46
Revises: 89e1e9baa53c
Create Date: 2021-01-30 13:59:49.022373
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '6ac7a1dadc46'
down_revision = '89e1e9baa53c'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'otx_state_log',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False),
sa.Column('date', sa.DateTime, nullable=False),
sa.Column('status', sa.Integer, nullable=False),
)
def downgrade():
op.drop_table('otx_state_log')

31
apps/cic-eth/cic_eth/db/migrations/default/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py
View File

@ -1,31 +0,0 @@
"""Add attempts and version log for otx
Revision ID: 71708e943dbd
Revises: 7e8d7626e38f
Create Date: 2020-09-26 14:41:19.298651
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '71708e943dbd'
down_revision = '7e8d7626e38f'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'otx_attempts',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False),
sa.Column('date', sa.DateTime, nullable=False),
)
pass
def downgrade():
op.drop_table('otx_attempts')
pass

15
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/89e1e9baa53c_add_account_lock.py → apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
View File

@ -1,8 +1,8 @@
"""Add account lock
"""Lock
Revision ID: 89e1e9baa53c
Revises: 2a07b543335e
Create Date: 2021-01-27 19:57:36.793882
Revision ID: 75d4767b3031
Revises: 1f1b3b641d08
Create Date: 2021-04-02 18:41:20.864265
"""
from alembic import op
@ -10,8 +10,8 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '89e1e9baa53c'
down_revision = '2a07b543335e'
revision = '75d4767b3031'
down_revision = '1f1b3b641d08'
branch_labels = None
depends_on = None
@ -24,10 +24,11 @@ def upgrade():
sa.Column('blockchain', sa.String),
sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
sa.Column("date_created", sa.DateTime, nullable=False),
sa.Column("otx_id", sa.Integer, nullable=True),
sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
)
op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
def downgrade():
op.drop_index('idx_chain_address')
op.drop_table('lock')

31
apps/cic-eth/cic_eth/db/migrations/default/versions/7cb65b893934_add_blocknumber_pointer.py
View File

@ -1,31 +0,0 @@
"""add blocknumber pointer
Revision ID: 7cb65b893934
Revises: 8593fa1ca0f4
Create Date: 2020-09-24 19:29:13.543648
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '7cb65b893934'
down_revision = '8593fa1ca0f4'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'watcher_state',
sa.Column('block_number', sa.Integer)
)
conn = op.get_bind()
conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);')
pass
def downgrade():
op.drop_table('watcher_state')
pass

45
apps/cic-eth/cic_eth/db/migrations/default/versions/7e8d7626e38f_add_block_sync.py
View File

@ -1,45 +0,0 @@
"""Add block sync
Revision ID: 7e8d7626e38f
Revises: cd2052be6db2
Create Date: 2020-09-26 11:12:27.818524
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '7e8d7626e38f'
down_revision = 'cd2052be6db2'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'block_sync',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('blockchain', sa.String, nullable=False, unique=True),
sa.Column('block_height_backlog', sa.Integer, nullable=False, default=0),
sa.Column('tx_height_backlog', sa.Integer, nullable=False, default=0),
sa.Column('block_height_session', sa.Integer, nullable=False, default=0),
sa.Column('tx_height_session', sa.Integer, nullable=False, default=0),
sa.Column('block_height_head', sa.Integer, nullable=False, default=0),
sa.Column('tx_height_head', sa.Integer, nullable=False, default=0),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('date_updated', sa.DateTime),
)
op.drop_table('watcher_state')
pass
def downgrade():
op.drop_table('block_sync')
op.create_table(
'watcher_state',
sa.Column('block_number', sa.Integer)
)
conn = op.get_bind()
conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);')
pass

35
apps/cic-eth/cic_eth/db/migrations/default/versions/8593fa1ca0f4_add_transaction_queue.py
View File

@ -1,35 +0,0 @@
"""Add transaction queue
Revision ID: 8593fa1ca0f4
Revises:
Create Date: 2020-09-22 21:56:42.117047
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '8593fa1ca0f4'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'otx',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('nonce', sa.Integer, nullable=False),
sa.Column('tx_hash', sa.String(66), nullable=False),
sa.Column('signed_tx', sa.Text, nullable=False),
sa.Column('status', sa.Integer, nullable=False, default=-9),
sa.Column('block', sa.Integer),
)
op.create_index('idx_otx_tx', 'otx', ['tx_hash'], unique=True)
def downgrade():
op.drop_index('idx_otx_tx')
op.drop_table('otx')

33
apps/cic-eth/cic_eth/db/migrations/default/versions/89e1e9baa53c_add_account_lock.py
View File

@ -1,33 +0,0 @@
"""Add account lock
Revision ID: 89e1e9baa53c
Revises: 2a07b543335e
Create Date: 2021-01-27 19:57:36.793882
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '89e1e9baa53c'
down_revision = '2a07b543335e'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'lock',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column("address", sa.String(42), nullable=True),
sa.Column('blockchain', sa.String),
sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
sa.Column("date_created", sa.DateTime, nullable=False),
sa.Column("otx_id", sa.Integer, nullable=True),
)
op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
def downgrade():
op.drop_index('idx_chain_address')
op.drop_table('lock')

22
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/3b693afd526a_nonce_reservation.py → apps/cic-eth/cic_eth/db/migrations/default/versions/9c420530eeb2_nonce.py
View File

@ -1,8 +1,8 @@
"""Nonce reservation
"""Nonce
Revision ID: 3b693afd526a
Revises: f738d9962fdf
Create Date: 2021-03-05 07:09:50.898728
Revision ID: 9c420530eeb2
Revises: b125cbf81e32
Create Date: 2021-04-02 18:38:56.459334
"""
from alembic import op
@ -10,15 +10,22 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3b693afd526a'
down_revision = 'f738d9962fdf'
revision = '9c420530eeb2'
down_revision = 'b125cbf81e32'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'nonce_task_reservation',
'nonce',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('address_hex', sa.String(42), nullable=False, unique=True),
sa.Column('nonce', sa.Integer, nullable=False),
)
op.create_table(
'nonce_task_reservation',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('address_hex', sa.String(42), nullable=False),
sa.Column('nonce', sa.Integer, nullable=False),
@ -29,3 +36,4 @@ def upgrade():
def downgrade():
op.drop_table('nonce_task_reservation')
op.drop_table('nonce')

26
apps/cic-eth/cic_eth/db/migrations/default/versions/9c4bd7491015_rename_block_sync_table.py
View File

@ -1,26 +0,0 @@
"""Rename block sync table
Revision ID: 9c4bd7491015
Revises: 9daa16518a91
Create Date: 2020-10-15 23:45:56.306898
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9c4bd7491015'
down_revision = '9daa16518a91'
branch_labels = None
depends_on = None
def upgrade():
op.rename_table('block_sync', 'otx_sync')
pass
def downgrade():
op.rename_table('otx_sync', 'block_sync')
pass

30
apps/cic-eth/cic_eth/db/migrations/default/versions/9daa16518a91_add_tx_sync_state.py
View File

@ -1,30 +0,0 @@
"""add tx sync state
Revision ID: 9daa16518a91
Revises: e3b5330ee71c
Create Date: 2020-10-10 14:43:18.699276
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9daa16518a91'
down_revision = 'e3b5330ee71c'
branch_labels = None
depends_on = None
def upgrade():
# op.create_table(
# 'tx_sync',
# sa.Column('tx', sa.String(66), nullable=False),
# )
# op.execute("INSERT INTO tx_sync VALUES('0x0000000000000000000000000000000000000000000000000000000000000000')")
pass
def downgrade():
# op.drop_table('tx_sync')
pass

34
apps/cic-eth/cic_eth/db/migrations/default/versions/a2e2aab8f331_add_date_accessed_to_txcache.py
View File

@ -1,34 +0,0 @@
"""Add date accessed to txcache
Revision ID: a2e2aab8f331
Revises: 49b348246d70
Create Date: 2020-12-24 18:58:06.137812
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'a2e2aab8f331'
down_revision = '49b348246d70'
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
'tx_cache',
sa.Column(
'date_checked',
sa.DateTime,
nullable=False
)
)
pass
def downgrade():
# drop does not work withs qlite
#op.drop_column('tx_cache', 'date_checked')
pass

14
apps/cic-eth/cic_eth/db/migrations/default/versions/cd2052be6db2_convert_tx_index.py → apps/cic-eth/cic_eth/db/migrations/default/versions/aee12aeb47ec_convert.py
View File

@ -1,8 +1,8 @@
"""convert tx index
"""Convert
Revision ID: cd2052be6db2
Revises: 7cb65b893934
Create Date: 2020-09-24 21:20:51.580500
Revision ID: aee12aeb47ec
Revises: 5ca4b77ce205
Create Date: 2021-04-02 18:42:45.233356
"""
from alembic import op
@ -10,8 +10,8 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'cd2052be6db2'
down_revision = '7cb65b893934'
revision = 'aee12aeb47ec'
down_revision = '5ca4b77ce205'
branch_labels = None
depends_on = None
@ -20,10 +20,8 @@ def upgrade():
op.create_table(
'tx_convert_transfer',
sa.Column('id', sa.Integer, primary_key=True),
#sa.Column('approve_tx_hash', sa.String(66), nullable=False, unique=True),
sa.Column('convert_tx_hash', sa.String(66), nullable=False, unique=True),
sa.Column('transfer_tx_hash', sa.String(66), unique=True),
# sa.Column('holder_address', sa.String(42), nullable=False),
sa.Column('recipient_address', sa.String(42), nullable=False),
)
op.create_index('idx_tx_convert_address', 'tx_convert_transfer', ['recipient_address'])

13
apps/cic-eth/cic_eth/db/migrations/default/versions/ec40ac0974c1_add_chain_syncer.py → apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py
View File

@ -1,12 +1,13 @@
"""Add chain syncer
Revision ID: ec40ac0974c1
Revises: 6ac7a1dadc46
Create Date: 2021-02-23 06:10:19.246304
Revision ID: b125cbf81e32
Revises: 0ec0d6d1e785
Create Date: 2021-04-02 18:36:44.459603
"""
from alembic import op
import sqlalchemy as sa
from chainsyncer.db.migrations.sqlalchemy import (
chainsyncer_upgrade,
chainsyncer_downgrade,
@ -14,15 +15,15 @@ from chainsyncer.db.migrations.sqlalchemy import (
# revision identifiers, used by Alembic.
revision = 'ec40ac0974c1'
down_revision = '6ac7a1dadc46'
revision = 'b125cbf81e32'
down_revision = '0ec0d6d1e785'
branch_labels = None
depends_on = None
def upgrade():
chainsyncer_upgrade(0, 0, 1)
def downgrade():
chainsyncer_downgrade(0, 0, 1)

31
apps/cic-eth/cic_eth/db/migrations/default/versions/df19f4e69676_add_tx_track.py
View File

@ -1,31 +0,0 @@
"""Add tx tracker record
Revision ID: df19f4e69676
Revises: 71708e943dbd
Create Date: 2020-10-09 23:31:44.563498
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'df19f4e69676'
down_revision = '71708e943dbd'
branch_labels = None
depends_on = None
def upgrade():
# op.create_table(
# 'tx',
# sa.Column('id', sa.Integer, primary_key=True),
# sa.Column('date_added', sa.DateTime, nullable=False),
# sa.Column('tx_hash', sa.String(66), nullable=False, unique=True),
# sa.Column('success', sa.Boolean(), nullable=False),
# )
pass
def downgrade():
# op.drop_table('tx')
pass

38
apps/cic-eth/cic_eth/db/migrations/default/versions/e3b5330ee71c_.py
View File

@ -1,38 +0,0 @@
"""Add cached values for tx
Revision ID: e3b5330ee71c
Revises: df19f4e69676
Create Date: 2020-10-10 00:17:07.094893
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'e3b5330ee71c'
down_revision = 'df19f4e69676'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'tx_cache',
sa.Column('id', sa.Integer, primary_key=True),
# sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=True),
sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('date_updated', sa.DateTime, nullable=False),
sa.Column('source_token_address', sa.String(42), nullable=False),
sa.Column('destination_token_address', sa.String(42), nullable=False),
sa.Column('sender', sa.String(42), nullable=False),
sa.Column('recipient', sa.String(42), nullable=False),
sa.Column('from_value', sa.NUMERIC(), nullable=False),
sa.Column('to_value', sa.NUMERIC(), nullable=True),
sa.Column('block_number', sa.BIGINT(), nullable=True),
sa.Column('tx_index', sa.Integer, nullable=True),
)
def downgrade():
op.drop_table('tx_cache')
pass

85
apps/cic-eth/cic_eth/db/migrations/postgresql/alembic.ini
View File

@ -1,85 +0,0 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = .
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# timezone to use when rendering the date
# within the migration file as well as the filename.
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; this defaults
# to migrations/versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat migrations/versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
#sqlalchemy.url = driver://user:pass@localhost/dbname
sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic-eth
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks=black
# black.type=console_scripts
# black.entrypoint=black
# black.options=-l 79
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

77
apps/cic-eth/cic_eth/db/migrations/postgresql/env.py
View File

@ -1,77 +0,0 @@
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
apps/cic-eth/cic_eth/db/migrations/postgresql/script.py.mako
View File

@ -1,24 +0,0 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

35
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/2a07b543335e_add_new_syncer_table.py
View File

@ -1,35 +0,0 @@
"""Add new syncer table
Revision ID: 2a07b543335e
Revises: a2e2aab8f331
Create Date: 2020-12-27 09:35:44.017981
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2a07b543335e'
down_revision = 'a2e2aab8f331'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'blockchain_sync',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('blockchain', sa.String, nullable=False),
sa.Column('block_start', sa.Integer, nullable=False, default=0),
sa.Column('tx_start', sa.Integer, nullable=False, default=0),
sa.Column('block_cursor', sa.Integer, nullable=False, default=0),
sa.Column('tx_cursor', sa.Integer, nullable=False, default=0),
sa.Column('block_target', sa.Integer, nullable=True),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('date_updated', sa.DateTime),
)
def downgrade():
op.drop_table('blockchain_sync')

29
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/49b348246d70_add_nonce_index.py
View File

@ -1,29 +0,0 @@
"""Add nonce index
Revision ID: 49b348246d70
Revises: 52c7c59cd0b1
Create Date: 2020-12-19 09:45:36.186446
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '49b348246d70'
down_revision = '52c7c59cd0b1'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'nonce',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('address_hex', sa.String(42), nullable=False, unique=True),
sa.Column('nonce', sa.Integer, nullable=False),
)
def downgrade():
op.drop_table('nonce')

31
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/52c7c59cd0b1_add_account_roles.py
View File

@ -1,31 +0,0 @@
"""Add account roles
Revision ID: 52c7c59cd0b1
Revises: 9c4bd7491015
Create Date: 2020-12-19 07:21:38.249237
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '52c7c59cd0b1'
down_revision = '9c4bd7491015'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'account_role',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('tag', sa.Text, nullable=False, unique=True),
sa.Column('address_hex', sa.String(42), nullable=False),
)
pass
def downgrade():
op.drop_table('account_role')
pass

30
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/6ac7a1dadc46_add_otx_state_log.py
View File

@ -1,30 +0,0 @@
"""Add otx state log
Revision ID: 6ac7a1dadc46
Revises: 89e1e9baa53c
Create Date: 2021-01-30 13:59:49.022373
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '6ac7a1dadc46'
down_revision = '89e1e9baa53c'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'otx_state_log',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False),
sa.Column('date', sa.DateTime, nullable=False),
sa.Column('status', sa.Integer, nullable=False),
)
def downgrade():
op.drop_table('otx_state_log')

31
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/71708e943dbd_add_attempts_and_version_log_for_otx.py
View File

@ -1,31 +0,0 @@
"""Add attempts and version log for otx
Revision ID: 71708e943dbd
Revises: 7e8d7626e38f
Create Date: 2020-09-26 14:41:19.298651
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '71708e943dbd'
down_revision = '7e8d7626e38f'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'otx_attempts',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=False),
sa.Column('date', sa.DateTime, nullable=False),
)
pass
def downgrade():
op.drop_table('otx_attempts')
pass

31
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7cb65b893934_add_blocknumber_pointer.py
View File

@ -1,31 +0,0 @@
"""add blocknumber pointer
Revision ID: 7cb65b893934
Revises: 8593fa1ca0f4
Create Date: 2020-09-24 19:29:13.543648
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '7cb65b893934'
down_revision = '8593fa1ca0f4'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'watcher_state',
sa.Column('block_number', sa.Integer)
)
conn = op.get_bind()
conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);')
pass
def downgrade():
op.drop_table('watcher_state')
pass

42
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/7e8d7626e38f_add_block_sync.py
View File

@ -1,42 +0,0 @@
"""Add block sync
Revision ID: 7e8d7626e38f
Revises: cd2052be6db2
Create Date: 2020-09-26 11:12:27.818524
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '7e8d7626e38f'
down_revision = 'cd2052be6db2'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'block_sync',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('blockchain', sa.String, nullable=False, unique=True),
sa.Column('height_backlog', sa.Integer, nullable=False, default=0),
sa.Column('height_session', sa.Integer, nullable=False, default=0),
sa.Column('height_head', sa.Integer, nullable=False, default=0),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('date_updated', sa.DateTime),
)
op.drop_table('watcher_state')
pass
def downgrade():
op.drop_table('block_sync')
op.create_table(
'watcher_state',
sa.Column('block_number', sa.Integer)
)
conn = op.get_bind()
conn.execute('INSERT INTO watcher_state (block_number) VALUES (0);')
pass

35
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/8593fa1ca0f4_add_transaction_queue.py
View File

@ -1,35 +0,0 @@
"""Add transaction queue
Revision ID: 8593fa1ca0f4
Revises:
Create Date: 2020-09-22 21:56:42.117047
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '8593fa1ca0f4'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'otx',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('nonce', sa.Integer, nullable=False),
sa.Column('tx_hash', sa.String(66), nullable=False),
sa.Column('signed_tx', sa.Text, nullable=False),
sa.Column('status', sa.Integer, nullable=False, default=-9),
sa.Column('block', sa.Integer),
)
op.create_index('idx_otx_tx', 'otx', ['tx_hash'], unique=True)
def downgrade():
op.drop_index('idx_otx_tx')
op.drop_table('otx')

26
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9c4bd7491015_rename_block_sync_table.py
View File

@ -1,26 +0,0 @@
"""Rename block sync table
Revision ID: 9c4bd7491015
Revises: 9daa16518a91
Create Date: 2020-10-15 23:45:56.306898
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9c4bd7491015'
down_revision = '9daa16518a91'
branch_labels = None
depends_on = None
def upgrade():
op.rename_table('block_sync', 'otx_sync')
pass
def downgrade():
op.rename_table('otx_sync', 'block_sync')
pass

30
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/9daa16518a91_add_tx_sync_state.py
View File

@ -1,30 +0,0 @@
"""add tx sync state
Revision ID: 9daa16518a91
Revises: e3b5330ee71c
Create Date: 2020-10-10 14:43:18.699276
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9daa16518a91'
down_revision = 'e3b5330ee71c'
branch_labels = None
depends_on = None
def upgrade():
# op.create_table(
# 'tx_sync',
# sa.Column('tx', sa.String(66), nullable=False),
# )
# op.execute("INSERT INTO tx_sync VALUES('0x0000000000000000000000000000000000000000000000000000000000000000')")
pass
def downgrade():
# op.drop_table('tx_sync')
pass

33
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/a2e2aab8f331_add_date_accessed_to_txcache.py
View File

@ -1,33 +0,0 @@
"""Add date accessed to txcache
Revision ID: a2e2aab8f331
Revises: 49b348246d70
Create Date: 2020-12-24 18:58:06.137812
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'a2e2aab8f331'
down_revision = '49b348246d70'
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
'tx_cache',
sa.Column(
'date_checked',
sa.DateTime,
nullable=False
)
)
pass
def downgrade():
op.drop_column('tx_cache', 'date_checked')
pass

34
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/cd2052be6db2_convert_tx_index.py
View File

@ -1,34 +0,0 @@
"""convert tx index
Revision ID: cd2052be6db2
Revises: 7cb65b893934
Create Date: 2020-09-24 21:20:51.580500
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'cd2052be6db2'
down_revision = '7cb65b893934'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'tx_convert_transfer',
sa.Column('id', sa.Integer, primary_key=True),
#sa.Column('approve_tx_hash', sa.String(66), nullable=False, unique=True),
sa.Column('convert_tx_hash', sa.String(66), nullable=False, unique=True),
sa.Column('transfer_tx_hash', sa.String(66), unique=True),
# sa.Column('holder_address', sa.String(42), nullable=False),
sa.Column('recipient_address', sa.String(42), nullable=False),
)
op.create_index('idx_tx_convert_address', 'tx_convert_transfer', ['recipient_address'])
def downgrade():
op.drop_index('idx_tx_convert_address')
op.drop_table('tx_convert_transfer')

31
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/df19f4e69676_add_tx_track.py
View File

@ -1,31 +0,0 @@
"""Add tx tracker record
Revision ID: df19f4e69676
Revises: 71708e943dbd
Create Date: 2020-10-09 23:31:44.563498
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'df19f4e69676'
down_revision = '71708e943dbd'
branch_labels = None
depends_on = None
def upgrade():
# op.create_table(
# 'tx',
# sa.Column('id', sa.Integer, primary_key=True),
# sa.Column('date_added', sa.DateTime, nullable=False),
# sa.Column('tx_hash', sa.String(66), nullable=False, unique=True),
# sa.Column('success', sa.Boolean(), nullable=False),
# )
pass
def downgrade():
# op.drop_table('tx')
pass

37
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/e3b5330ee71c_.py
View File

@ -1,37 +0,0 @@
"""Add cached values for tx
Revision ID: e3b5330ee71c
Revises: df19f4e69676
Create Date: 2020-10-10 00:17:07.094893
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'e3b5330ee71c'
down_revision = 'df19f4e69676'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'tx_cache',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
sa.Column('date_created', sa.DateTime, nullable=False),
sa.Column('date_updated', sa.DateTime, nullable=False),
sa.Column('source_token_address', sa.String(42), nullable=False),
sa.Column('destination_token_address', sa.String(42), nullable=False),
sa.Column('sender', sa.String(42), nullable=False),
sa.Column('recipient', sa.String(42), nullable=False),
sa.Column('from_value', sa.NUMERIC(), nullable=False),
sa.Column('to_value', sa.NUMERIC(), nullable=True),
sa.Column('block_number', sa.BIGINT(), nullable=True),
sa.Column('tx_index', sa.Integer, nullable=True),
)
def downgrade():
op.drop_table('tx_cache')
pass

28
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/ec40ac0974c1_add_chain_syncer.py
View File

@ -1,28 +0,0 @@
"""Add chain syncer
Revision ID: ec40ac0974c1
Revises: 6ac7a1dadc46
Create Date: 2021-02-23 06:10:19.246304
"""
from alembic import op
import sqlalchemy as sa
from chainsyncer.db.migrations.sqlalchemy import (
chainsyncer_upgrade,
chainsyncer_downgrade,
)
# revision identifiers, used by Alembic.
revision = 'ec40ac0974c1'
down_revision = '6ac7a1dadc46'
branch_labels = None
depends_on = None
def upgrade():
chainsyncer_upgrade(0, 0, 1)
def downgrade():
chainsyncer_downgrade(0, 0, 1)

32
apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py
View File

@ -1,32 +0,0 @@
"""debug output
Revision ID: f738d9962fdf
Revises: ec40ac0974c1
Create Date: 2021-03-04 08:32:43.281214
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'f738d9962fdf'
down_revision = 'ec40ac0974c1'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'debug',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('tag', sa.String, nullable=False),
sa.Column('description', sa.String, nullable=False),
sa.Column('date_created', sa.DateTime, nullable=False),
)
pass
def downgrade():
op.drop_table('debug')
pass

10
apps/cic-eth/cic_eth/db/models/lock.py
View File

@ -5,11 +5,11 @@ import logging
# third-party imports
from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
from chainlib.eth.constant import ZERO_ADDRESS
from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.otx import Otx
logg = logging.getLogger()
@ -22,10 +22,12 @@ class Lock(SessionBase):
__tablename__ = "lock"
blockchain = Column(String)
address = Column(String, ForeignKey('tx_cache.sender'))
#address = Column(String, ForeignKey('tx_cache.sender'))
address = Column(String, ForeignKey(TxCache.sender))
flags = Column(Integer)
date_created = Column(DateTime, default=datetime.datetime.utcnow)
otx_id = Column(Integer, ForeignKey('otx.id'))
otx_id = Column(Integer, ForeignKey(Otx.id))
#otx_id = Column(Integer)
def chain(self):

680
apps/cic-eth/cic_eth/db/models/otx.py
View File

@ -1,680 +0,0 @@
# standard imports
import datetime
import logging
# external imports
from sqlalchemy import Column, Enum, String, Integer, DateTime, Text, or_, ForeignKey
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
# local imports
from .base import SessionBase
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
status_str,
is_error_status,
)
from cic_eth.db.error import TxStateChangeError
logg = logging.getLogger()
class OtxStateLog(SessionBase):
__tablename__ = 'otx_state_log'
date = Column(DateTime, default=datetime.datetime.utcnow)
status = Column(Integer)
otx_id = Column(Integer, ForeignKey('otx.id'))
def __init__(self, otx):
self.otx_id = otx.id
self.status = otx.status
class Otx(SessionBase):
"""Outgoing transactions with local origin.
:param nonce: Transaction nonce
:type nonce: number
:param address: Ethereum address of recipient - NOT IN USE, REMOVE
:type address: str
:param tx_hash: Tranasction hash
:type tx_hash: str, 0x-hex
:param signed_tx: Signed raw transaction data
:type signed_tx: str, 0x-hex
"""
__tablename__ = 'otx'
tracing = False
"""Whether to enable queue state tracing"""
nonce = Column(Integer)
date_created = Column(DateTime, default=datetime.datetime.utcnow)
tx_hash = Column(String(66))
signed_tx = Column(Text)
status = Column(Integer)
block = Column(Integer)
def __set_status(self, status, session):
self.status |= status
session.add(self)
session.flush()
def __reset_status(self, status, session):
status_edit = ~status & self.status
self.status &= status_edit
session.add(self)
session.flush()
def __status_already_set(self, status):
r = bool(self.status & status)
if r:
logg.warning('status bit {} already set on {}'.format(status.name, self.tx_hash))
return r
def __status_not_set(self, status):
r = not(self.status & status)
if r:
logg.warning('status bit {} not set on {}'.format(status.name, self.tx_hash))
return r
def set_block(self, block, session=None):
"""Set block number transaction was mined in.
Only manipulates object, does not transaction or commit to backend.
:param block: Block number
:type block: number
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
session = SessionBase.bind_session(session)
if self.block != None:
SessionBase.release_session(session)
raise TxStateChangeError('Attempted set block {} when block was already {}'.format(block, self.block))
self.block = block
session.add(self)
session.flush()
SessionBase.release_session(session)
def waitforgas(self, session=None):
"""Marks transaction as suspended pending gas funding.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.GAS_ISSUES):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.GAS_ISSUES, session)
self.__reset_status(StatusBits.QUEUED | StatusBits.DEFERRED, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def fubar(self, session=None):
"""Marks transaction as "fubar." Any transaction marked this way is an anomaly and may be a symptom of a serious problem.
Only manipulates object, does not transaction or commit to backend.
"""
if self.__status_already_set(StatusBits.UNKNOWN_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('FUBAR cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('FUBAR cannot be set on an entry with an error state already set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.UNKNOWN_ERROR | StatusBits.FINAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def reject(self, session=None):
"""Marks transaction as "rejected," which means the node rejected sending the transaction to the network. The nonce has not been spent, and the transaction should be replaced.
Only manipulates object, does not transaction or commit to backend.
"""
if self.__status_already_set(StatusBits.NODE_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('REJECTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('REJECTED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('REJECTED cannot be set on an entry with an error state already set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.NODE_ERROR | StatusBits.FINAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def override(self, manual=False, session=None):
"""Marks transaction as manually overridden.
Only manipulates object, does not transaction or commit to backend.
"""
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status)))
if self.status & StatusBits.OBSOLETE:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already OBSOLETE ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.OBSOLETE, session)
#if manual:
# self.__set_status(StatusBits.MANUAL, session)
self.__reset_status(StatusBits.QUEUED | StatusBits.IN_NETWORK, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def manual(self, session=None):
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.MANUAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def retry(self, session=None):
"""Marks transaction as ready to retry after a timeout following a sendfail or a completed gas funding.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.QUEUED):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('RETRY cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if not is_error_status(self.status) and not StatusBits.IN_NETWORK & self.status > 0:
SessionBase.release_session(session)
raise TxStateChangeError('RETRY cannot be set on an entry that has no error ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.QUEUED, session)
self.__reset_status(StatusBits.GAS_ISSUES, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def readysend(self, session=None):
"""Marks transaction as ready for initial send attempt.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.QUEUED):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('READYSEND cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('READYSEND cannot be set on an errored state ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.QUEUED, session)
self.__reset_status(StatusBits.GAS_ISSUES, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def sent(self, session=None):
"""Marks transaction as having been sent to network.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.IN_NETWORK):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('SENT cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.IN_NETWORK, session)
self.__reset_status(StatusBits.DEFERRED | StatusBits.QUEUED | StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def sendfail(self, session=None):
"""Marks that an attempt to send the transaction to the network has failed.
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.NODE_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('SENDFAIL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('SENDFAIL cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
self.__set_status(StatusBits.LOCAL_ERROR | StatusBits.DEFERRED, session)
self.__reset_status(StatusBits.QUEUED | StatusBits.GAS_ISSUES, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def dequeue(self, session=None):
"""Marks that a process to execute send attempt is underway
Only manipulates object, does not transaction or commit to backend.
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_not_set(StatusBits.QUEUED):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('QUEUED cannot be unset on an entry with FINAL state set ({})'.format(status_str(self.status)))
if self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('QUEUED cannot be unset on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
self.__reset_status(StatusBits.QUEUED, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def minefail(self, block, session=None):
"""Marks that transaction was mined but code execution did not succeed.
Only manipulates object, does not transaction or commit to backend.
:param block: Block number transaction was mined in.
:type block: number
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
if self.__status_already_set(StatusBits.NETWORK_ERROR):
return
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('REVERTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if not self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('REVERTED cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status)))
if block != None:
self.block = block
self.__set_status(StatusBits.NETWORK_ERROR | StatusBits.FINAL, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def cancel(self, confirmed=False, session=None):
"""Marks that the transaction has been succeeded by a new transaction with same nonce.
If set to confirmed, the previous state must be OBSOLETED, and will transition to CANCELLED - a finalized state. Otherwise, the state must follow a non-finalized state, and will be set to OBSOLETED.
Only manipulates object, does not transaction or commit to backend.
:param confirmed: Whether transition is to a final state.
:type confirmed: bool
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('CANCEL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if confirmed:
if self.status > 0 and not self.status & StatusBits.OBSOLETE:
SessionBase.release_session(session)
raise TxStateChangeError('CANCEL can only be set on an entry marked OBSOLETE ({})'.format(status_str(self.status)))
self.__set_status(StatusEnum.CANCELLED, session)
else:
self.__set_status(StatusEnum.OBSOLETED, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
def success(self, block, session=None):
"""Marks that transaction was successfully mined.
Only manipulates object, does not transaction or commit to backend.
:param block: Block number transaction was mined in.
:type block: number
:raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
"""
session = SessionBase.bind_session(session)
if self.status & StatusBits.FINAL:
SessionBase.release_session(session)
raise TxStateChangeError('SUCCESS cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
if not self.status & StatusBits.IN_NETWORK:
SessionBase.release_session(session)
raise TxStateChangeError('SUCCESS cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status)))
if is_error_status(self.status):
SessionBase.release_session(session)
raise TxStateChangeError('SUCCESS cannot be set on an entry with error state set ({})'.format(status_str(self.status)))
if block != None:
self.block = block
self.__set_status(StatusEnum.SUCCESS, session)
if self.tracing:
self.__state_log(session=session)
SessionBase.release_session(session)
@staticmethod
def get(status=0, limit=4096, status_exact=True, session=None):
"""Returns outgoing transaction lists by status.
Status may either be matched exactly, or be an upper bound of the integer value of the status enum.
:param status: Status value to use in query
:type status: cic_eth.db.enum.StatusEnum
:param limit: Max results to return
:type limit: number
:param status_exact: Whether or not to perform exact status match
:type bool:
:returns: List of transaction hashes
:rtype: tuple, where first element is transaction hash
"""
e = None
session = SessionBase.bind_session(session)
if status_exact:
e = session.query(Otx.tx_hash).filter(Otx.status==status).order_by(Otx.date_created.asc()).limit(limit).all()
else:
e = session.query(Otx.tx_hash).filter(Otx.status<=status).order_by(Otx.date_created.asc()).limit(limit).all()
SessionBase.release_session(session)
return e
@staticmethod
def load(tx_hash, session=None):
"""Retrieves the outgoing transaction record by transaction hash.
:param tx_hash: Transaction hash
:type tx_hash: str, 0x-hex
"""
session = SessionBase.bind_session(session)
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
SessionBase.release_session(session)
return q.first()
@staticmethod
def account(account_address):
"""Retrieves all transaction hashes for which the given Ethereum address is sender or recipient.
:param account_address: Ethereum address to use in query.
:type account_address: str, 0x-hex
:returns: Outgoing transactions
:rtype: tuple, where first element is transaction hash
"""
session = Otx.create_session()
q = session.query(Otx.tx_hash)
q = q.join(TxCache)
q = q.filter(or_(TxCache.sender==account_address, TxCache.recipient==account_address))
txs = q.all()
session.close()
return list(txs)
def __state_log(self, session):
l = OtxStateLog(self)
session.add(l)
# TODO: it is not safe to return otx here unless session has been passed in
@staticmethod
def add(nonce, address, tx_hash, signed_tx, session=None):
external_session = session != None
session = SessionBase.bind_session(session)
otx = Otx(nonce, address, tx_hash, signed_tx)
session.add(otx)
session.flush()
if otx.tracing:
otx.__state_log(session=session)
session.flush()
SessionBase.release_session(session)
if not external_session:
return None
return otx
def __init__(self, nonce, address, tx_hash, signed_tx):
self.nonce = nonce
self.tx_hash = tx_hash
self.signed_tx = signed_tx
self.status = StatusEnum.PENDING
signed_tx_bytes = bytes.fromhex(signed_tx[2:])
# sender_address = address_hex_from_signed_tx(signed_tx_bytes)
# logg.debug('decoded tx {}'.format(sender_address))
# TODO: Most of the methods on this object are obsolete, but it contains a static function for retrieving "expired" outgoing transactions that should be moved to Otx instead.
class OtxSync(SessionBase):
"""Obsolete
"""
__tablename__ = 'otx_sync'
blockchain = Column(String)
block_height_backlog = Column(Integer)
tx_height_backlog = Column(Integer)
block_height_session = Column(Integer)
tx_height_session = Column(Integer)
block_height_head = Column(Integer)
tx_height_head = Column(Integer)
date_created = Column(DateTime, default=datetime.datetime.utcnow)
date_updated = Column(DateTime)
def backlog(self, block_height=None, tx_height=None):
#session = OtxSync.create_session()
if block_height != None:
if tx_height == None:
raise ValueError('tx height missing')
self.block_height_backlog = block_height
self.tx_height_backlog = tx_height
#session.add(self)
self.date_updated = datetime.datetime.utcnow()
#session.commit()
block_height = self.block_height_backlog
tx_height = self.tx_height_backlog
#session.close()
return (block_height, tx_height)
def session(self, block_height=None, tx_height=None):
#session = OtxSync.create_session()
if block_height != None:
if tx_height == None:
raise ValueError('tx height missing')
self.block_height_session = block_height
self.tx_height_session = tx_height
#session.add(self)
self.date_updated = datetime.datetime.utcnow()
#session.commit()
block_height = self.block_height_session
tx_height = self.tx_height_session
#session.close()
return (block_height, tx_height)
def head(self, block_height=None, tx_height=None):
#session = OtxSync.create_session()
if block_height != None:
if tx_height == None:
raise ValueError('tx height missing')
self.block_height_head = block_height
self.tx_height_head = tx_height
#session.add(self)
self.date_updated = datetime.datetime.utcnow()
#session.commit()
block_height = self.block_height_head
tx_height = self.tx_height_head
#session.close()
return (block_height, tx_height)
@hybrid_property
def synced(self):
#return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.block_height_backlog
return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.tx_height_backlog
@staticmethod
def load(blockchain_string, session):
q = session.query(OtxSync)
q = q.filter(OtxSync.blockchain==blockchain_string)
return q.first()
@staticmethod
def latest(nonce):
session = SessionBase.create_session()
otx = session.query(Otx).filter(Otx.nonce==nonce).order_by(Otx.created.desc()).first()
session.close()
return otx
@staticmethod
def get_expired(datetime_threshold):
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.date_created<datetime_threshold)
q = q.filter(Otx.status==StatusEnum.SENT)
q = q.order_by(Otx.date_created.desc())
q = q.group_by(Otx.nonce)
q = q.group_by(Otx.id)
otxs = q.all()
session.close()
return otxs
def chain(self):
return self.blockchain
def __init__(self, blockchain):
self.blockchain = blockchain
self.block_height_head = 0
self.tx_height_head = 0
self.block_height_session = 0
self.tx_height_session = 0
self.block_height_backlog = 0
self.tx_height_backlog = 0

151
apps/cic-eth/cic_eth/db/models/tx.py
View File

@ -1,151 +0,0 @@
# standard imports
import datetime
# third-party imports
from sqlalchemy import Column, String, Integer, DateTime, Enum, ForeignKey, Boolean, NUMERIC
from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
#from sqlalchemy.orm import relationship, backref
#from sqlalchemy.ext.declarative import declarative_base
# local imports
from .base import SessionBase
from .otx import Otx
from cic_eth.db.util import num_serialize
from cic_eth.error import NotLocalTxError
from cic_eth.db.error import TxStateChangeError
class TxCache(SessionBase):
"""Metadata expansions for outgoing transactions.
These records are not essential for handling of outgoing transaction queues. It is implemented to reduce the amount of computation spent of parsing and analysing raw signed transaction data.
Instantiation of the object will fail if an outgoing transaction record with the same transaction hash does not exist.
Typically three types of transactions are recorded:
- Token transfers; where source and destination token values and addresses are identical, sender and recipient differ.
- Token conversions; source and destination token values and addresses differ, sender and recipient are identical.
- Any other transaction; source and destination token addresses are zero-address.
:param tx_hash: Transaction hash
:type tx_hash: str, 0x-hex
:param sender: Ethereum address of transaction sender
:type sender: str, 0x-hex
:param recipient: Ethereum address of transaction beneficiary (e.g. token transfer recipient)
:type recipient: str, 0x-hex
:param source_token_address: Contract address of token that sender spent from
:type source_token_address: str, 0x-hex
:param destination_token_address: Contract address of token that recipient will receive balance of
:type destination_token_address: str, 0x-hex
:param from_value: Amount of source tokens spent
:type from_value: number
:param to_value: Amount of destination tokens received
:type to_value: number
:param block_number: Block height the transaction was mined at, or None if not yet mined
:type block_number: number or None
:param tx_number: Block transaction height the transaction was mined at, or None if not yet mined
:type tx_number: number or None
:raises FileNotFoundError: Outgoing transaction for given transaction hash does not exist
"""
__tablename__ = 'tx_cache'
otx_id = Column(Integer, ForeignKey('otx.id'))
source_token_address = Column(String(42))
destination_token_address = Column(String(42))
sender = Column(String(42))
recipient = Column(String(42))
from_value = Column(NUMERIC())
to_value = Column(NUMERIC())
block_number = Column(Integer())
tx_index = Column(Integer())
date_created = Column(DateTime, default=datetime.datetime.utcnow)
date_updated = Column(DateTime, default=datetime.datetime.utcnow)
date_checked = Column(DateTime, default=datetime.datetime.utcnow)
def check(self):
"""Update the "checked" timestamp to current time.
Only manipulates object, does not transaction or commit to backend.
"""
self.date_checked = datetime.datetime.now()
@staticmethod
def clone(
tx_hash_original,
tx_hash_new,
session=None,
):
"""Copy tx cache data and associate it with a new transaction.
:param tx_hash_original: tx cache data to copy
:type tx_hash_original: str, 0x-hex
:param tx_hash_new: tx hash to associate the copied entry with
:type tx_hash_new: str, 0x-hex
"""
session = SessionBase.bind_session(session)
q = session.query(TxCache)
q = q.join(Otx)
q = q.filter(Otx.tx_hash==tx_hash_original)
txc = q.first()
if txc == None:
SessionBase.release_session(session)
raise NotLocalTxError('original {}'.format(tx_hash_original))
if txc.block_number != None:
SessionBase.release_session(session)
raise TxStateChangeError('cannot clone tx cache of confirmed tx {}'.format(tx_hash_original))
session.flush()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash_new)
otx = q.first()
if otx == None:
SessionBase.release_session(session)
raise NotLocalTxError('new {}'.format(tx_hash_new))
txc_new = TxCache(
otx.tx_hash,
txc.sender,
txc.recipient,
txc.source_token_address,
txc.destination_token_address,
int(txc.from_value),
int(txc.to_value),
session=session,
)
session.add(txc_new)
session.commit()
SessionBase.release_session(session)
def __init__(self, tx_hash, sender, recipient, source_token_address, destination_token_address, from_value, to_value, block_number=None, tx_index=None, session=None):
session = SessionBase.bind_session(session)
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
tx = q.first()
if tx == None:
SessionBase.release_session(session)
raise FileNotFoundError('outgoing transaction record unknown {} (add a Tx first)'.format(tx_hash))
self.otx_id = tx.id
self.sender = sender
self.recipient = recipient
self.source_token_address = source_token_address
self.destination_token_address = destination_token_address
self.from_value = from_value
self.to_value = to_value
self.block_number = block_number
self.tx_index = tx_index
# not automatically set in sqlite, it seems:
self.date_created = datetime.datetime.utcnow()
self.date_updated = self.date_created
self.date_checked = self.date_created
SessionBase.release_session(session)

6
apps/cic-eth/cic_eth/error.py
View File

@ -4,12 +4,6 @@ class TokenCountError(Exception):
pass
class NotLocalTxError(Exception):
"""Exception raised when trying to access a tx not originated from a local task
"""
pass
class PermanentTxError(Exception):
"""Exception raised when encountering a permanent error when sending a tx.

10
apps/cic-eth/cic_eth/eth/account.py
View File

@ -21,6 +21,7 @@ from chainlib.eth.tx import (
from chainlib.chain import ChainSpec
from eth_accounts_index import AccountRegistry
from sarafu_faucet import MinterFaucet as Faucet
from chainqueue.db.models.tx import TxCache
# local import
from cic_eth_registry import CICRegistry
@ -31,7 +32,6 @@ from cic_eth.eth.gas import (
from cic_eth.db.models.nonce import Nonce
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.tx import TxCache
from cic_eth.error import (
RoleMissingError,
SignerError,
@ -125,7 +125,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
account_registry = AccountRegistry(signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
account_registry = AccountRegistry(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
rpc_signer.disconnect()
@ -178,7 +178,7 @@ def gift(self, account_address, chain_spec_dict):
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
faucet = Faucet(signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
rpc_signer.disconnect()
@ -285,7 +285,7 @@ def cache_gift_data(
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
tx = unpack(tx_signed_raw_bytes, chain_spec)
tx_data = Faucet.parse_give_to_request(tx['data'])
session = self.create_session()
@ -328,7 +328,7 @@ def cache_account_data(
"""
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
tx = unpack(tx_signed_raw_bytes, chain_id=chain_spec.chain_id())
tx = unpack(tx_signed_raw_bytes, chain_spec)
tx_data = AccountRegistry.parse_add_request(tx['data'])
session = SessionBase.create_session()

6
apps/cic-eth/cic_eth/eth/bancor.py.bak
View File

@ -209,7 +209,7 @@ def convert_with_default_reserve(self, tokens, from_address, source_amount, mini
# s_queue.apply_async()
#
# s_check_gas = celery.signature(
# 'cic_eth.eth.tx.check_gas',
# 'cic_eth.eth.gas.check_gas',
# [
# c['address'],
# [c['signed_tx']],
@ -222,7 +222,7 @@ def convert_with_default_reserve(self, tokens, from_address, source_amount, mini
# )
#
# s_set_sent = celery.signature(
# 'cic_eth.queue.tx.set_sent_status',
# 'cic_eth.queue.state.set_sent',
# [False],
# )
# s_send.link(s_set_sent)
@ -364,7 +364,7 @@ def otx_cache_convert(
chain_spec = ChainSpec.from_chain_str(chain_str)
tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
tx = unpack(tx_signed_raw_bytes, chain_spec)
tx_data = unpack_convert(tx['data'])
logg.debug('tx data {}'.format(tx_data))

19
apps/cic-eth/cic_eth/eth/erc20.py
View File

@ -14,18 +14,22 @@ from chainlib.eth.tx import (
from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
from hexathon import strip_0x
from chainqueue.db.models.tx import TxCache
from chainqueue.error import NotLocalTxError
# local imports
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole
from cic_eth.error import TokenCountError, PermanentTxError, OutOfGasError, NotLocalTxError
from cic_eth.error import (
TokenCountError,
PermanentTxError,
OutOfGasError,
)
from cic_eth.queue.tx import register_tx
from cic_eth.eth.gas import (
create_check_gas_task,
MaxGasOracle,
)
#from cic_eth.eth.factory import TxFactory
from cic_eth.ext.address import translate_address
from cic_eth.task import (
CriticalSQLAlchemyTask,
@ -90,6 +94,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
:rtype: str, 0x-hex
"""
# we only allow one token, one transfer
logg.debug('tokens {}'.format(tokens))
if len(tokens) != 1:
raise TokenCountError
t = tokens[0]
@ -102,7 +107,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
session = self.create_session()
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
c = ERC20(signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_spec.chain_id())
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
rpc_signer.disconnect()
@ -165,7 +170,7 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
session = self.create_session()
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
c = ERC20(signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_spec.chain_id())
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
rpc_signer.disconnect()
@ -239,7 +244,7 @@ def cache_transfer_data(
"""
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
tx = unpack(tx_signed_raw_bytes, chain_spec)
tx_data = ERC20.parse_transfer_request(tx['data'])
recipient_address = tx_data[0]
@ -280,7 +285,7 @@ def cache_approve_data(
"""
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
tx = unpack(tx_signed_raw_bytes, chain_spec)
tx_data = ERC20.parse_approve_request(tx['data'])
recipient_address = tx_data[0]

443
apps/cic-eth/cic_eth/eth/gas.py
View File

@ -3,87 +3,59 @@ import logging
# external imports
import celery
from chainlib.eth.gas import price
from hexathon import strip_0x
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.chain import ChainSpec
from chainlib.eth.address import is_checksum_address
from chainlib.connection import RPCConnection
from chainqueue.db.enum import StatusBits
from chainlib.eth.gas import (
balance,
price,
)
from chainlib.eth.error import (
NotFoundEthException,
EthException,
)
from chainlib.eth.tx import (
TxFactory,
TxFormat,
unpack,
)
from chainlib.eth.contract import (
abi_decode_single,
ABIContractType,
)
from chainlib.eth.gas import (
Gas,
OverrideGasOracle,
)
from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
# local imports
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.base import SessionBase
from cic_eth.error import (
AlreadyFillingGasError,
OutOfGasError,
)
from cic_eth.eth.nonce import CustodialTaskNonceOracle
from cic_eth.queue.tx import (
queue_create,
register_tx,
)
from cic_eth.queue.query import get_tx
from cic_eth.task import (
CriticalSQLAlchemyTask,
CriticalSQLAlchemyAndWeb3Task,
CriticalSQLAlchemyAndSignerTask,
CriticalWeb3AndSignerTask,
)
celery_app = celery.current_app
logg = logging.getLogger()
#
#class GasOracle():
# """Provides gas pricing for transactions.
#
# :param w3: Web3 object
# :type w3: web3.Web3
# """
#
# __safe_threshold_amount_value = 2000000000 * 60000 * 3
# __refill_amount_value = __safe_threshold_amount_value * 5
# default_gas_limit = 21000
#
# def __init__(self, conn):
# o = price()
# r = conn.do(o)
# b = bytes.from_hex(strip_0x(r))
# self.gas_price_current = int.from_bytes(b, 'big')
#
# #self.w3 = w3
# #self.gas_price_current = w3.eth.gas_price()
#
#
# def safe_threshold_amount(self):
# """The gas balance threshold under which a new gas refill transaction should be initiated.
#
# :returns: Gas token amount
# :rtype: number
# """
# g = GasOracle.__safe_threshold_amount_value
# logg.warning('gas safe threshold is currently hardcoded to {}'.format(g))
# return g
#
#
# def refill_amount(self):
# """The amount of gas tokens to send in a gas refill transaction.
#
# :returns: Gas token amount
# :rtype: number
# """
# g = GasOracle.__refill_amount_value
# logg.warning('gas refill amount is currently hardcoded to {}'.format(g))
# return g
#
#
# def gas_provider(self):
# """Gas provider address.
#
# :returns: Etheerum account address
# :rtype: str, 0x-hex
# """
# session = SessionBase.create_session()
# a = AccountRole.get_address('GAS_GIFTER', session)
# logg.debug('gasgifter {}'.format(a))
# session.close()
# return a
#
#
# def gas_price(self, category='safe'):
# """Get projected gas price to use for a transaction at the current moment.
#
# When the category parameter is implemented, it can be used to control the priority of a transaction in the network.
#
# :param category: Bid level category to return price for. Currently has no effect.
# :type category: str
# :returns: Gas price
# :rtype: number
# """
# #logg.warning('gas price hardcoded to category "safe"')
# #g = 100
# #return g
# return self.gas_price_current
class MaxGasOracle:
@ -114,7 +86,7 @@ def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=No
s_check_gas = None
if tx_hashes_hex != None:
s_check_gas = celery.signature(
'cic_eth.eth.tx.check_gas',
'cic_eth.eth.gas.check_gas',
[
tx_hashes_hex,
chain_spec.asdict(),
@ -126,7 +98,7 @@ def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=No
)
else:
s_check_gas = celery.signature(
'cic_eth.eth.tx.check_gas',
'cic_eth.eth.gas.check_gas',
[
chain_spec.asdict(),
tx_signed_raws_hex,
@ -136,3 +108,324 @@ def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=No
queue=queue,
)
return s_check_gas
@celery_app.task(base=CriticalSQLAlchemyTask)
def cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
chain_spec_dict,
):
"""Helper function for otx_cache_parse_tx
:param tx_hash_hex: Transaction hash
:type tx_hash_hex: str, 0x-hex
:param tx: Signed raw transaction
:type tx: str, 0x-hex
:returns: Transaction hash and id of cache element in storage backend, respectively
:rtype: tuple
"""
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw_bytes, chain_spec)
session = SessionBase.create_session()
tx_cache = TxCache(
tx_hash_hex,
tx['from'],
tx['to'],
ZERO_ADDRESS,
ZERO_ADDRESS,
tx['value'],
tx['value'],
session=session,
)
session.add(tx_cache)
session.commit()
cache_id = tx_cache.id
session.close()
return (tx_hash_hex, cache_id)
@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
"""Check the gas level of the sender address of a transaction.
If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
If account balance is sufficient, but level of gas before spend is below "safe" threshold, gas refill is requested, and execution continues normally.
:param tx_hashes: Transaction hashes due to be submitted
:type tx_hashes: list of str, 0x-hex
:param chain_spec_dict: Chain spec dict representation
:type chain_spec_dict: dict
:param txs: Signed raw transaction data, corresponding to tx_hashes
:type txs: list of str, 0x-hex
:param address: Sender address
:type address: str, 0x-hex
:param gas_required: Gas limit * gas price for transaction, (optional, if not set will be retrived from transaction data)
:type gas_required: int
:return: Signed raw transaction data list
:rtype: param txs, unchanged
"""
if len(txs) == 0:
for i in range(len(tx_hashes)):
o = get_tx(tx_hashes[i])
txs.append(o['signed_tx'])
if address == None:
address = o['address']
#if not web3.Web3.isChecksumAddress(address):
if not is_checksum_address(address):
raise ValueError('invalid address {}'.format(address))
chain_spec = ChainSpec.from_dict(chain_spec_dict)
queue = self.request.delivery_info.get('routing_key')
conn = RPCConnection.connect(chain_spec)
# TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
gas_balance = 0
try:
o = balance(address)
r = conn.do(o)
conn.disconnect()
gas_balance = abi_decode_single(ABIContractType.UINT256, r)
except EthException as e:
conn.disconnect()
raise EthError('gas_balance call for {}: {}'.format(address, e))
logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
session = SessionBase.create_session()
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
session.close()
if gas_required > gas_balance:
s_nonce = celery.signature(
'cic_eth.eth.nonce.reserve_nonce',
[
address,
chain_spec_dict,
gas_provider,
],
queue=queue,
)
s_refill_gas = celery.signature(
'cic_eth.eth.gas.refill_gas',
[
chain_spec_dict,
],
queue=queue,
)
s_nonce.link(s_refill_gas)
s_nonce.apply_async()
wait_tasks = []
for tx_hash in tx_hashes:
s = celery.signature(
'cic_eth.queue.state.set_waitforgas',
[
chain_spec_dict,
tx_hash,
],
queue=queue,
)
wait_tasks.append(s)
celery.group(wait_tasks)()
raise OutOfGasError('need to fill gas, required {}, had {}'.format(gas_required, gas_balance))
safe_gas = self.safe_gas_threshold_amount
if gas_balance < safe_gas:
s_nonce = celery.signature(
'cic_eth.eth.nonce.reserve_nonce',
[
address,
chain_spec_dict,
gas_provider,
],
queue=queue,
)
s_refill_gas = celery.signature(
'cic_eth.eth.gas.refill_gas',
[
chain_spec_dict,
],
queue=queue,
)
s_nonce.link(s_refill_gas)
s_nonce.apply_async()
logg.debug('requested refill from {} to {}'.format(gas_provider, address))
ready_tasks = []
for tx_hash in tx_hashes:
s = celery.signature(
'cic_eth.queue.state.set_ready',
[
chain_spec_dict,
tx_hash,
],
queue=queue,
)
ready_tasks.append(s)
celery.group(ready_tasks)()
return txs
# TODO: if this method fails the nonce will be out of sequence. session needs to be extended to include the queue create, so that nonce is rolled back if the second sql query fails. Better yet, split each state change into separate tasks.
# TODO: method is too long, factor out code for clarity
@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3AndSignerTask)
def refill_gas(self, recipient_address, chain_spec_dict):
"""Executes a native token transaction to fund the recipient's gas expenditures.
:param recipient_address: Recipient in need of gas
:type recipient_address: str, 0x-hex
:param chain_str: Chain spec, string representation
:type chain_str: str
:raises AlreadyFillingGasError: A gas refill transaction for this address is already executing
:returns: Transaction hash.
:rtype: str, 0x-hex
"""
# essentials
chain_spec = ChainSpec.from_dict(chain_spec_dict)
queue = self.request.delivery_info.get('routing_key')
# Determine value of gas tokens to send
# if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
# however, we will perform a 0-value transaction instead
zero_amount = False
session = SessionBase.create_session()
status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
q = session.query(Otx.tx_hash)
q = q.join(TxCache)
q = q.filter(Otx.status.op('&')(StatusBits.FINAL.value)==0)
q = q.filter(TxCache.from_value!=0)
q = q.filter(TxCache.recipient==recipient_address)
c = q.count()
if c > 0:
logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address))))
zero_amount = True
session.flush()
# finally determine the value to send
refill_amount = 0
if not zero_amount:
refill_amount = self.safe_gas_refill_amount
# determine sender
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
session.flush()
# set up evm RPC connection
rpc = RPCConnection.connect(chain_spec, 'default')
# set up transaction builder
nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc)
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
c = Gas(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
# build and add transaction
logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address))
(tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex))
cache_task = 'cic_eth.eth.gas.cache_gas_data'
register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
# add transaction to send queue
s_status = celery.signature(
'cic_eth.queue.state.set_ready',
[
chain_spec.asdict(),
tx_hash_hex,
],
queue=queue,
)
t = s_status.apply_async()
return tx_signed_raw_hex
@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, default_factor=1.1):
"""Create a new transaction from an existing one with same nonce and higher gas price.
:param txold_hash_hex: Transaction to re-create
:type txold_hash_hex: str, 0x-hex
:param chain_str: Chain spec, string representation
:type chain_str: str
:param gas: Explicitly use the specified gas amount
:type gas: number
:param default_factor: Default factor by which to increment the gas price by
:type default_factor: float
:raises NotLocalTxError: Transaction does not exist in the local queue
:returns: Transaction hash
:rtype: str, 0x-hex
"""
session = SessionBase.create_session()
otx = Otx.load(txold_hash_hex, session)
if otx == None:
session.close()
raise NotLocalTxError(txold_hash_hex)
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(otx.signed_tx)
tx = unpack(tx_signed_raw_bytes, chain_spec)
logg.debug('resend otx {} {}'.format(tx, otx.signed_tx))
queue = self.request.delivery_info.get('routing_key')
logg.debug('before {}'.format(tx))
rpc = RPCConnection.connect(chain_spec, 'default')
new_gas_price = gas
if new_gas_price == None:
o = price()
r = rpc.do(o)
current_gas_price = int(r, 16)
if tx['gasPrice'] > current_gas_price:
logg.info('Network gas price {} is lower than overdue tx gas price {}'.format(curent_gas_price, tx['gasPrice']))
#tx['gasPrice'] = int(tx['gasPrice'] * default_factor)
new_gas_price = tx['gasPrice'] + 1
else:
new_gas_price = int(tx['gasPrice'] * default_factor)
#if gas_price > new_gas_price:
# tx['gasPrice'] = gas_price
#else:
# tx['gasPrice'] = new_gas_price
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
gas_oracle = OverrideGasOracle(price=new_gas_price, conn=rpc)
c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
logg.debug('change gas price from old {} to new {} for tx {}'.format(tx['gasPrice'], new_gas_price, tx))
tx['gasPrice'] = new_gas_price
(tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
queue_create(
chain_spec,
tx['nonce'],
tx['from'],
tx_hash_hex,
tx_signed_raw_hex,
session=session,
)
TxCache.clone(txold_hash_hex, tx_hash_hex, session=session)
session.close()
s = create_check_gas_task(
[tx_signed_raw_hex],
chain_spec,
tx['from'],
tx['gasPrice'] * tx['gas'],
[tx_hash_hex],
queue=queue,
)
s.apply_async()
return tx_hash_hex

47
apps/cic-eth/cic_eth/eth/nonce.py
View File

@ -1,9 +1,23 @@
# standard imports
import logging
# external imports
import celery
from chainlib.eth.address import is_checksum_address
# local imports
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.base import SessionBase
from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.db.models.nonce import (
Nonce,
NonceReservation,
)
celery_app = celery.current_app
logg = logging.getLogger()
class CustodialTaskNonceOracle():
"""Ensures atomic nonce increments for all transactions across all tasks and threads.
@ -30,3 +44,36 @@ class CustodialTaskNonceOracle():
"""
r = NonceReservation.release(self.address, self.uuid, session=self.session)
return r[1]
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None):
self.log_banner()
session = SessionBase.create_session()
address = None
if signer_address == None:
address = chained_input
logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
else:
if is_checksum_address(signer_address):
address = signer_address
logg.debug('explicit address for reserve nonce {}'.format(signer_address))
else:
address = AccountRole.get_address(signer_address, session=session)
logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
if not is_checksum_address(address):
raise ValueError('invalid result when resolving address for nonce {}'.format(address))
root_id = self.request.root_id
r = NonceReservation.next(address, root_id, session=session)
logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
session.commit()
session.close()
return chained_input

425
apps/cic-eth/cic_eth/eth/tx.py
View File

@ -1,76 +1,34 @@
# standard imports
import logging
# third-party imports
# external imports
import celery
import requests
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.chain import ChainSpec
from chainlib.eth.address import is_checksum_address
from chainlib.eth.gas import (
balance,
price,
)
from chainlib.eth.error import (
EthException,
NotFoundEthException,
)
from chainlib.eth.error import NotFoundEthException
from chainlib.eth.tx import (
transaction,
receipt,
raw,
TxFormat,
TxFactory,
unpack,
)
from chainlib.connection import RPCConnection
from chainlib.hash import keccak256_hex_to_hex
from chainlib.eth.gas import (
Gas,
OverrideGasOracle,
)
from chainlib.eth.contract import (
abi_decode_single,
ABIContractType,
)
from hexathon import (
add_0x,
strip_0x,
)
from chainqueue.db.models.tx import Otx
from chainqueue.db.models.tx import TxCache
from chainqueue.db.enum import StatusBits
from chainqueue.error import NotLocalTxError
# local imports
from cic_eth.db import (
Otx,
SessionBase,
)
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.nonce import NonceReservation
from cic_eth.db.models.lock import Lock
from cic_eth.db.models.role import AccountRole
from cic_eth.db.enum import (
LockEnum,
StatusBits,
)
from cic_eth.error import PermanentTxError
from cic_eth.error import TemporaryTxError
from cic_eth.error import NotLocalTxError
#from cic_eth.queue.tx import create as queue_create
from cic_eth.queue.tx import (
get_tx,
register_tx,
get_nonce_tx,
create as queue_create,
)
from cic_eth.error import OutOfGasError
from cic_eth.error import LockedError
from cic_eth.eth.gas import (
create_check_gas_task,
)
from cic_eth.eth.nonce import CustodialTaskNonceOracle
from cic_eth.db import SessionBase
from cic_eth.error import (
AlreadyFillingGasError,
EthError,
PermanentTxError,
TemporaryTxError,
)
from cic_eth.eth.gas import create_check_gas_task
from cic_eth.admin.ctrl import lock_send
from cic_eth.task import (
CriticalSQLAlchemyTask,
@ -86,127 +44,6 @@ logg = logging.getLogger()
MAX_NONCE_ATTEMPTS = 3
# TODO this function is too long
@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
"""Check the gas level of the sender address of a transaction.
If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
If account balance is sufficient, but level of gas before spend is below "safe" threshold, gas refill is requested, and execution continues normally.
:param tx_hashes: Transaction hashes due to be submitted
:type tx_hashes: list of str, 0x-hex
:param chain_spec_dict: Chain spec dict representation
:type chain_spec_dict: dict
:param txs: Signed raw transaction data, corresponding to tx_hashes
:type txs: list of str, 0x-hex
:param address: Sender address
:type address: str, 0x-hex
:param gas_required: Gas limit * gas price for transaction, (optional, if not set will be retrived from transaction data)
:type gas_required: int
:return: Signed raw transaction data list
:rtype: param txs, unchanged
"""
if len(txs) == 0:
for i in range(len(tx_hashes)):
o = get_tx(tx_hashes[i])
txs.append(o['signed_tx'])
if address == None:
address = o['address']
#if not web3.Web3.isChecksumAddress(address):
if not is_checksum_address(address):
raise ValueError('invalid address {}'.format(address))
chain_spec = ChainSpec.from_dict(chain_spec_dict)
queue = self.request.delivery_info.get('routing_key')
conn = RPCConnection.connect(chain_spec)
# TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
gas_balance = 0
try:
o = balance(address)
r = conn.do(o)
conn.disconnect()
gas_balance = abi_decode_single(ABIContractType.UINT256, r)
except EthException as e:
conn.disconnect()
raise EthError('gas_balance call for {}: {}'.format(address, e))
logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
session = SessionBase.create_session()
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
session.close()
if gas_required > gas_balance:
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[
address,
gas_provider,
],
queue=queue,
)
s_refill_gas = celery.signature(
'cic_eth.eth.tx.refill_gas',
[
chain_spec_dict,
],
queue=queue,
)
s_nonce.link(s_refill_gas)
s_nonce.apply_async()
wait_tasks = []
for tx_hash in tx_hashes:
s = celery.signature(
'cic_eth.queue.tx.set_waitforgas',
[
tx_hash,
],
queue=queue,
)
wait_tasks.append(s)
celery.group(wait_tasks)()
raise OutOfGasError('need to fill gas, required {}, had {}'.format(gas_required, gas_balance))
safe_gas = self.safe_gas_threshold_amount
if gas_balance < safe_gas:
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
[
address,
gas_provider,
],
queue=queue,
)
s_refill_gas = celery.signature(
'cic_eth.eth.tx.refill_gas',
[
chain_spec_dict,
],
queue=queue,
)
s_nonce.link(s_refill_gas)
s_nonce.apply_async()
logg.debug('requested refill from {} to {}'.format(gas_provider, address))
ready_tasks = []
for tx_hash in tx_hashes:
s = celery.signature(
'cic_eth.queue.tx.set_ready',
[
tx_hash,
],
queue=queue,
)
ready_tasks.append(s)
celery.group(ready_tasks)()
return txs
# TODO: chain chainable transactions that use hashes as inputs may be chained to this function to output signed txs instead.
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
def hashes_to_txs(self, tx_hashes):
@ -268,7 +105,7 @@ def send(self, txs, chain_spec_dict):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_hex = txs[0]
tx_hex = add_0x(txs[0])
tx_hash_hex = add_0x(keccak256_hex_to_hex(tx_hex))
@ -278,8 +115,9 @@ def send(self, txs, chain_spec_dict):
r = None
s_set_sent = celery.signature(
'cic_eth.queue.tx.set_sent_status',
'cic_eth.queue.state.set_sent',
[
chain_spec_dict,
tx_hash_hex,
False
],
@ -296,7 +134,10 @@ def send(self, txs, chain_spec_dict):
if len(tx_tail) > 0:
s = celery.signature(
'cic_eth.eth.tx.send',
[tx_tail],
[
tx_tail,
chain_spec_dict,
],
queue=queue,
)
s.apply_async()
@ -304,196 +145,6 @@ def send(self, txs, chain_spec_dict):
return tx_hash_hex
# TODO: if this method fails the nonce will be out of sequence. session needs to be extended to include the queue create, so that nonce is rolled back if the second sql query fails. Better yet, split each state change into separate tasks.
# TODO: method is too long, factor out code for clarity
@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3AndSignerTask)
def refill_gas(self, recipient_address, chain_spec_dict):
"""Executes a native token transaction to fund the recipient's gas expenditures.
:param recipient_address: Recipient in need of gas
:type recipient_address: str, 0x-hex
:param chain_str: Chain spec, string representation
:type chain_str: str
:raises AlreadyFillingGasError: A gas refill transaction for this address is already executing
:returns: Transaction hash.
:rtype: str, 0x-hex
"""
# essentials
chain_spec = ChainSpec.from_dict(chain_spec_dict)
queue = self.request.delivery_info.get('routing_key')
# Determine value of gas tokens to send
# if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
# however, we will perform a 0-value transaction instead
zero_amount = False
session = SessionBase.create_session()
status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
q = session.query(Otx.tx_hash)
q = q.join(TxCache)
q = q.filter(Otx.status.op('&')(StatusBits.FINAL.value)==0)
q = q.filter(TxCache.from_value!=0)
q = q.filter(TxCache.recipient==recipient_address)
c = q.count()
if c > 0:
logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address))))
zero_amount = True
session.flush()
# finally determine the value to send
refill_amount = 0
if not zero_amount:
refill_amount = self.safe_gas_refill_amount
# determine sender
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
session.flush()
# set up evm RPC connection
rpc = RPCConnection.connect(chain_spec, 'default')
# set up transaction builder
nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc)
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
c = Gas(signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
# build and add transaction
logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address))
(tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex))
cache_task = 'cic_eth.eth.tx.cache_gas_data'
register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
# add transaction to send queue
s_status = celery.signature(
'cic_eth.queue.tx.set_ready',
[
tx_hash_hex,
],
queue=queue,
)
t = s_status.apply_async()
return tx_signed_raw_hex
@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, default_factor=1.1):
"""Create a new transaction from an existing one with same nonce and higher gas price.
:param txold_hash_hex: Transaction to re-create
:type txold_hash_hex: str, 0x-hex
:param chain_str: Chain spec, string representation
:type chain_str: str
:param gas: Explicitly use the specified gas amount
:type gas: number
:param default_factor: Default factor by which to increment the gas price by
:type default_factor: float
:raises NotLocalTxError: Transaction does not exist in the local queue
:returns: Transaction hash
:rtype: str, 0x-hex
"""
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==txold_hash_hex)
otx = q.first()
if otx == None:
session.close()
raise NotLocalTxError(txold_hash_hex)
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(otx.signed_tx[2:])
tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
logg.debug('resend otx {} {}'.format(tx, otx.signed_tx))
queue = self.request.delivery_info.get('routing_key')
logg.debug('before {}'.format(tx))
rpc = RPCConnection.connect(chain_spec, 'default')
new_gas_price = gas
if new_gas_price == None:
o = price()
r = rpc.do(o)
current_gas_price = int(r, 16)
if tx['gasPrice'] > current_gas_price:
logg.info('Network gas price {} is lower than overdue tx gas price {}'.format(curent_gas_price, tx['gasPrice']))
#tx['gasPrice'] = int(tx['gasPrice'] * default_factor)
new_gas_price = tx['gasPrice'] + 1
else:
new_gas_price = int(tx['gasPrice'] * default_factor)
#if gas_price > new_gas_price:
# tx['gasPrice'] = gas_price
#else:
# tx['gasPrice'] = new_gas_price
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
gas_oracle = OverrideGasOracle(price=new_gas_price, conn=rpc)
c = TxFactory(signer=rpc_signer, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
logg.debug('change gas price from old {} to new {} for tx {}'.format(tx['gasPrice'], new_gas_price, tx))
tx['gasPrice'] = new_gas_price
(tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
queue_create(
tx['nonce'],
tx['from'],
tx_hash_hex,
tx_signed_raw_hex,
chain_spec,
session=session,
)
TxCache.clone(txold_hash_hex, tx_hash_hex, session=session)
session.close()
s = create_check_gas_task(
[tx_signed_raw_hex],
chain_spec,
tx['from'],
tx['gasPrice'] * tx['gas'],
[tx_hash_hex],
queue=queue,
)
s.apply_async()
return tx_hash_hex
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
def reserve_nonce(self, chained_input, signer_address=None):
self.log_banner()
session = SessionBase.create_session()
address = None
if signer_address == None:
address = chained_input
logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
else:
#if web3.Web3.isChecksumAddress(signer_address):
if is_checksum_address(signer_address):
address = signer_address
logg.debug('explicit address for reserve nonce {}'.format(signer_address))
else:
address = AccountRole.get_address(signer_address, session=session)
logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
if not is_checksum_address(address):
raise ValueError('invalid result when resolving address for nonce {}'.format(address))
root_id = self.request.root_id
r = NonceReservation.next(address, root_id)
logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
session.commit()
session.close()
return chained_input
@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
def sync_tx(self, tx_hash_hex, chain_spec_dict):
@ -525,7 +176,7 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success))
s = celery.signature(
'cic_eth.queue.tx.set_final_status',
'cic_eth.queue.state.set_final',
[
tx_hash_hex,
rcpt['blockNumber'],
@ -537,7 +188,7 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
logg.debug('sync tx {} mempool'.format(tx_hash_hex))
s = celery.signature(
'cic_eth.queue.tx.set_sent_status',
'cic_eth.queue.state.set_sent',
[
tx_hash_hex,
],
@ -573,7 +224,7 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
#
# tx_signed_raw_hex = r[0]
# tx_signed_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
# tx = unpack_signed_raw_tx(tx_signed_bytes, chain_spec.chain_id())
# tx = unpack(tx_signed_bytes, chain_spec)
#
# queue = self.request.delivery_info['routing_key']
#
@ -589,39 +240,3 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
# return txpending_hash_hex
# TODO: Move to cic_eth.eth.gas
@celery_app.task(base=CriticalSQLAlchemyTask)
def cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
chain_spec_dict,
):
"""Helper function for otx_cache_parse_tx
:param tx_hash_hex: Transaction hash
:type tx_hash_hex: str, 0x-hex
:param tx: Signed raw transaction
:type tx: str, 0x-hex
:returns: Transaction hash and id of cache element in storage backend, respectively
:rtype: tuple
"""
chain_spec = ChainSpec.from_dict(chain_spec_dict)
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
tx_cache = TxCache(
tx_hash_hex,
tx['from'],
tx['to'],
ZERO_ADDRESS,
ZERO_ADDRESS,
tx['value'],
tx['value'],
)
session = SessionBase.create_session()
session.add(tx_cache)
session.commit()
cache_id = tx_cache.id
session.close()
return (tx_hash_hex, cache_id)

16
apps/cic-eth/cic_eth/ext/tx.py
View File

@ -18,13 +18,14 @@ from chainlib.eth.erc20 import ERC20
from hexathon import strip_0x
from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
from chainqueue.db.models.otx import Otx
from chainqueue.db.enum import StatusEnum
from chainqueue.query import get_tx_cache
# local imports
from cic_eth.db.models.otx import Otx
from cic_eth.db.enum import StatusEnum
from cic_eth.queue.tx import get_tx_cache
from cic_eth.queue.time import tx_times
from cic_eth.task import BaseTask
from cic_eth.db.models.base import SessionBase
celery_app = celery.current_app
logg = logging.getLogger()
@ -168,14 +169,16 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
if isinstance(tx_batches, dict):
tx_batches = [tx_batches]
session = SessionBase.create_session()
for b in tx_batches:
for v in b.values():
tx = None
k = None
try:
hx = strip_0x(v)
tx = unpack(bytes.fromhex(hx), chain_spec.chain_id())
txc = get_tx_cache(tx['hash'])
tx = unpack(bytes.fromhex(hx), chain_spec)
txc = get_tx_cache(chain_spec, tx['hash'], session)
txc['timestamp'] = int(txc['date_created'].timestamp())
txc['hash'] = txc['tx_hash']
tx = txc
@ -185,6 +188,8 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
k = '{}.{}.{}'.format(tx['timestamp'], tx['sender'], tx['nonce'])
txs_by_block[k] = tx
session.close()
txs = []
ks = list(txs_by_block.keys())
ks.sort()
@ -192,4 +197,5 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
ks.reverse()
for k in ks:
txs.append(txs_by_block[k])
return txs

2
apps/cic-eth/cic_eth/queue/__init__.py
View File

@ -1,4 +1,2 @@
# third-party imports
import celery
from .tx import get_tx

12
apps/cic-eth/cic_eth/queue/balance.py
View File

@ -5,15 +5,15 @@ import logging
import celery
from chainlib.chain import ChainSpec
from hexathon import strip_0x
# local imports
from cic_eth.db import SessionBase
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.db.enum import (
from chainqueue.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache
from chainqueue.db.enum import (
StatusBits,
dead,
)
# local imports
from cic_eth.db import SessionBase
from cic_eth.task import CriticalSQLAlchemyTask
celery_app = celery.current_app

48
apps/cic-eth/cic_eth/queue/lock.py Normal file
View File

@ -0,0 +1,48 @@
# external imports
from chainqueue.db.models.otx import Otx
import celery
# local imports
from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.db import SessionBase
from cic_eth.db.models.lock import Lock
celery_app = celery.current_app
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_lock(address=None):
"""Retrieve all active locks
If address is set, the query will look up the lock for the specified address only. A list of zero or one elements is returned, depending on whether a lock is set or not.
:param address: Get lock for only the specified address
:type address: str, 0x-hex
:returns: List of locks
:rtype: list of dicts
"""
session = SessionBase.create_session()
q = session.query(
Lock.date_created,
Lock.address,
Lock.flags,
Otx.tx_hash,
)
q = q.join(Otx, isouter=True)
if address != None:
q = q.filter(Lock.address==address)
else:
q = q.order_by(Lock.date_created.asc())
locks = []
for lock in q.all():
o = {
'date': lock[0],
'address': lock[1],
'tx_hash': lock[3],
'flags': lock[2],
}
locks.append(o)
session.close()
return locks

157
apps/cic-eth/cic_eth/queue/query.py Normal file
View File

@ -0,0 +1,157 @@
# standard imports
import datetime
# external imports
import celery
from chainlib.chain import ChainSpec
from chainlib.eth.tx import unpack
import chainqueue.query
from chainqueue.db.enum import (
StatusEnum,
is_alive,
)
from sqlalchemy import func
from sqlalchemy import or_
from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
# local imports
from cic_eth.db.enum import LockEnum
from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.db.models.lock import Lock
from cic_eth.db.models.base import SessionBase
celery_app = celery.current_app
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_tx_cache(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.query.get_tx_cache(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_tx(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.query.get_tx(chain_spec, tx_hash)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
session.close()
return r
def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
return chainqueue.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
return chainqueue.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
def get_nonce_tx(chain_spec, nonce, sender):
return get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack)
def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
"""Returns the next pending transaction, specifically the transaction with the lowest nonce, for every recipient that has pending transactions.
Will omit addresses that have the LockEnum.SEND bit in Lock set.
(TODO) Will not return any rows if LockEnum.SEND bit in Lock is set for zero address.
:param status: Defines the status used to filter as upcoming.
:type status: cic_eth.db.enum.StatusEnum
:param recipient: Ethereum address of recipient to return transaction for
:type recipient: str, 0x-hex
:param before: Only return transactions if their modification date is older than the given timestamp
:type before: datetime.datetime
:param chain_id: Chain id to use to parse signed transaction data
:type chain_id: number
:raises ValueError: Status is finalized, sent or never attempted sent
:returns: Transactions
:rtype: dict, with transaction hash as key, signed raw transaction as value
"""
session = SessionBase.bind_session(session)
q_outer = session.query(
TxCache.sender,
func.min(Otx.nonce).label('nonce'),
)
q_outer = q_outer.join(TxCache)
q_outer = q_outer.join(Lock, isouter=True)
q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
if not is_alive(status):
SessionBase.release_session(session)
raise ValueError('not a valid non-final tx value: {}'.format(status))
if status == StatusEnum.PENDING:
q_outer = q_outer.filter(Otx.status==status.value)
else:
q_outer = q_outer.filter(Otx.status.op('&')(status)==status)
if not_status != None:
q_outer = q_outer.filter(Otx.status.op('&')(not_status)==0)
if recipient != None:
q_outer = q_outer.filter(TxCache.recipient==recipient)
q_outer = q_outer.group_by(TxCache.sender)
txs = {}
i = 0
for r in q_outer.all():
q = session.query(Otx)
q = q.join(TxCache)
q = q.filter(TxCache.sender==r.sender)
q = q.filter(Otx.nonce==r.nonce)
if before != None:
q = q.filter(TxCache.date_checked<before)
q = q.order_by(TxCache.date_created.desc())
o = q.first()
# TODO: audit; should this be possible if a row is found in the initial query? If not, at a minimum log error.
if o == None:
continue
tx_signed_bytes = bytes.fromhex(o.signed_tx)
tx = unpack(tx_signed_bytes, chain_spec)
txs[o.tx_hash] = o.signed_tx
q = session.query(TxCache)
q = q.filter(TxCache.otx_id==o.id)
o = q.first()
o.date_checked = datetime.datetime.now()
session.add(o)
session.commit()
i += 1
if limit > 0 and limit == i:
break
SessionBase.release_session(session)
return txs

100
apps/cic-eth/cic_eth/queue/state.py Normal file
View File

@ -0,0 +1,100 @@
# external imports
from chainlib.chain import ChainSpec
import chainqueue.state
# local imports
import celery
from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.db.models.base import SessionBase
celery_app = celery.current_app
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_sent(chain_spec_dict, tx_hash, fail=False):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_sent(chain_spec, tx_hash, fail, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_final(chain_spec_dict, tx_hash, block=None, fail=False):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_final(chain_spec, tx_hash, block, fail, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_cancel(chain_spec_dict, tx_hash, manual=False):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_cancel(chain_spec, tx_hash, manual, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_rejected(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_rejected(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_fubar(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_fubar(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_manual(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_manual(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_ready(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_ready(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_reserved(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_reserved(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_waitforgas(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.set_waitforgas(chain_spec, tx_hash, session=session)
session.close()
return r
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_state_log(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session()
r = chainqueue.state.get_state_log(chain_spec, tx_hash, session=session)
session.close()
return r

13
apps/cic-eth/cic_eth/queue/time.py
View File

@ -7,10 +7,10 @@ from chainlib.chain import ChainSpec
from chainlib.connection import RPCConnection
from chainlib.eth.block import block_by_hash
from chainlib.eth.tx import receipt
from chainqueue.db.models.otx import Otx
from chainqueue.error import NotLocalTxError
# local imports
from cic_eth.db.models.otx import Otx
from cic_eth.error import NotLocalTxError
from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
celery_app = celery.current_app
@ -18,7 +18,10 @@ celery_app = celery.current_app
logg = logging.getLogger()
def tx_times(tx_hash, chain_spec):
def tx_times(tx_hash, chain_spec, session=None):
session = SessionBase.bind_session(session)
rpc = RPCConnection.connect(chain_spec, 'default')
time_pair = {
'network': None,
@ -35,8 +38,10 @@ def tx_times(tx_hash, chain_spec):
logg.debug('error with getting timestamp details for {}: {}'.format(tx_hash, e))
pass
otx = Otx.load(tx_hash)
otx = Otx.load(tx_hash, session=session)
if otx != None:
time_pair['queue'] = int(otx['date_created'].timestamp())
SessionBase.release_session(session)
return time_pair

766
apps/cic-eth/cic_eth/queue/tx.py
View File

@ -5,92 +5,50 @@ import datetime
# external imports
import celery
from chainqueue.db.models.otx import Otx
from chainqueue.db.models.otx import OtxStateLog
from chainqueue.db.models.tx import TxCache
from hexathon import strip_0x
from sqlalchemy import or_
from sqlalchemy import not_
from sqlalchemy import tuple_
from sqlalchemy import func
from chainlib.chain import ChainSpec
from chainlib.eth.tx import unpack
# local imports
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.otx import OtxStateLog
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.lock import Lock
from cic_eth.db import SessionBase
from cic_eth.db.enum import (
import chainqueue.state
from chainqueue.db.enum import (
StatusEnum,
LockEnum,
StatusBits,
is_alive,
dead,
)
from chainqueue.tx import create
from chainqueue.error import NotLocalTxError
from chainqueue.db.enum import status_str
# local imports
from cic_eth.db.models.lock import Lock
from cic_eth.db import SessionBase
from cic_eth.db.enum import LockEnum
from cic_eth.task import CriticalSQLAlchemyTask
from cic_eth.error import NotLocalTxError
from cic_eth.error import LockedError
from cic_eth.db.enum import status_str
celery_app = celery.current_app
#logg = celery_app.log.get_default_logger()
logg = logging.getLogger()
def create(nonce, holder_address, tx_hash, signed_tx, chain_spec, obsolete_predecessors=True, session=None):
"""Create a new transaction queue record.
:param nonce: Transaction nonce
:type nonce: int
:param holder_address: Sender address
:type holder_address: str, 0x-hex
:param tx_hash: Transaction hash
:type tx_hash: str, 0x-hex
:param signed_tx: Signed raw transaction
:type signed_tx: str, 0x-hex
:param chain_spec: Chain spec to create transaction for
:type chain_spec: ChainSpec
:returns: transaction hash
:rtype: str, 0x-hash
"""
def queue_create(chain_spec, nonce, holder_address, tx_hash, signed_tx, session=None):
session = SessionBase.bind_session(session)
lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session)
if lock > 0:
SessionBase.release_session(session)
raise LockedError(lock)
o = Otx.add(
nonce=nonce,
address=holder_address,
tx_hash=tx_hash,
signed_tx=signed_tx,
session=session,
)
session.flush()
if obsolete_predecessors:
q = session.query(Otx)
q = q.join(TxCache)
q = q.filter(Otx.nonce==nonce)
q = q.filter(TxCache.sender==holder_address)
q = q.filter(Otx.tx_hash!=tx_hash)
q = q.filter(Otx.status.op('&')(StatusBits.FINAL)==0)
for otx in q.all():
logg.info('otx {} obsoleted by {}'.format(otx.tx_hash, tx_hash))
try:
otx.cancel(confirmed=False, session=session)
except TxStateChangeError as e:
logg.exception('obsolete fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('obsolete UNEXPECTED fail: {}'.format(e))
session.close()
raise(e)
session.commit()
tx_hash = create(chain_spec, nonce, holder_address, tx_hash, signed_tx, chain_spec, session=session)
SessionBase.release_session(session)
logg.debug('queue created nonce {} from {} hash {}'.format(nonce, holder_address, tx_hash))
return tx_hash
@ -111,14 +69,14 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No
"""
logg.debug('adding queue tx {}:{} -> {}'.format(chain_spec, tx_hash_hex, tx_signed_raw_hex))
tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx = unpack(tx_signed_raw, chain_id=chain_spec.chain_id())
tx = unpack(tx_signed_raw, chain_spec)
create(
tx_hash = queue_create(
chain_spec,
tx['nonce'],
tx['from'],
tx_hash_hex,
tx_signed_raw_hex,
chain_spec,
session=session,
)
@ -134,683 +92,3 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No
queue=queue,
)
s_cache.apply_async()
return (tx_hash_hex, tx_signed_raw_hex,)
# TODO: Replace set_* with single task for set status
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_sent_status(tx_hash, fail=False):
"""Used to set the status after a send attempt
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:param fail: if True, will set a SENDFAIL status, otherwise a SENT status. (Default: False)
:type fail: boolean
:raises NotLocalTxError: If transaction not found in queue.
:returns: True if tx is known, False otherwise
:rtype: boolean
"""
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
o = q.first()
if o == None:
logg.warning('not local tx, skipping {}'.format(tx_hash))
session.close()
return False
try:
if fail:
o.sendfail(session=session)
else:
o.sent(session=session)
except TxStateChangeError as e:
logg.exception('set sent fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('set sent UNEXPECED fail: {}'.format(e))
session.close()
raise(e)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_final_status(tx_hash, block=None, fail=False):
"""Used to set the status of an incoming transaction result.
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:param block: Block number if final status represents a confirmation on the network
:type block: number
:param fail: if True, will set a SUCCESS status, otherwise a REVERTED status. (Default: False)
:type fail: boolean
:raises NotLocalTxError: If transaction not found in queue.
"""
session = SessionBase.create_session()
q = session.query(
Otx.nonce.label('nonce'),
TxCache.sender.label('sender'),
Otx.id.label('otxid'),
)
q = q.join(TxCache)
q = q.filter(Otx.tx_hash==tx_hash)
o = q.first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
nonce = o.nonce
sender = o.sender
otxid = o.otxid
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
o = q.first()
try:
if fail:
o.minefail(block, session=session)
else:
o.success(block, session=session)
session.commit()
except TxStateChangeError as e:
logg.exception('set final fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('set final UNEXPECED fail: {}'.format(e))
session.close()
raise(e)
q = session.query(Otx)
q = q.join(TxCache)
q = q.filter(Otx.nonce==nonce)
q = q.filter(TxCache.sender==sender)
q = q.filter(Otx.tx_hash!=tx_hash)
for otwo in q.all():
try:
otwo.cancel(True, session=session)
except TxStateChangeError as e:
logg.exception('cancel non-final fail: {}'.format(e))
session.close()
raise(e)
except Exception as e:
logg.exception('cancel non-final UNEXPECTED fail: {}'.format(e))
session.close()
raise(e)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_cancel(tx_hash, manual=False):
"""Used to set the status when a transaction is cancelled.
Will set the state to CANCELLED or OVERRIDDEN
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:param manual: If set, status will be OVERRIDDEN. Otherwise CANCELLED.
:type manual: boolean
:raises NotLocalTxError: If transaction not found in queue.
"""
session = SessionBase.create_session()
o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
try:
if manual:
o.override(session=session)
else:
o.cancel(session=session)
session.commit()
except TxStateChangeError as e:
logg.exception('set cancel fail: {}'.format(e))
except Exception as e:
logg.exception('set cancel UNEXPECTED fail: {}'.format(e))
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_rejected(tx_hash):
"""Used to set the status when the node rejects sending a transaction to network
Will set the state to REJECTED
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:raises NotLocalTxError: If transaction not found in queue.
"""
session = SessionBase.create_session()
o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
o.reject(session=session)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_fubar(tx_hash):
"""Used to set the status when an unexpected error occurs.
Will set the state to FUBAR
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:raises NotLocalTxError: If transaction not found in queue.
"""
session = SessionBase.create_session()
o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
o.fubar(session=session)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_manual(tx_hash):
"""Used to set the status when queue is manually changed
Will set the state to MANUAL
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:raises NotLocalTxError: If transaction not found in queue.
"""
session = SessionBase.create_session()
o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
o.manual(session=session)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_ready(tx_hash):
"""Used to mark a transaction as ready to be sent to network
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:raises NotLocalTxError: If transaction not found in queue.
"""
session = SessionBase.create_session()
o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
if o.status & StatusBits.GAS_ISSUES or o.status == StatusEnum.PENDING:
o.readysend(session=session)
else:
o.retry(session=session)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_dequeue(tx_hash):
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
o = q.first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
o.dequeue(session=session)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def set_waitforgas(tx_hash):
"""Used to set the status when a transaction must be deferred due to gas refill
Will set the state to WAITFORGAS
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:raises NotLocalTxError: If transaction not found in queue.
"""
session = SessionBase.create_session()
o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
if o == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
session.flush()
o.waitforgas(session=session)
session.commit()
session.close()
return tx_hash
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_state_log(tx_hash):
logs = []
session = SessionBase.create_session()
q = session.query(OtxStateLog)
q = q.join(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
q = q.order_by(OtxStateLog.date.asc())
for l in q.all():
logs.append((l.date, l.status,))
session.close()
return logs
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_tx_cache(tx_hash):
"""Returns an aggregate dictionary of outgoing transaction data and metadata
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:raises NotLocalTxError: If transaction not found in queue.
:returns: Transaction data
:rtype: dict
"""
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
otx = q.first()
if otx == None:
session.close()
raise NotLocalTxError(tx_hash)
session.flush()
q = session.query(TxCache)
q = q.filter(TxCache.otx_id==otx.id)
txc = q.first()
session.close()
tx = {
'tx_hash': otx.tx_hash,
'signed_tx': otx.signed_tx,
'nonce': otx.nonce,
'status': status_str(otx.status),
'status_code': otx.status,
'source_token': txc.source_token_address,
'destination_token': txc.destination_token_address,
'block_number': txc.block_number,
'tx_index': txc.tx_index,
'sender': txc.sender,
'recipient': txc.recipient,
'from_value': int(txc.from_value),
'to_value': int(txc.to_value),
'date_created': txc.date_created,
'date_updated': txc.date_updated,
'date_checked': txc.date_checked,
}
return tx
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_lock(address=None):
"""Retrieve all active locks
If address is set, the query will look up the lock for the specified address only. A list of zero or one elements is returned, depending on whether a lock is set or not.
:param address: Get lock for only the specified address
:type address: str, 0x-hex
:returns: List of locks
:rtype: list of dicts
"""
session = SessionBase.create_session()
q = session.query(
Lock.date_created,
Lock.address,
Lock.flags,
Otx.tx_hash,
)
q = q.join(Otx, isouter=True)
if address != None:
q = q.filter(Lock.address==address)
else:
q = q.order_by(Lock.date_created.asc())
locks = []
for lock in q.all():
o = {
'date': lock[0],
'address': lock[1],
'tx_hash': lock[3],
'flags': lock[2],
}
locks.append(o)
session.close()
return locks
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_tx(tx_hash):
"""Retrieve a transaction queue record by transaction hash
:param tx_hash: Transaction hash of record to modify
:type tx_hash: str, 0x-hex
:raises NotLocalTxError: If transaction not found in queue.
:returns: nonce, address and signed_tx (raw signed transaction)
:rtype: dict
"""
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
tx = q.first()
if tx == None:
session.close()
raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
o = {
'otx_id': tx.id,
'nonce': tx.nonce,
'signed_tx': tx.signed_tx,
'status': tx.status,
}
logg.debug('get tx {}'.format(o))
session.close()
return o
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_nonce_tx(nonce, sender, chain_id):
"""Retrieve all transactions for address with specified nonce
:param nonce: Nonce
:type nonce: number
:param address: Ethereum address
:type address: str, 0x-hex
:returns: Transactions
:rtype: dict, with transaction hash as key, signed raw transaction as value
"""
session = SessionBase.create_session()
q = session.query(Otx)
q = q.join(TxCache)
q = q.filter(TxCache.sender==sender)
q = q.filter(Otx.nonce==nonce)
txs = {}
for r in q.all():
tx_signed_bytes = bytes.fromhex(r.signed_tx[2:])
tx = unpack(tx_signed_bytes, chain_id)
if sender == None or tx['from'] == sender:
txs[r.tx_hash] = r.signed_tx
session.close()
return txs
# TODO: pass chain spec instead of chain id
def get_paused_txs(status=None, sender=None, chain_id=0, session=None):
"""Returns not finalized transactions that have been attempted sent without success.
:param status: If set, will return transactions with this local queue status only
:type status: cic_eth.db.enum.StatusEnum
:param recipient: Recipient address to return transactions for
:type recipient: str, 0x-hex
:param chain_id: Numeric chain id to use to parse signed transaction data
:type chain_id: number
:raises ValueError: Status is finalized, sent or never attempted sent
:returns: Transactions
:rtype: dict, with transaction hash as key, signed raw transaction as value
"""
session = SessionBase.bind_session(session)
q = session.query(Otx)
if status != None:
#if status == StatusEnum.PENDING or status >= StatusEnum.SENT:
if status == StatusEnum.PENDING or status & StatusBits.IN_NETWORK or not is_alive(status):
SessionBase.release_session(session)
raise ValueError('not a valid paused tx value: {}'.format(status))
q = q.filter(Otx.status.op('&')(status.value)==status.value)
q = q.join(TxCache)
else:
q = q.filter(Otx.status>StatusEnum.PENDING.value)
q = q.filter(not_(Otx.status.op('&')(StatusBits.IN_NETWORK.value)>0))
if sender != None:
q = q.filter(TxCache.sender==sender)
txs = {}
for r in q.all():
tx_signed_bytes = bytes.fromhex(r.signed_tx[2:])
tx = unpack(tx_signed_bytes, chain_id)
if sender == None or tx['from'] == sender:
#gas += tx['gas'] * tx['gasPrice']
txs[r.tx_hash] = r.signed_tx
SessionBase.release_session(session)
return txs
def get_status_tx(status, not_status=None, before=None, exact=False, limit=0, session=None):
"""Retrieve transaction with a specific queue status.
:param status: Status to match transactions with
:type status: str
:param before: If set, return only transactions older than the timestamp
:type status: datetime.dateTime
:param limit: Limit amount of returned transactions
:type limit: number
:returns: Transactions
:rtype: list of cic_eth.db.models.otx.Otx
"""
txs = {}
session = SessionBase.bind_session(session)
q = session.query(Otx)
q = q.join(TxCache)
# before = datetime.datetime.utcnow()
if before != None:
q = q.filter(TxCache.date_updated<before)
if exact:
q = q.filter(Otx.status==status)
else:
q = q.filter(Otx.status.op('&')(status)>0)
if not_status != None:
q = q.filter(Otx.status.op('&')(not_status)==0)
q = q.order_by(Otx.nonce.asc(), Otx.date_created.asc())
i = 0
for o in q.all():
if limit > 0 and i == limit:
break
txs[o.tx_hash] = o.signed_tx
i += 1
SessionBase.release_session(session)
return txs
# TODO: move query to model
def get_upcoming_tx(status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, chain_id=0, session=None):
"""Returns the next pending transaction, specifically the transaction with the lowest nonce, for every recipient that has pending transactions.
Will omit addresses that have the LockEnum.SEND bit in Lock set.
(TODO) Will not return any rows if LockEnum.SEND bit in Lock is set for zero address.
:param status: Defines the status used to filter as upcoming.
:type status: cic_eth.db.enum.StatusEnum
:param recipient: Ethereum address of recipient to return transaction for
:type recipient: str, 0x-hex
:param before: Only return transactions if their modification date is older than the given timestamp
:type before: datetime.datetime
:param chain_id: Chain id to use to parse signed transaction data
:type chain_id: number
:raises ValueError: Status is finalized, sent or never attempted sent
:returns: Transactions
:rtype: dict, with transaction hash as key, signed raw transaction as value
"""
session = SessionBase.bind_session(session)
q_outer = session.query(
TxCache.sender,
func.min(Otx.nonce).label('nonce'),
)
q_outer = q_outer.join(TxCache)
q_outer = q_outer.join(Lock, isouter=True)
q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
if not is_alive(status):
SessionBase.release_session(session)
raise ValueError('not a valid non-final tx value: {}'.format(status))
if status == StatusEnum.PENDING:
q_outer = q_outer.filter(Otx.status==status.value)
else:
q_outer = q_outer.filter(Otx.status.op('&')(status)==status)
if not_status != None:
q_outer = q_outer.filter(Otx.status.op('&')(not_status)==0)
if recipient != None:
q_outer = q_outer.filter(TxCache.recipient==recipient)
q_outer = q_outer.group_by(TxCache.sender)
txs = {}
i = 0
for r in q_outer.all():
q = session.query(Otx)
q = q.join(TxCache)
q = q.filter(TxCache.sender==r.sender)
q = q.filter(Otx.nonce==r.nonce)
if before != None:
q = q.filter(TxCache.date_checked<before)
q = q.order_by(TxCache.date_created.desc())
o = q.first()
# TODO: audit; should this be possible if a row is found in the initial query? If not, at a minimum log error.
if o == None:
continue
tx_signed_bytes = bytes.fromhex(o.signed_tx[2:])
tx = unpack(tx_signed_bytes, chain_id)
txs[o.tx_hash] = o.signed_tx
q = session.query(TxCache)
q = q.filter(TxCache.otx_id==o.id)
o = q.first()
o.date_checked = datetime.datetime.now()
session.add(o)
session.commit()
i += 1
if limit > 0 and limit == i:
break
SessionBase.release_session(session)
return txs
@celery_app.task(base=CriticalSQLAlchemyTask)
def get_account_tx(address, as_sender=True, as_recipient=True, counterpart=None):
"""Returns all local queue transactions for a given Ethereum address
:param address: Ethereum address
:type address: str, 0x-hex
:param as_sender: If False, will omit transactions where address is sender
:type as_sender: bool
:param as_sender: If False, will omit transactions where address is recipient
:type as_sender: bool
:param counterpart: Only return transactions where this Ethereum address is the other end of the transaction (not in use)
:type counterpart: str, 0x-hex
:raises ValueError: If address is set to be neither sender nor recipient
:returns: Transactions
:rtype: dict, with transaction hash as key, signed raw transaction as value
"""
if not as_sender and not as_recipient:
raise ValueError('at least one of as_sender and as_recipient must be True')
txs = {}
session = SessionBase.create_session()
q = session.query(Otx)
q = q.join(TxCache)
if as_sender and as_recipient:
q = q.filter(or_(TxCache.sender==address, TxCache.recipient==address))
elif as_sender:
q = q.filter(TxCache.sender==address)
else:
q = q.filter(TxCache.recipient==address)
q = q.order_by(Otx.nonce.asc(), Otx.date_created.asc())
results = q.all()
for r in results:
if txs.get(r.tx_hash) != None:
logg.debug('tx {} already recorded'.format(r.tx_hash))
continue
txs[r.tx_hash] = r.signed_tx
session.close()
return txs

4
apps/cic-eth/cic_eth/registry.py
View File

@ -13,7 +13,7 @@ def connect_token_registry(rpc, chain_spec):
registry = CICRegistry(chain_spec, rpc)
token_registry_address = registry.by_name('TokenRegistry')
logg.debug('using token registry address {}'.format(token_registry_address))
lookup = TokenIndexLookup(token_registry_address)
lookup = TokenIndexLookup(chain_spec, token_registry_address)
CICRegistry.add_lookup(lookup)
@ -21,7 +21,7 @@ def connect_declarator(rpc, chain_spec, trusted_addresses):
registry = CICRegistry(chain_spec, rpc)
declarator_address = registry.by_name('AddressDeclarator')
logg.debug('using declarator address {}'.format(declarator_address))
lookup = AddressDeclaratorLookup(declarator_address, trusted_addresses)
lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
CICRegistry.add_lookup(lookup)

23
apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py
View File

@ -17,26 +17,25 @@ from chainlib.eth.tx import unpack
from chainlib.connection import RPCConnection
from chainsyncer.error import SyncDone
from hexathon import strip_0x
from chainqueue.db.enum import (
StatusEnum,
StatusBits,
)
from chainqueue.error import NotLocalTxError
from chainqueue.state import set_reserved
# local imports
import cic_eth
from cic_eth.db import SessionBase
from cic_eth.db.enum import StatusEnum
from cic_eth.db.enum import StatusBits
from cic_eth.db.enum import LockEnum
from cic_eth.db import dsn_from_config
from cic_eth.queue.tx import (
get_upcoming_tx,
set_dequeue,
)
from cic_eth.queue.query import get_upcoming_tx
from cic_eth.admin.ctrl import lock_send
from cic_eth.eth.tx import send as task_tx_send
from cic_eth.error import (
PermanentTxError,
TemporaryTxError,
NotLocalTxError,
)
#from cic_eth.eth.util import unpack_signed_raw_tx_hex
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
@ -92,7 +91,6 @@ class DispatchSyncer:
def __init__(self, chain_spec):
self.chain_spec = chain_spec
self.chain_id = chain_spec.chain_id()
def chain(self):
@ -103,13 +101,14 @@ class DispatchSyncer:
c = len(txs.keys())
logg.debug('processing {} txs {}'.format(c, list(txs.keys())))
chain_str = str(self.chain_spec)
session = SessionBase.create_session()
for k in txs.keys():
tx_raw = txs[k]
tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
tx = unpack(tx_raw_bytes, self.chain_spec.chain_id())
tx = unpack(tx_raw_bytes, self.chain_spec)
try:
set_dequeue(tx['hash'])
set_reserved(self.chain_spec, tx['hash'], session=session)
except NotLocalTxError as e:
logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash']))
continue
@ -140,7 +139,7 @@ class DispatchSyncer:
while run:
txs = {}
typ = StatusBits.QUEUED
utxs = get_upcoming_tx(typ, chain_id=self.chain_id)
utxs = get_upcoming_tx(self.chain_spec, typ)
for k in utxs.keys():
txs[k] = utxs[k]
self.process(w3, txs)

20
apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py
View File

@ -2,14 +2,18 @@
import logging
# external imports
from hexathon import add_0x
from hexathon import (
add_0x,
strip_0x,
)
from chainlib.eth.tx import unpack
from chainqueue.db.enum import StatusBits
from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
from chainqueue.query import get_paused_tx_cache as get_paused_tx
# local imports
from cic_eth.db.enum import StatusBits
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.otx import Otx
from cic_eth.queue.tx import get_paused_txs
from cic_eth.eth.gas import create_check_gas_task
from .base import SyncFilter
@ -24,13 +28,13 @@ class GasFilter(SyncFilter):
def filter(self, conn, block, tx, session):
tx_hash_hex = add_0x(tx.hash)
if tx.value > 0:
tx_hash_hex = add_0x(tx.hash)
logg.debug('gas refill tx {}'.format(tx_hash_hex))
session = SessionBase.bind_session(session)
q = session.query(TxCache.recipient)
q = q.join(Otx)
q = q.filter(Otx.tx_hash==tx_hash_hex)
q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex))
r = q.first()
if r == None:
@ -38,7 +42,7 @@ class GasFilter(SyncFilter):
SessionBase.release_session(session)
return
txs = get_paused_txs(StatusBits.GAS_ISSUES, r[0], self.chain_spec.chain_id(), session=session)
txs = get_paused_tx(self.chain_spec, status=StatusBits.GAS_ISSUES, sender=r[0], session=session, decoder=unpack)
SessionBase.release_session(session)

3
apps/cic-eth/cic_eth/runnable/daemons/filters/register.py
View File

@ -35,9 +35,10 @@ class RegistrationFilter(SyncFilter):
address = to_checksum_address(add_0x(address_hex))
logg.info('request token gift to {}'.format(address))
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
address,
self.chain_spec.asdict(),
],
queue=self.queue,
)

4
apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py
View File

@ -61,7 +61,7 @@ class TransferAuthFilter(SyncFilter):
}
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
[token_data],
sender,
@ -69,7 +69,7 @@ class TransferAuthFilter(SyncFilter):
queue=self.queue,
)
s_approve = celery.signature(
'cic_eth.eth.token.approve',
'cic_eth.eth.erc20.approve',
[
sender,
recipient,

11
apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py
View File

@ -1,16 +1,16 @@
# standard imports
import logging
# third-party imports
# external imports
import celery
from hexathon import (
add_0x,
)
from chainsyncer.db.models.base import SessionBase
from chainqueue.db.models.otx import Otx
from chainlib.status import Status
# local imports
from cic_eth.db.models.otx import Otx
from chainsyncer.db.models.base import SessionBase
from chainlib.status import Status
from .base import SyncFilter
logg = logging.getLogger().getChild(__name__)
@ -34,8 +34,9 @@ class TxFilter(SyncFilter):
db_session.flush()
SessionBase.release_session(db_session)
s = celery.signature(
'cic_eth.queue.tx.set_final_status',
'cic_eth.queue.state.set_final',
[
self.chain_spec.asdict(),
add_0x(tx_hash_hex),
tx.block.number,
tx.status == Status.ERROR,

12
apps/cic-eth/cic_eth/runnable/daemons/retry.py
View File

@ -92,7 +92,7 @@ straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
# TODO: we already have the signed raw tx in get, so its a waste of cycles to get_tx here
def sendfail_filter(w3, tx_hash, rcpt, chain_spec):
tx_dict = get_tx(tx_hash)
tx = unpack_signed_raw_tx_hex(tx_dict['signed_tx'], chain_spec.chain_id())
tx = unpack(tx_dict['signed_tx'], chain_spec)
logg.debug('submitting tx {} for retry'.format(tx_hash))
s_check = celery.signature(
'cic_eth.admin.ctrl.check_lock',
@ -113,12 +113,12 @@ def sendfail_filter(w3, tx_hash, rcpt, chain_spec):
# )
# s_retry_status = celery.signature(
# 'cic_eth.queue.tx.set_ready',
# 'cic_eth.queue.state.set_ready',
# [],
# queue=queue,
# )
s_resend = celery.signature(
'cic_eth.eth.tx.resend_with_higher_gas',
'cic_eth.eth.gas.resend_with_higher_gas',
[
chain_str,
],
@ -143,7 +143,7 @@ def dispatch(conn, chain_spec):
for k in txs.keys():
#tx_cache = get_tx_cache(k)
tx_raw = txs[k]
tx = unpack_signed_raw_tx_hex(tx_raw, chain_spec.chain_id())
tx = unpack(tx_raw, chain_spec)
s_check = celery.signature(
'cic_eth.admin.ctrl.check_lock',
@ -184,7 +184,7 @@ def dispatch(conn, chain_spec):
# txs = list(txs.keys())
# logg.debug('straggler txs {} chain {}'.format(signed_txs, chain_str))
# s_send = celery.signature(
# 'cic_eth.eth.resend_with_higher_gas',
# 'cic_eth.eth.gas.resend_with_higher_gas',
# [
# txs,
# chain_str,
@ -204,7 +204,7 @@ class StragglerFilter:
def filter(self, conn, block, tx, db_session=None):
logg.debug('tx {}'.format(tx))
s_send = celery.signature(
'cic_eth.eth.tx.resend_with_higher_gas',
'cic_eth.eth.gas.resend_with_higher_gas',
[
tx,
self.chain_spec.asdict(),

43
apps/cic-eth/cic_eth/runnable/daemons/tasker.py
View File

@ -14,21 +14,35 @@ import confini
from chainlib.connection import RPCConnection
from chainlib.eth.connection import EthUnixSignerConnection
from chainlib.chain import ChainSpec
from chainqueue.db.models.otx import Otx
# local imports
from cic_eth.eth import erc20
from cic_eth.eth import tx
from cic_eth.eth import account
from cic_eth.admin import debug
from cic_eth.admin import ctrl
from cic_eth.queue import tx
from cic_eth.queue import balance
from cic_eth.callbacks import Callback
from cic_eth.callbacks import http
from cic_eth.callbacks import tcp
from cic_eth.callbacks import redis
from cic_eth.eth import (
erc20,
tx,
account,
nonce,
gas,
)
from cic_eth.admin import (
debug,
ctrl,
)
from cic_eth.queue import (
query,
balance,
state,
tx,
lock,
time,
)
from cic_eth.callbacks import (
Callback,
http,
#tcp,
redis,
)
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.otx import Otx
from cic_eth.db import dsn_from_config
from cic_eth.ext import tx
from cic_eth.registry import (
@ -162,5 +176,10 @@ def main():
current_app.worker_main(argv)
@celery.signals.eventlet_pool_postshutdown.connect
def shutdown(sender=None, headers=None, body=None, **kwargs):
logg.warning('in shudown event hook')
if __name__ == '__main__':
main()

6
apps/cic-eth/cic_eth/version.py
View File

@ -8,9 +8,9 @@ import semver
version = (
0,
10,
1,
'beta.2',
11,
0,
'alpha.3',
)
version_object = semver.VersionInfo(

2
apps/cic-eth/docker/Dockerfile
View File

@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
# python merge_requirements.py | tee merged_requirements.txt
#RUN cd cic-base && \
# pip install $pip_extra_index_url_flag -r ./merged_requirements.txt
RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a44
RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a57
COPY cic-eth/scripts/ scripts/
COPY cic-eth/setup.cfg cic-eth/setup.py ./

19
apps/cic-eth/requirements.txt
View File

@ -1,24 +1,25 @@
cic-base~=0.1.2a53
cic-base~=0.1.2a57
celery==4.4.7
crypto-dev-signer~=0.4.14a16
crypto-dev-signer~=0.4.14a17
confini~=0.3.6rc3
cic-eth-registry~=0.5.4a9
cic-eth-registry~=0.5.4a11
#cic-bancor~=0.0.6
redis==3.5.3
alembic==1.4.2
websockets==8.1
requests~=2.24.0
eth_accounts_index~=0.0.11a6
erc20-transfer-authorization~=0.3.1a2
eth_accounts_index~=0.0.11a7
erc20-transfer-authorization~=0.3.1a3
#simple-rlp==0.1.2
uWSGI==2.0.19.1
semver==2.13.0
websocket-client==0.57.0
moolb~=0.1.1b2
eth-address-index~=0.1.1a5
chainlib~=0.0.1a43
eth-address-index~=0.1.1a7
chainlib~=0.0.2a1
hexathon~=0.0.1a7
chainsyncer~=0.0.1a20
chainsyncer~=0.0.1a21
chainqueue~=0.0.1a3
pysha3==1.0.2
coincurve==15.0.0
sarafu-faucet~=0.0.2a15
sarafu-faucet~=0.0.2a16

3
apps/cic-eth/tests/fixtures_celery.py
View File

@ -27,6 +27,9 @@ def celery_includes():
'cic_eth.eth.tx',
'cic_eth.ext.tx',
'cic_eth.queue.tx',
'cic_eth.queue.lock',
'cic_eth.queue.query',
'cic_eth.queue.state',
'cic_eth.queue.balance',
'cic_eth.admin.ctrl',
'cic_eth.admin.nonce',

46
apps/cic-eth/tests/task/api/test_admin.py
View File

@ -16,27 +16,22 @@ from hexathon import (
strip_0x,
add_0x,
)
from chainqueue.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache
from chainqueue.db.enum import (
StatusEnum,
StatusBits,
status_str,
)
from chainqueue.query import get_tx
# local imports
from cic_eth.api import AdminApi
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
status_str,
LockEnum,
)
from cic_eth.db.enum import LockEnum
from cic_eth.error import InitializationError
from cic_eth.eth.tx import (
cache_gas_data,
)
#from cic_eth.eth.gas import cache_gas_tx
from cic_eth.queue.tx import (
create as queue_create,
get_tx,
)
from cic_eth.eth.gas import cache_gas_data
from cic_eth.queue.tx import queue_create
logg = logging.getLogger()
@ -56,7 +51,7 @@ logg = logging.getLogger()
# gas_provider = c.gas_provider()
#
# s_nonce = celery.signature(
# 'cic_eth.eth.tx.reserve_nonce',
# 'cic_eth.eth.nonce.reserve_nonce',
# [
# init_w3.eth.accounts[0],
# gas_provider,
@ -64,7 +59,7 @@ logg = logging.getLogger()
# queue=None,
# )
# s_refill = celery.signature(
# 'cic_eth.eth.tx.refill_gas',
# 'cic_eth.eth.gas.refill_gas',
# [
# chain_str,
# ],
@ -83,7 +78,7 @@ logg = logging.getLogger()
# o = q.first()
# tx_raw = o.signed_tx
#
# tx_dict = unpack_signed_raw_tx(bytes.fromhex(tx_raw[2:]), default_chain_spec.chain_id())
# tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec)
# gas_price_before = tx_dict['gasPrice']
#
# s = celery.signature(
@ -109,7 +104,7 @@ logg = logging.getLogger()
#
# tx_raw_new = get_tx(tx_hash_new_hex)
# logg.debug('get {}'.format(tx_raw_new))
# tx_dict_new = unpack_signed_raw_tx(bytes.fromhex(tx_raw_new['signed_tx'][2:]), default_chain_spec.chain_id())
# tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec)
# assert tx_hash_new_hex != tx_dict['hash']
# assert tx_dict_new['gasPrice'] > gas_price_before
#
@ -133,7 +128,7 @@ logg = logging.getLogger()
# sigs = []
# for i in range(5):
# s = celery.signature(
# 'cic_eth.eth.tx.refill_gas',
# 'cic_eth.eth.gas.refill_gas',
# [
# eth_empty_accounts[i],
# chain_str,
@ -278,15 +273,14 @@ def test_tx(
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
celery_worker,
):
chain_id = default_chain_spec.chain_id()
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
queue_create(tx['nonce'], agent_roles['ALICE'], tx_hash_hex, tx_signed_raw_hex, default_chain_spec, session=init_database)
tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
queue_create(default_chain_spec, tx['nonce'], agent_roles['ALICE'], tx_hash_hex, tx_signed_raw_hex)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])

25
apps/cic-eth/tests/task/api/test_list.py
View File

@ -1,23 +1,29 @@
# standard imports
import logging
# local imports
# external imports
import pytest
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.erc20 import ERC20
from chainlib.eth.tx import receipt
# local imports
from cic_eth.api.api_task import Api
from tests.mock.filter import (
block_filter,
tx_filter,
)
from cic_eth.db.models.nonce import (
Nonce,
NonceReservation,
)
# test imports
from tests.mock.filter import (
block_filter,
tx_filter,
)
logg = logging.getLogger()
@pytest.mark.xfail()
def test_list_tx(
default_chain_spec,
init_database,
@ -29,11 +35,9 @@ def test_list_tx(
foo_token,
register_tokens,
init_eth_tester,
celery_session_worker,
celery_worker,
):
chain_id = default_chain_spec.chain_id()
tx_hashes = []
# external tx
@ -53,7 +57,7 @@ def test_list_tx(
init_database.commit()
init_eth_tester.mine_blocks(13)
c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id)
c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
(tx_hash_hex, o) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024)
eth_rpc.do(o)
o = receipt(tx_hash_hex)
@ -73,7 +77,7 @@ def test_list_tx(
init_eth_tester.mine_blocks(13)
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id)
c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
(tx_hash_hex, o) = c.transfer(foo_token, agent_roles['ALICE'], agent_roles['BOB'], 256)
eth_rpc.do(o)
o = receipt(tx_hash_hex)
@ -110,7 +114,6 @@ def test_list_tx(
r = t.get_leaf()
assert t.successful()
assert len(r) == 3
logg.debug('rrrr {}'.format(r))

13
apps/cic-eth/tests/task/test_task_account.py
View File

@ -11,13 +11,12 @@ from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import receipt
from eth_accounts_index import AccountRegistry
from hexathon import strip_0x
from chainqueue.db.enum import StatusEnum
from chainqueue.db.models.otx import Otx
# local imports
from cic_eth.error import OutOfGasError
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.base import SessionBase
from cic_eth.db.enum import StatusEnum
from cic_eth.db.enum import StatusEnum
from cic_eth.db.models.nonce import Nonce
from cic_eth.db.models.role import AccountRole
@ -74,9 +73,10 @@ def test_register_account(
):
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
eth_empty_accounts[0],
default_chain_spec.asdict(),
custodial_roles['ACCOUNT_REGISTRY_WRITER'],
],
queue=None,
@ -160,7 +160,7 @@ def test_gift(
):
nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], eth_rpc)
c = AccountRegistry(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id())
c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
(tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
eth_rpc.do(o)
o = receipt(tx_hash_hex)
@ -168,9 +168,10 @@ def test_gift(
assert r['status'] == 1
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
agent_roles['ALICE'],
default_chain_spec.asdict(),
],
queue=None,
)

10
apps/cic-eth/tests/task/test_task_erc20.py
View File

@ -28,7 +28,7 @@ def test_otx_cache_transfer(
celery_session_worker,
):
nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id())
c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
transfer_value = 100 * (10**6)
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value, tx_format=TxFormat.RLP_SIGNED)
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
@ -59,7 +59,7 @@ def test_erc20_balance_task(
):
nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
c = ERC20(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id())
c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
transfer_value = 100 * (10**6)
(tx_hash_hex, o) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
eth_rpc.do(o)
@ -102,9 +102,10 @@ def test_erc20_transfer_task(
transfer_value = 100 * (10 ** 6)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
[token_object],
default_chain_spec.asdict(),
custodial_roles['FOO_TOKEN_GIFTER'],
],
queue=None,
@ -143,9 +144,10 @@ def test_erc20_approve_task(
transfer_value = 100 * (10 ** 6)
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
'cic_eth.eth.nonce.reserve_nonce',
[
[token_object],
default_chain_spec.asdict(),
custodial_roles['FOO_TOKEN_GIFTER'],
],
queue=None,

26
apps/cic-eth/tests/task/test_task_tx.py
View File

@ -13,16 +13,15 @@ from chainlib.eth.tx import (
receipt,
)
from hexathon import strip_0x
from chainqueue.db.models.otx import Otx
# local imports
from cic_eth.queue.tx import register_tx
from cic_eth.eth.tx import cache_gas_data
from cic_eth.db.models.otx import Otx
from cic_eth.eth.gas import cache_gas_data
logg = logging.getLogger()
@pytest.mark.skip()
def test_tx_send(
init_database,
default_chain_spec,
@ -33,11 +32,9 @@ def test_tx_send(
celery_session_worker,
):
chain_id = default_chain_spec.chain_id()
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
#unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
@ -62,12 +59,11 @@ def test_tx_send(
assert rcpt['status'] == 1
@pytest.mark.skip()
def test_sync_tx(
default_chain_spec,
eth_rpc,
eth_signer,
celery_worker,
celery_session_worker,
):
pass
@ -78,20 +74,18 @@ def test_resend_with_higher_gas(
eth_rpc,
eth_signer,
agent_roles,
celery_worker,
celery_session_worker,
):
chain_id = default_chain_spec.chain_id()
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
#unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec.chain_id())
tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
s = celery.signature(
'cic_eth.eth.tx.resend_with_higher_gas',
'cic_eth.eth.gas.resend_with_higher_gas',
[
tx_hash_hex,
default_chain_spec.asdict(),
@ -102,12 +96,12 @@ def test_resend_with_higher_gas(
r = t.get_leaf()
q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==r)
q = q.filter(Otx.tx_hash==strip_0x(r))
otx = q.first()
if otx == None:
raise NotLocalTxError(r)
tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec.chain_id())
tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
assert tx_after['gasPrice'] > tx_before['gasPrice']

29
apps/cic-eth/tests/unit/db/test_block_sync.py
View File

@ -1,29 +0,0 @@
# standard imports
import logging
# local imports
from cic_eth.db.models.otx import OtxSync
logg = logging.getLogger()
def test_db_block_sync(
init_database,
):
s = OtxSync('eip155:8995:bloxberg')
s.head(666, 12)
assert s.head() == (666, 12)
s.session(42, 13)
assert s.session() == (42, 13)
s.backlog(13, 2)
assert s.backlog() == (13, 2)
assert not s.synced
s.backlog(42, 13)
assert s.backlog() == (42, 13)
assert s.synced

2
apps/cic-eth/tests/unit/db/test_db_convert_transfer.py
View File

@ -3,7 +3,7 @@ import logging
import pytest
from cic_eth.db import TxConvertTransfer
from cic_eth.db.models.convert import TxConvertTransfer
from cic_eth.db.error import UnknownConvertError
logg = logging.getLogger()

107
apps/cic-eth/tests/unit/db/test_otx.py
View File

@ -1,107 +0,0 @@
# standard imports
import os
import logging
# third-party imports
import pytest
# local imports
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.otx import OtxStateLog
from cic_eth.db.models.otx import Otx
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
is_alive,
)
logg = logging.getLogger()
#def test_get(
# rpc_eth,
# rpc_signer,
# agent_roles,
# init_database,
# ):
#
# rpc = RPCConnection.connect(default_chain_spec, 'default')
# nonce_oracle = RPCNonceOracle(agent_roles['ALICE'])
# gas_oracle = RPCGasOracle(eth_rpc)
# c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
#
# for i in range(10):
#
# (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)),
#
# tx_def = {
# 'from': init_w3.eth.accounts[0],
# 'to': init_w3.eth.accounts[1],
# 'nonce': 0,
# 'value': 101,
# 'gasPrice': 2000000000,
# 'gas': 21000,
# 'data': '',
# 'chainId': 1,
# }
#
# session = init_database
# txs = []
# for i in range(10):
# nonce = init_w3.eth.getTransactionCount(init_w3.eth.accounts[0], 'pending')
# tx_def['nonce'] = nonce
# tx = init_w3.eth.sign_transaction(tx_def)
# tx_hash = init_w3.eth.send_raw_transaction(tx['raw'])
# logg.debug('tx {}'.format(tx))
#
# address = init_w3.eth.accounts[i%3]
# otx = Otx(int((i/3)+1), address, '0x'+tx_hash.hex(), tx['raw'])
# txs.append(otx)
# session.add(otx)
# session.flush()
#
# logg.debug(txs)
# session.commit()
#
# txs[0].status = 0
# session.add(txs[0])
# session.commit()
# session.close()
#
# get_txs = Otx.get()
# logg.debug(get_txs)
def test_state_log(
init_database,
):
Otx.tracing = True
address = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex()
otx = Otx.add(0, address, tx_hash, signed_tx, session=init_database)
otx.waitforgas(session=init_database)
init_database.commit()
otx.readysend(session=init_database)
init_database.commit()
otx.sent(session=init_database)
init_database.commit()
otx.success(1024, session=init_database)
init_database.commit()
q = init_database.query(OtxStateLog)
q = q.filter(OtxStateLog.otx_id==otx.id)
q = q.order_by(OtxStateLog.date.asc())
logs = q.all()
assert logs[0].status == StatusEnum.PENDING
assert logs[1].status == StatusEnum.WAITFORGAS
assert logs[2].status & StatusBits.QUEUED
assert logs[3].status & StatusBits.IN_NETWORK
assert not is_alive(logs[4].status)

97
apps/cic-eth/tests/unit/db/test_status.py
View File

@ -1,97 +0,0 @@
# standard imports
import os
# third-party imports
import pytest
# local imports
from cic_eth.db.models.otx import Otx
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
is_alive,
)
@pytest.fixture(scope='function')
def otx(
init_database,
):
bogus_hash = '0x' + os.urandom(32).hex()
bogus_address = '0x' + os.urandom(20).hex()
bogus_tx_raw = '0x' + os.urandom(128).hex()
return Otx(0, bogus_address, bogus_hash, bogus_tx_raw)
def test_status_chain_gas(
init_database,
otx,
):
otx.waitforgas(init_database)
otx.readysend(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_straight_success(
init_database,
otx,
):
otx.readysend(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_straight_revert(
init_database,
otx,
):
otx.readysend(init_database)
otx.sent(init_database)
otx.minefail(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_nodeerror(
init_database,
otx,
):
otx.readysend(init_database)
otx.sendfail(init_database)
otx.retry(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_nodeerror_multiple(
init_database,
otx,
):
otx.readysend(init_database)
otx.sendfail(init_database)
otx.retry(init_database)
otx.sendfail(init_database)
otx.retry(init_database)
otx.sent(init_database)
otx.success(1024, init_database)
assert not is_alive(otx.status)
def test_status_chain_nodeerror(
init_database,
otx,
):
otx.readysend(init_database)
otx.reject(init_database)
assert not is_alive(otx.status)

26
apps/cic-eth/tests/unit/db/test_tx.py
View File

@ -18,10 +18,8 @@ from hexathon import (
add_0x,
strip_0x,
)
# local imports
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
# test imports
from tests.util.gas import StaticGasOracle
@ -35,18 +33,16 @@ def test_set(
agent_roles,
):
chain_id = default_chain_spec.chain_id()
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_id)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
otx = Otx(
tx['nonce'],
tx['from'],
tx_hash_hex,
tx_signed_raw_hex,
)
@ -66,6 +62,7 @@ def test_set(
to_value,
666,
13,
session=init_database,
)
init_database.add(txc)
init_database.commit()
@ -89,18 +86,17 @@ def test_clone(
agent_roles,
):
chain_id = default_chain_spec.chain_id()
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
gas_oracle = StaticGasOracle(2 * (10 ** 9), 21000)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_id)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
txs_rpc = [
c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED),
]
gas_oracle = StaticGasOracle(4 * (10 ** 9), 21000)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_id)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
txs_rpc += [
c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED),
]
@ -109,10 +105,9 @@ def test_clone(
for tx_rpc in txs_rpc:
tx_hash_hex = tx_rpc[0]
tx_signed_raw_hex = tx_rpc[1]
tx_dict = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
tx_dict = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
otx = Otx(
tx_dict['nonce'],
tx_dict['from'],
tx_hash_hex,
tx_signed_raw_hex,
)
@ -130,15 +125,16 @@ def test_clone(
ZERO_ADDRESS,
txs[0]['value'],
txs[0]['value'],
session=init_database,
)
init_database.add(txc)
init_database.commit()
TxCache.clone(txs[0]['hash'], txs[1]['hash'])
TxCache.clone(txs[0]['hash'], txs[1]['hash'], session=init_database)
q = init_database.query(TxCache)
q = q.join(Otx)
q = q.filter(Otx.tx_hash==txs[1]['hash'])
q = q.filter(Otx.tx_hash==strip_0x(txs[1]['hash']))
txc_clone = q.first()
assert txc_clone != None

5
apps/cic-eth/tests/unit/eth/test_raw.py
View File

@ -18,13 +18,12 @@ def test_unpack(
agent_roles,
):
chain_id = default_chain_spec.chain_id()
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id=chain_id)
tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
assert tx_hash_hex == tx['hash']

2
apps/cic-eth/tests/unit/ext/test_address.py
View File

@ -23,7 +23,7 @@ def test_translate(
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
c = AddressDeclarator(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=default_chain_spec.chain_id())
c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
description = 'alice'.encode('utf-8').ljust(32, b'\x00').hex()
(tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], add_0x(description))

36
apps/cic-eth/tests/unit/queue/test_balances.py
View File

@ -2,12 +2,12 @@
import os
import logging
# third-party imports
# external imports
import pytest
from chainqueue.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache
# local imports
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.tx import TxCache
from cic_eth.queue.balance import (
balance_outgoing,
balance_incoming,
@ -59,17 +59,15 @@ def test_assemble():
assert r[1].get('balance_xyzzy') != None
@pytest.mark.skip()
def test_outgoing_balance(
default_chain_spec,
init_database,
):
chain_str = str(default_chain_spec)
recipient = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex()
otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database)
otx = Otx.add(0, tx_hash, signed_tx, session=init_database)
init_database.add(otx)
init_database.commit()
@ -83,6 +81,7 @@ def test_outgoing_balance(
token_address,
1000,
1000,
session=init_database,
)
init_database.add(txc)
init_database.commit()
@ -91,33 +90,35 @@ def test_outgoing_balance(
'address': token_address,
'converters': [],
}
b = balance_outgoing([token_data], sender, chain_str)
b = balance_outgoing([token_data], sender, default_chain_spec.asdict())
assert b[0]['balance_outgoing'] == 1000
otx.readysend(session=init_database)
init_database.flush()
otx.reserve(session=init_database)
init_database.flush()
otx.sent(session=init_database)
init_database.commit()
b = balance_outgoing([token_data], sender, chain_str)
b = balance_outgoing([token_data], sender, default_chain_spec.asdict())
assert b[0]['balance_outgoing'] == 1000
otx.success(block=1024, session=init_database)
init_database.commit()
b = balance_outgoing([token_data], sender, chain_str)
b = balance_outgoing([token_data], sender, default_chain_spec.asdict())
assert b[0]['balance_outgoing'] == 0
@pytest.mark.skip()
def test_incoming_balance(
default_chain_spec,
init_database,
):
chain_str = str(default_chain_spec)
recipient = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex()
otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database)
otx = Otx.add(0, tx_hash, signed_tx, session=init_database)
init_database.add(otx)
init_database.commit()
@ -131,6 +132,7 @@ def test_incoming_balance(
token_address,
1000,
1000,
session=init_database,
)
init_database.add(txc)
init_database.commit()
@ -139,19 +141,23 @@ def test_incoming_balance(
'address': token_address,
'converters': [],
}
b = balance_incoming([token_data], recipient, chain_str)
b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 0
otx.readysend(session=init_database)
init_database.flush()
otx.reserve(session=init_database)
init_database.flush()
otx.sent(session=init_database)
init_database.commit()
b = balance_incoming([token_data], recipient, chain_str)
b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 1000
otx.success(block=1024, session=init_database)
init_database.commit()
b = balance_incoming([token_data], recipient, chain_str)
b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 0

76
apps/cic-eth/tests/unit/queue/test_list_tx.py
View File

@ -1,76 +0,0 @@
# standard imports
import logging
# external imports
from chainlib.connection import RPCConnection
from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import Gas
# local imports
from cic_eth.queue.tx import get_status_tx
from cic_eth.db.enum import (
StatusEnum,
StatusBits,
)
from cic_eth.queue.tx import create as queue_create
from cic_eth.eth.tx import cache_gas_data
from cic_eth.queue.tx import register_tx
from cic_eth.db.models.otx import Otx
logg = logging.getLogger()
def test_status_tx_list(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
(tx_hash_hex, o) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024)
r = rpc.do(o)
tx_signed_raw_hex = o['params'][0]
#queue_create(tx['nonce'], tx['from'], tx_hash.hex(), tx_signed['raw'], str(default_chain_spec))
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
q = init_database.query(Otx)
otx = q.get(1)
otx.sendfail(session=init_database)
init_database.add(otx)
init_database.commit()
init_database.refresh(otx)
txs = get_status_tx(StatusBits.LOCAL_ERROR, session=init_database)
assert len(txs) == 1
otx.sendfail(session=init_database)
otx.retry(session=init_database)
init_database.add(otx)
init_database.commit()
init_database.refresh(otx)
txs = get_status_tx(StatusBits.LOCAL_ERROR, session=init_database)
assert len(txs) == 1
txs = get_status_tx(StatusBits.QUEUED, session=init_database)
assert len(txs) == 1
txs = get_status_tx(StatusBits.QUEUED, not_status=StatusBits.LOCAL_ERROR, session=init_database)
assert len(txs) == 0
txs = get_status_tx(StatusBits.QUEUED, not_status=StatusBits.IN_NETWORK, session=init_database)
assert len(txs) == 1
txs = get_status_tx(StatusBits.IN_NETWORK, session=init_database)
assert len(txs) == 0

22
apps/cic-eth/tests/unit/queue/test_otx_state_log.py
View File

@ -1,22 +0,0 @@
# standard imports
import os
# local imports
from cic_eth.db.models.otx import Otx
from cic_eth.queue.tx import get_state_log
def test_otx_state_log(
init_database,
):
Otx.tracing = True
address = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex()
otx = Otx.add(0, address, tx_hash, signed_tx, session=init_database)
init_database.commit()
log = get_state_log(tx_hash)
assert len(log) == 1

58
apps/cic-eth/tests/unit/queue/test_query.py Normal file
View File

@ -0,0 +1,58 @@
# external imports
from chainqueue.db.enum import (
StatusEnum,
StatusBits,
)
from chainlib.connection import RPCConnection
from chainlib.eth.gas import (
RPCGasOracle,
Gas,
)
from chainlib.chain import ChainSpec
# local imports
from cic_eth.db.enum import LockEnum
from cic_eth.db.models.lock import Lock
from cic_eth.queue.query import get_upcoming_tx
from cic_eth.queue.tx import register_tx
from cic_eth.eth.gas import cache_gas_data
# test imports
from tests.util.nonce import StaticNonceOracle
def test_upcoming_with_lock(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = StaticNonceOracle(42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6))
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
assert len(txs.keys()) == 1
Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE'])
txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
assert len(txs.keys()) == 0
(tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6))
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING)
assert len(txs.keys()) == 1

10
apps/cic-eth/tests/unit/queue/test_queue_lock.py
View File

@ -5,10 +5,10 @@ import os
import pytest
# local imports
from cic_eth.queue.tx import create as queue_create
from cic_eth.db.models.lock import Lock
from cic_eth.db.enum import LockEnum
from cic_eth.error import LockedError
from cic_eth.queue.tx import queue_create
def test_queue_lock(
@ -25,41 +25,41 @@ def test_queue_lock(
Lock.set(chain_str, LockEnum.QUEUE)
with pytest.raises(LockedError):
queue_create(
default_chain_spec,
0,
address,
tx_hash,
tx_raw,
chain_str
)
Lock.set(chain_str, LockEnum.QUEUE, address=address)
with pytest.raises(LockedError):
queue_create(
default_chain_spec,
0,
address,
tx_hash,
tx_raw,
chain_str
)
Lock.reset(chain_str, LockEnum.QUEUE)
with pytest.raises(LockedError):
queue_create(
default_chain_spec,
0,
address,
tx_hash,
tx_raw,
chain_str
)
Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
with pytest.raises(LockedError):
queue_create(
default_chain_spec,
0,
address,
tx_hash,
tx_raw,
chain_str
)

577
apps/cic-eth/tests/unit/queue/test_tx_queue.py
View File

@ -1,577 +0,0 @@
# standard imports import logging
import datetime
import os
import logging
# external imports
import pytest
from sqlalchemy import DateTime
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import OverrideNonceOracle
from chainlib.eth.tx import unpack
from chainlib.eth.gas import (
RPCGasOracle,
Gas,
)
from chainlib.eth.constant import ZERO_ADDRESS
from hexathon import strip_0x
# local imports
from cic_eth.eth.tx import cache_gas_data
from cic_eth.db.models.otx import Otx
from cic_eth.db.models.otx import OtxSync
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.lock import Lock
from cic_eth.db.models.base import SessionBase
from cic_eth.db.enum import (
StatusEnum,
LockEnum,
StatusBits,
is_alive,
is_error_status,
status_str,
)
from cic_eth.queue.tx import create as queue_create
from cic_eth.queue.tx import set_final_status
from cic_eth.queue.tx import set_sent_status
from cic_eth.queue.tx import set_waitforgas
from cic_eth.queue.tx import set_ready
from cic_eth.queue.tx import get_paused_txs
from cic_eth.queue.tx import get_upcoming_tx
from cic_eth.queue.tx import get_account_tx
from cic_eth.queue.tx import get_tx
from cic_eth.db.error import TxStateChangeError
from cic_eth.queue.tx import register_tx
# test imports
from tests.util.nonce import StaticNonceOracle
logg = logging.getLogger()
def test_finalize(
default_chain_spec,
eth_rpc,
eth_signer,
init_database,
agent_roles,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = StaticNonceOracle(0)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc = [
c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['BOB'], 200 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['BOB'], 300 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['BOB'], 400 * (10 ** 6)),
]
nonce_oracle = StaticNonceOracle(1)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc.append(c.create(agent_roles['ALICE'], agent_roles['BOB'], 500 * (10 ** 6)))
tx_hashes = []
i = 0
for entry in txs_rpc:
tx_hash_hex = entry[0]
tx_rpc = entry[1]
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_hashes.append(tx_hash_hex)
if i < 3:
set_sent_status(tx_hash_hex)
i += 1
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[0]).first()
assert otx.status & StatusBits.OBSOLETE
assert not is_alive(otx.status)
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[1]).first()
assert otx.status & StatusBits.OBSOLETE
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[2]).first()
assert otx.status & StatusBits.OBSOLETE
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[3]).first()
assert otx.status == StatusEnum.PENDING
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[4]).first()
assert otx.status == StatusEnum.PENDING
set_sent_status(tx_hashes[3], False)
set_sent_status(tx_hashes[4], False)
set_final_status(tx_hashes[3], 1024)
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[0]).first()
assert otx.status & (StatusBits.OBSOLETE | StatusBits.FINAL)
assert not is_alive(otx.status)
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[1]).first()
assert otx.status & (StatusBits.OBSOLETE | StatusBits.FINAL)
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[2]).first()
assert otx.status & (StatusBits.OBSOLETE | StatusBits.FINAL)
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[3]).first()
assert otx.status & (StatusBits.IN_NETWORK | StatusBits.FINAL)
assert not is_error_status(otx.status)
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hashes[4]).first()
assert otx.status & (StatusBits.IN_NETWORK | StatusBits.FINAL)
assert not is_error_status(otx.status)
def test_expired(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = StaticNonceOracle(42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc = [
c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['BOB'], 200 * (10 ** 6)),
]
nonce_oracle = StaticNonceOracle(43)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc += [
c.create(agent_roles['ALICE'], agent_roles['BOB'], 300 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['BOB'], 400 * (10 ** 6)),
]
nonce_oracle = StaticNonceOracle(44)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc.append(c.create(agent_roles['ALICE'], agent_roles['BOB'], 500 * (10 ** 6)))
tx_hashes = []
i = 0
for entry in txs_rpc:
tx_hash_hex = entry[0]
tx_rpc = entry[1]
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_hashes.append(tx_hash_hex)
set_sent_status(tx_hash_hex, False)
otx = init_database.query(Otx).filter(Otx.tx_hash==tx_hash_hex).first()
fake_created = datetime.datetime.utcnow() - datetime.timedelta(seconds=40*i)
otx.date_created = fake_created
init_database.add(otx)
init_database.commit()
init_database.refresh(otx)
i += 1
now = datetime.datetime.utcnow()
delta = datetime.timedelta(seconds=61)
then = now - delta
otxs = OtxSync.get_expired(then)
nonce_acc = 0
for otx in otxs:
nonce_acc += otx.nonce
assert nonce_acc == (43 + 44)
def test_get_paused(
init_database,
default_chain_spec,
eth_rpc,
eth_signer,
agent_roles,
):
chain_id = default_chain_spec.chain_id()
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc = [
c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['BOB'], 200 * (10 ** 6)),
]
tx_hashes = []
for entry in txs_rpc:
tx_hash_hex = entry[0]
tx_rpc = entry[1]
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_hashes.append(tx_hash_hex)
txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id)
assert len(txs.keys()) == 0
q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==tx_hashes[0])
r = q.first()
r.waitforgas(session=init_database)
init_database.add(r)
init_database.commit()
chain_id = default_chain_spec.chain_id()
txs = get_paused_txs(chain_id=chain_id)
assert len(txs.keys()) == 1
txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id) # init_w3.eth.accounts[0])
assert len(txs.keys()) == 1
txs = get_paused_txs(status=StatusBits.GAS_ISSUES)
assert len(txs.keys()) == 1
txs = get_paused_txs(sender=agent_roles['ALICE'], status=StatusBits.GAS_ISSUES, chain_id=chain_id)
assert len(txs.keys()) == 1
q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==tx_hashes[1])
o = q.first()
o.waitforgas(session=init_database)
init_database.add(o)
init_database.commit()
txs = get_paused_txs()
assert len(txs.keys()) == 2
txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id) # init_w3.eth.accounts[0])
assert len(txs.keys()) == 2
txs = get_paused_txs(status=StatusBits.GAS_ISSUES, chain_id=chain_id)
assert len(txs.keys()) == 2
txs = get_paused_txs(sender=agent_roles['ALICE'], status=StatusBits.GAS_ISSUES, chain_id=chain_id) # init_w3.eth.accounts[0])
assert len(txs.keys()) == 2
q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==tx_hashes[1])
o = q.first()
o.sendfail(session=init_database)
init_database.add(o)
init_database.commit()
txs = get_paused_txs()
assert len(txs.keys()) == 2
txs = get_paused_txs(sender=agent_roles['ALICE'], chain_id=chain_id) # init_w3.eth.accounts[0])
assert len(txs.keys()) == 2
txs = get_paused_txs(status=StatusBits.GAS_ISSUES, chain_id=chain_id)
txs = get_paused_txs(status=StatusEnum.WAITFORGAS, chain_id=chain_id)
assert len(txs.keys()) == 1
txs = get_paused_txs(sender=agent_roles['ALICE'], status=StatusBits.GAS_ISSUES, chain_id=chain_id) # init_w3.eth.accounts[0])
assert len(txs.keys()) == 1
def test_get_upcoming(
default_chain_spec,
eth_rpc,
eth_signer,
init_database,
agent_roles,
):
chain_id = default_chain_spec.chain_id()
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = StaticNonceOracle(42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc = [
c.create(agent_roles['ALICE'], agent_roles['DAVE'], 100 * (10 ** 6)),
c.create(agent_roles['BOB'], agent_roles['DAVE'], 200 * (10 ** 6)),
c.create(agent_roles['CAROL'], agent_roles['DAVE'], 300 * (10 ** 6)),
]
nonce_oracle = StaticNonceOracle(43)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc += [
c.create(agent_roles['ALICE'], agent_roles['DAVE'], 400 * (10 ** 6)),
c.create(agent_roles['BOB'], agent_roles['DAVE'], 500 * (10 ** 6)),
c.create(agent_roles['CAROL'], agent_roles['DAVE'], 600 * (10 ** 6)),
]
nonce_oracle = StaticNonceOracle(44)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc += [
c.create(agent_roles['ALICE'], agent_roles['DAVE'], 700 * (10 ** 6)),
]
tx_hashes = []
for entry in txs_rpc:
tx_hash_hex = entry[0]
tx_rpc = entry[1]
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_hashes.append(tx_hash_hex)
set_ready(tx_hash_hex)
txs = get_upcoming_tx(StatusBits.QUEUED, chain_id=chain_id)
assert len(txs.keys()) == 3
tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[0]])), chain_id)
assert tx['nonce'] == 42
tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[1]])), chain_id)
assert tx['nonce'] == 42
tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[2]])), chain_id)
assert tx['nonce'] == 42
q = init_database.query(TxCache)
q = q.filter(TxCache.sender==agent_roles['ALICE'])
for o in q.all():
o.date_checked -= datetime.timedelta(seconds=30)
init_database.add(o)
init_database.commit()
before = datetime.datetime.now() - datetime.timedelta(seconds=20)
logg.debug('before {}'.format(before))
txs = get_upcoming_tx(StatusBits.QUEUED, before=before)
logg.debug('txs {} {}'.format(txs.keys(), txs.values()))
assert len(txs.keys()) == 1
# Now date checked has been set to current time, and the check returns no results
txs = get_upcoming_tx(StatusBits.QUEUED, before=before)
logg.debug('txs {} {}'.format(txs.keys(), txs.values()))
assert len(txs.keys()) == 0
set_sent_status(tx_hashes[0])
txs = get_upcoming_tx(StatusBits.QUEUED)
assert len(txs.keys()) == 3
with pytest.raises(KeyError):
tx = txs[tx_hashes[0]]
tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[3]])), chain_id)
assert tx['nonce'] == 43
set_waitforgas(tx_hashes[1])
txs = get_upcoming_tx(StatusBits.QUEUED)
assert len(txs.keys()) == 3
with pytest.raises(KeyError):
tx = txs[tx_hashes[1]]
tx = unpack(bytes.fromhex(strip_0x(txs[tx_hashes[3]])), chain_id)
assert tx['nonce'] == 43
txs = get_upcoming_tx(StatusBits.GAS_ISSUES)
assert len(txs.keys()) == 1
def test_upcoming_with_lock(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
):
chain_id = int(default_chain_spec.chain_id())
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = StaticNonceOracle(42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
(tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6))
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
txs = get_upcoming_tx(StatusEnum.PENDING, chain_id=chain_id)
assert len(txs.keys()) == 1
Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE'])
txs = get_upcoming_tx(StatusEnum.PENDING, chain_id=chain_id)
assert len(txs.keys()) == 0
(tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6))
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
txs = get_upcoming_tx(StatusEnum.PENDING, chain_id=chain_id)
assert len(txs.keys()) == 1
def test_obsoletion(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
):
chain_id = default_chain_spec.chain_id()
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = StaticNonceOracle(42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc = [
c.create(agent_roles['ALICE'], agent_roles['DAVE'], 100 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['DAVE'], 200 * (10 ** 6)),
c.create(agent_roles['BOB'], agent_roles['DAVE'], 300 * (10 ** 6)),
]
nonce_oracle = StaticNonceOracle(43)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc += [
c.create(agent_roles['BOB'], agent_roles['DAVE'], 400 * (10 ** 6)),
]
tx_hashes = []
i = 0
for entry in txs_rpc:
tx_hash_hex = entry[0]
tx_rpc = entry[1]
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_hashes.append(tx_hash_hex)
if i < 2:
set_sent_status(tx_hash_hex)
i += 1
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.status.op('&')(StatusEnum.OBSOLETED.value)==StatusEnum.OBSOLETED.value)
z = 0
for o in q.all():
z += o.nonce
session.close()
assert z == 42
set_final_status(tx_hashes[1], 1023, True)
session = SessionBase.create_session()
q = session.query(Otx)
q = q.filter(Otx.status.op('&')(StatusEnum.CANCELLED.value)==StatusEnum.OBSOLETED.value)
zo = 0
for o in q.all():
zo += o.nonce
q = session.query(Otx)
q = q.filter(Otx.status.op('&')(StatusEnum.CANCELLED.value)==StatusEnum.CANCELLED.value)
zc = 0
for o in q.all():
zc += o.nonce
session.close()
assert zo == 0
assert zc == 42
def test_retry(
init_database,
):
address = '0x' + os.urandom(20).hex()
tx_hash = '0x' + os.urandom(32).hex()
signed_tx = '0x' + os.urandom(128).hex()
otx = Otx(0, address, tx_hash, signed_tx)
init_database.add(otx)
init_database.commit()
set_sent_status(tx_hash, True)
set_ready(tx_hash)
q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
otx = q.first()
assert (otx.status & StatusEnum.RETRY.value) == StatusEnum.RETRY.value
assert is_error_status(otx.status)
set_sent_status(tx_hash, False)
set_ready(tx_hash)
init_database.commit()
q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==tx_hash)
otx = q.first()
assert (otx.status & StatusEnum.RETRY.value) == StatusBits.QUEUED.value
assert not is_error_status(otx.status)
def test_get_account_tx(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
):
chain_id = default_chain_spec.chain_id()
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = OverrideNonceOracle(ZERO_ADDRESS, 42)
gas_oracle = RPCGasOracle(eth_rpc)
c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=default_chain_spec.chain_id())
txs_rpc = [
c.create(agent_roles['ALICE'], agent_roles['DAVE'], 100 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6)),
c.create(agent_roles['ALICE'], agent_roles['BOB'], 300 * (10 ** 6)),
c.create(agent_roles['BOB'], agent_roles['ALICE'], 300 * (10 ** 6)),
]
tx_hashes = []
for entry in txs_rpc:
tx_hash_hex = entry[0]
tx_rpc = entry[1]
tx_signed_raw_hex = tx_rpc['params'][0]
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_hashes.append(tx_hash_hex)
txs = get_account_tx(agent_roles['ALICE'])
logg.debug('tx {} tx {}'.format(list(txs.keys()), tx_hashes))
assert list(txs.keys()) == tx_hashes
txs = get_account_tx(agent_roles['ALICE'], as_recipient=False)
assert list(txs.keys()) == tx_hashes[:3]
txs = get_account_tx(agent_roles['ALICE'], as_sender=False)
assert list(txs.keys()) == tx_hashes[3:]

6
apps/contract-migration/docker/Dockerfile
View File

@ -57,9 +57,9 @@ WORKDIR /home/grassroots
USER grassroots
ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
ARG cic_base_version=0.1.2a49
ARG cic_eth_version=0.10.1b1
ARG sarafu_faucet_version=0.0.2a13
ARG cic_base_version=0.1.2a57
ARG cic_eth_version=0.11.0a3
ARG sarafu_faucet_version=0.0.2a16
ARG cic_contracts_version=0.0.2a2
RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
cic-eth==$cic_eth_version \

40
apps/contract-migration/scripts/import_balance.py
View File

@ -174,25 +174,25 @@ class Handler:
# logg.error('key record not found in imports: {}'.format(e).ljust(200))
class BlockGetter:
def __init__(self, conn, gas_oracle, nonce_oracle, chain_id):
self.conn = conn
self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
def get(self, n):
o = block_by_number(n)
r = self.conn.do(o)
b = None
try:
b = Block(r)
except TypeError as e:
if r == None:
logg.debug('block not found {}'.format(n))
else:
logg.error('block retrieve error {}'.format(e))
return b
#class BlockGetter:
#
# def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
# self.conn = conn
# self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
#
#
# def get(self, n):
# o = block_by_number(n)
# r = self.conn.do(o)
# b = None
# try:
# b = Block(r)
# except TypeError as e:
# if r == None:
# logg.debug('block not found {}'.format(n))
# else:
# logg.error('block retrieve error {}'.format(e))
# return b
def progress_callback(block_number, tx_index, s):
@ -208,7 +208,7 @@ def main():
nonce_oracle = RPCNonceOracle(signer_address, conn)
# Get Token registry address
txf = TxFactory(signer=signer, gas_oracle=gas_oracle, nonce_oracle=None, chain_id=chain_spec.network_id())
txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]

2
apps/contract-migration/scripts/import_sovereign_users.py
View File

@ -103,7 +103,7 @@ def register_eth(i, u):
address = add_0x(to_checksum_address(address_hex))
gas_oracle = RPCGasOracle(rpc, code_callback=AccountRegistry.gas)
c = AccountRegistry(signer=signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
c = AccountRegistry(chain_spec, signer=signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, o) = c.add(account_registry_address, signer_address, address)
logg.debug('o {}'.format(o))
rpc.do(o)

8
apps/contract-migration/scripts/requirements.txt
View File

@ -1,5 +1,5 @@
cic-base[full_graph]==0.1.2a52
sarafu-faucet==0.0.2a15
cic-eth==0.10.1b2+build.9c750e70
cic-types==0.1.0a8
cic-base[full_graph]==0.1.2a57
sarafu-faucet==0.0.2a16
cic-eth==0.11.0a3
cic-types==0.1.0a10
crypto-dev-signer==0.4.14a17

8
apps/contract-migration/scripts/verify.py
View File

@ -203,12 +203,12 @@ class Verifier:
self.index_address = index_address
self.token_address = token_address
self.faucet_address = faucet_address
self.erc20_tx_factory = ERC20(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
self.tx_factory = TxFactory(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
self.erc20_tx_factory = ERC20(chain_spec, gas_oracle=gas_oracle)
self.tx_factory = TxFactory(chain_spec, gas_oracle=gas_oracle)
self.api = cic_eth_api
self.data_dir = data_dir
self.exit_on_error = exit_on_error
self.faucet_tx_factory = SingleShotFaucet(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
self.faucet_tx_factory = SingleShotFaucet(chain_spec, gas_oracle=gas_oracle)
verifymethods = []
for k in dir(self):
@ -326,7 +326,7 @@ def main():
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
# Get Token registry address
txf = TxFactory(signer=None, gas_oracle=gas_oracle, nonce_oracle=None, chain_id=chain_spec.chain_id())
txf = TxFactory(chain_spec, signer=None, gas_oracle=gas_oracle, nonce_oracle=None)
tx = txf.template(ZERO_ADDRESS, config.get('CIC_REGISTRY_ADDRESS'))
# TODO: replace with cic-eth-registry

2
apps/contract-migration/seed_cic_eth.sh
View File

@ -30,6 +30,8 @@ truncate $env_out_file -s 0
set -e
set -a
#pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL eth-address-index==0.1.1a7
# get required addresses from registries
DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry`
DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`