diff --git a/apps/cic-eth/cic_eth/admin/debug.py b/apps/cic-eth/cic_eth/admin/debug.py index 575be543..a8c233e4 100644 --- a/apps/cic-eth/cic_eth/admin/debug.py +++ b/apps/cic-eth/cic_eth/admin/debug.py @@ -1,12 +1,25 @@ +# standard imports import datetime +# external imports import celery +# local imports +from cic_eth.db.models.debug import Debug +from cic_eth.db.models.base import SessionBase +from cic_eth.task import CriticalSQLAlchemyTask + celery_app = celery.current_app -@celery_app.task() -def out_tmp(tag, txt): - f = open('/tmp/err.{}.txt'.format(tag), "w") - f.write(txt) - f.close() +@celery_app.task(base=CriticalSQLAlchemyTask) +def alert(chained_input, tag, txt): + session = SessionBase.create_session() + + o = Debug(tag, txt) + session.add(o) + session.commit() + + session.close() + + return chained_input diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py new file mode 100644 index 00000000..378f2965 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py @@ -0,0 +1,32 @@ +"""debug output + +Revision ID: f738d9962fdf +Revises: ec40ac0974c1 +Create Date: 2021-03-04 08:32:43.281214 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f738d9962fdf' +down_revision = 'ec40ac0974c1' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'debug', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('tag', sa.String, nullable=False), + sa.Column('description', sa.String, nullable=False), + sa.Column('date_created', sa.DateTime, nullable=False), + ) + pass + + +def downgrade(): + op.drop_table('debug') + pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py new file mode 100644 index 00000000..378f2965 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py @@ -0,0 +1,32 @@ +"""debug output + +Revision ID: f738d9962fdf +Revises: ec40ac0974c1 +Create Date: 2021-03-04 08:32:43.281214 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f738d9962fdf' +down_revision = 'ec40ac0974c1' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'debug', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('tag', sa.String, nullable=False), + sa.Column('description', sa.String, nullable=False), + sa.Column('date_created', sa.DateTime, nullable=False), + ) + pass + + +def downgrade(): + op.drop_table('debug') + pass diff --git a/apps/cic-eth/cic_eth/db/models/debug.py b/apps/cic-eth/cic_eth/db/models/debug.py new file mode 100644 index 00000000..04682c95 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/models/debug.py @@ -0,0 +1,23 @@ +# standard imports +import datetime +import logging + +# external imports +from sqlalchemy import Column, String, DateTime + +# local imports +from .base import SessionBase + + +class Debug(SessionBase): + + __tablename__ = 'debug' + + date_created = Column(DateTime, default=datetime.datetime.utcnow) + tag = Column(String) + description = Column(String) + + + def __init__(self, tag, description): + self.tag = tag + self.description = description diff --git a/apps/cic-eth/cic_eth/db/models/otx.py b/apps/cic-eth/cic_eth/db/models/otx.py index 523ae2e0..ec88fa4b 100644 --- a/apps/cic-eth/cic_eth/db/models/otx.py +++ b/apps/cic-eth/cic_eth/db/models/otx.py @@ -2,7 +2,7 @@ import datetime import logging -# third-party imports +# external imports from sqlalchemy import Column, Enum, String, Integer, DateTime, Text, or_, ForeignKey from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method diff --git a/apps/cic-eth/cic_eth/eth/tx.py b/apps/cic-eth/cic_eth/eth/tx.py index 1c34acca..4de09734 100644 --- a/apps/cic-eth/cic_eth/eth/tx.py +++ b/apps/cic-eth/cic_eth/eth/tx.py @@ -69,7 +69,6 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non for i in range(len(tx_hashes)): o = get_tx(tx_hashes[i]) txs.append(o['signed_tx']) - logg.debug('ooooo {}'.format(o)) if address == None: address = o['address'] @@ -178,12 +177,7 @@ class ParityNodeHandler: def handle(self, exception, tx_hash_hex, tx_hex): meth = self.handle_default if isinstance(exception, (ValueError)): -# s_debug = celery.signature( -# 'cic_eth.admin.debug.out_tmp', -# [tx_hash_hex, '{}: {}'.format(tx_hash_hex, exception)], -# queue=queue, -# ) -# s_debug.apply_async() + earg = exception.args[0] if earg['code'] == -32010: logg.debug('skipping lock for code {}'.format(earg['code'])) @@ -191,14 +185,15 @@ class ParityNodeHandler: elif earg['code'] == -32602: meth = self.handle_invalid_encoding else: + # TODO: move to status log db comment field meth = self.handle_invalid elif isinstance(exception, (requests.exceptions.ConnectionError)): meth = self.handle_connection - (t, e_fn, message) = meth(tx_hash_hex, tx_hex) + (t, e_fn, message) = meth(tx_hash_hex, tx_hex, str(exception)) return (t, e_fn, '{} {}'.format(message, exception)) - def handle_connection(self, tx_hash_hex, tx_hex): + def handle_connection(self, tx_hash_hex, tx_hex, debugstr=None): s_set_sent = celery.signature( 'cic_eth.queue.tx.set_sent_status', [ @@ -211,7 +206,7 @@ class ParityNodeHandler: return (t, TemporaryTxError, 'Sendfail {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) - def handle_invalid_encoding(self, tx_hash_hex, tx_hex): + def handle_invalid_encoding(self, tx_hash_hex, tx_hex, debugstr=None): tx_bytes = bytes.fromhex(tx_hex[2:]) tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id()) s_lock = celery.signature( @@ -258,7 +253,7 @@ class ParityNodeHandler: return (t, PermanentTxError, 'Reject invalid encoding {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) - def handle_invalid_parameters(self, tx_hash_hex, tx_hex): + def handle_invalid_parameters(self, tx_hash_hex, tx_hex, debugstr=None): s_sync = celery.signature( 'cic_eth.eth.tx.sync_tx', [ @@ -271,7 +266,7 @@ class ParityNodeHandler: return (t, PermanentTxError, 'Reject invalid parameters {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) - def handle_invalid(self, tx_hash_hex, tx_hex): + def handle_invalid(self, tx_hash_hex, tx_hex, debugstr=None): tx_bytes = bytes.fromhex(tx_hex[2:]) tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id()) s_lock = celery.signature( @@ -289,6 +284,16 @@ class ParityNodeHandler: [], queue=self.queue, ) + s_debug = celery.signature( + 'cic_eth.admin.debug.alert', + [ + tx_hash_hex, + tx_hash_hex, + debugstr, + ], + queue=queue, + ) + s_set_reject.link(s_debug) s_lock.link(s_set_reject) t = s_lock.apply_async() return (t, PermanentTxError, 'Reject invalid {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) diff --git a/apps/cic-eth/cic_eth/queue/tx.py b/apps/cic-eth/cic_eth/queue/tx.py index e4a8f818..c1fe0a3f 100644 --- a/apps/cic-eth/cic_eth/queue/tx.py +++ b/apps/cic-eth/cic_eth/queue/tx.py @@ -30,6 +30,7 @@ from cic_eth.task import CriticalSQLAlchemyTask from cic_eth.eth.util import unpack_signed_raw_tx # TODO: should not be in same sub-path as package that imports queue.tx from cic_eth.error import NotLocalTxError from cic_eth.error import LockedError +from cic_eth.db.enum import status_str celery_app = celery.current_app #logg = celery_app.log.get_default_logger() @@ -405,7 +406,7 @@ def get_tx_cache(tx_hash): 'tx_hash': otx.tx_hash, 'signed_tx': otx.signed_tx, 'nonce': otx.nonce, - 'status': StatusEnum(otx.status).name, + 'status': status_str(otx.status), 'status_code': otx.status, 'source_token': txc.source_token_address, 'destination_token': txc.destination_token_address, diff --git a/apps/cic-eth/cic_eth/runnable/view.py b/apps/cic-eth/cic_eth/runnable/view.py index a9d964c0..40e10cce 100644 --- a/apps/cic-eth/cic_eth/runnable/view.py +++ b/apps/cic-eth/cic_eth/runnable/view.py @@ -23,8 +23,11 @@ from hexathon import add_0x # local imports from cic_eth.api import AdminApi from cic_eth.eth.rpc import RpcClient -from cic_eth.db.enum import StatusEnum -from cic_eth.db.enum import LockEnum +from cic_eth.db.enum import ( + StatusEnum, + status_str, + LockEnum, +) logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -119,7 +122,7 @@ def render_tx(o, **kwargs): for v in o.get('status_log', []): d = datetime.datetime.fromisoformat(v[0]) - e = StatusEnum(v[1]).name + e = status_str(v[1]) content += '{}: {}\n'.format(d, e) return content diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 84b32f5c..bf832fbe 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,7 +1,7 @@ cic-base~=0.1.1a10 web3==5.12.2 celery==4.4.7 -crypto-dev-signer~=0.4.13rc3 +crypto-dev-signer~=0.4.13rc4 confini~=0.3.6rc3 cic-registry~=0.5.3a22 cic-bancor~=0.0.6 diff --git a/apps/cic-eth/tests/fixtures_celery.py b/apps/cic-eth/tests/fixtures_celery.py index 39395ac9..e8a322a3 100644 --- a/apps/cic-eth/tests/fixtures_celery.py +++ b/apps/cic-eth/tests/fixtures_celery.py @@ -19,6 +19,7 @@ def celery_includes(): 'cic_eth.queue.balance', 'cic_eth.admin.ctrl', 'cic_eth.admin.nonce', + 'cic_eth.admin.debug', 'cic_eth.eth.account', 'cic_eth.callbacks.noop', 'cic_eth.callbacks.http', diff --git a/apps/cic-eth/tests/tasks/test_debug.py b/apps/cic-eth/tests/tasks/test_debug.py new file mode 100644 index 00000000..65d8eed1 --- /dev/null +++ b/apps/cic-eth/tests/tasks/test_debug.py @@ -0,0 +1,29 @@ +# external imports +import celery + +# local imports +from cic_eth.db.models.debug import Debug + + +def test_debug_alert( + init_database, + celery_session_worker, + ): + + s = celery.signature( + 'cic_eth.admin.debug.alert', + [ + 'foo', + 'bar', + 'baz', + ], + queue=None, + ) + t = s.apply_async() + r = t.get() + assert r == 'foo' + + q = init_database.query(Debug) + q = q.filter(Debug.tag=='bar') + o = q.first() + assert o.description == 'baz' diff --git a/apps/cic-eth/tests/unit/db/test_debug.py b/apps/cic-eth/tests/unit/db/test_debug.py new file mode 100644 index 00000000..776247d7 --- /dev/null +++ b/apps/cic-eth/tests/unit/db/test_debug.py @@ -0,0 +1,16 @@ +# local imports +from cic_eth.db.models.debug import Debug + + +def test_debug( + init_database, + ): + + o = Debug('foo', 'bar') + init_database.add(o) + init_database.commit() + + q = init_database.query(Debug) + q = q.filter(Debug.tag=='foo') + o = q.first() + assert o.description == 'bar' diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index d311e7ae..1cb6cbb5 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,3 +1,3 @@ -cic-base[full_graph]==0.1.1a10 -cic-eth==0.10.0a36 +cic-base[full_graph]==0.1.1a12 +cic-eth==0.10.0a37 cic-types==0.1.0a8 diff --git a/apps/requirements.txt b/apps/requirements.txt index ed354d85..97ffa2bb 100644 --- a/apps/requirements.txt +++ b/apps/requirements.txt @@ -3,7 +3,7 @@ alembic==1.4.2 bcrypt==3.2.0 celery==4.4.7 confini==0.3.6rc3 -crypto-dev-signer==0.4.13rc3 +crypto-dev-signer==0.4.13rc4 cryptography==3.2.1 ecuth==0.4.5a1 eth-accounts-index==0.0.10a10 diff --git a/docker-compose.yml b/docker-compose.yml index f277b714..073d8085 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -192,7 +192,7 @@ services: if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi "/usr/local/bin/uwsgi" \ --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \ - --http :80 \ + --http :8000 \ --pyargv -vv cic-eth-tasker: @@ -237,7 +237,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tasker.sh -q cic-eth -vv + ./start_tasker.sh -q cic-eth -v # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ] cic-eth-tracker: @@ -274,7 +274,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tracker.sh -vv -c /usr/local/etc/cic-eth + ./start_tracker.sh -v -c /usr/local/etc/cic-eth # command: "/root/start_manager.sh head -vv"