From 6485f570450134e45c4f5a60c8ed88d614f8eb27 Mon Sep 17 00:00:00 2001 From: nolash Date: Sat, 6 Mar 2021 18:01:51 +0100 Subject: [PATCH] Fix signature length error in unpack --- apps/cic-eth/cic_eth/api/api_admin.py | 14 ++- apps/cic-eth/cic_eth/db/enum.py | 3 + apps/cic-eth/cic_eth/db/models/nonce.py | 1 - apps/cic-eth/cic_eth/db/models/otx.py | 2 +- apps/cic-eth/cic_eth/eth/tx.py | 6 +- apps/cic-eth/cic_eth/eth/util.py | 118 +++++++++--------- .../cic_eth/runnable/daemons/dispatcher.py | 8 +- .../runnable/daemons/filters/register.py | 16 ++- .../runnable/daemons/filters/transferauth.py | 22 ++-- .../cic_eth/runnable/daemons/filters/tx.py | 2 +- apps/cic-eth/cic_eth/runnable/view.py | 3 +- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/docker/start_tasker.sh | 2 +- apps/cic-eth/requirements.txt | 2 +- .../scripts/requirements.txt | 2 +- apps/contract-migration/seed_cic_eth.sh | 2 +- apps/requirements.txt | 2 +- docker-compose.yml | 3 +- 18 files changed, 121 insertions(+), 89 deletions(-) diff --git a/apps/cic-eth/cic_eth/api/api_admin.py b/apps/cic-eth/cic_eth/api/api_admin.py index ca43d037..402fd3a8 100644 --- a/apps/cic-eth/cic_eth/api/api_admin.py +++ b/apps/cic-eth/cic_eth/api/api_admin.py @@ -1,5 +1,6 @@ # standard imports import logging +import sys # third-party imports import celery @@ -317,6 +318,8 @@ class AdminApi: :return: Transaction details :rtype: dict """ + problems = [] + if tx_hash != None and tx_raw != None: ValueError('Specify only one of hash or raw tx') @@ -444,10 +447,12 @@ class AdminApi: r = c.w3.eth.getTransactionReceipt(tx_hash) if r.status == 1: tx['network_status'] = 'Confirmed' - tx['block'] = r.blockNumber - tx['tx_index'] = r.transactionIndex else: tx['network_status'] = 'Reverted' + tx['network_block_number'] = r.blockNumber + tx['network_tx_index'] = r.transactionIndex + if tx['block_number'] == None: + problems.append('Queue is missing block number {} for mined tx'.format(r.blockNumber)) except web3.exceptions.TransactionNotFound: pass @@ -469,4 +474,9 @@ class AdminApi: t = s.apply_async() tx['status_log'] = t.get() + if len(problems) > 0: + sys.stderr.write('\n') + for p in problems: + sys.stderr.write('!!!{}\n'.format(p)) + return tx diff --git a/apps/cic-eth/cic_eth/db/enum.py b/apps/cic-eth/cic_eth/db/enum.py index 9b77ba18..67a50f0d 100644 --- a/apps/cic-eth/cic_eth/db/enum.py +++ b/apps/cic-eth/cic_eth/db/enum.py @@ -103,6 +103,9 @@ def status_str(v, bits_only=False): except ValueError: pass + if v == 0: + return 'NONE' + for i in range(16): b = (1 << i) if (b & 0xffff) & v: diff --git a/apps/cic-eth/cic_eth/db/models/nonce.py b/apps/cic-eth/cic_eth/db/models/nonce.py index 42bb8fd3..e6194299 100644 --- a/apps/cic-eth/cic_eth/db/models/nonce.py +++ b/apps/cic-eth/cic_eth/db/models/nonce.py @@ -115,7 +115,6 @@ class Nonce(SessionBase): return nonce - class NonceReservation(SessionBase): __tablename__ = 'nonce_task_reservation' diff --git a/apps/cic-eth/cic_eth/db/models/otx.py b/apps/cic-eth/cic_eth/db/models/otx.py index ec88fa4b..06f2d655 100644 --- a/apps/cic-eth/cic_eth/db/models/otx.py +++ b/apps/cic-eth/cic_eth/db/models/otx.py @@ -400,7 +400,7 @@ class Otx(SessionBase): raise TxStateChangeError('CANCEL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status))) if confirmed: - if not self.status & StatusBits.OBSOLETE: + if self.status > 0 and not self.status & StatusBits.OBSOLETE: raise TxStateChangeError('CANCEL can only be set on an entry marked OBSOLETE ({})'.format(status_str(self.status))) self.__set_status(StatusEnum.CANCELLED, session) else: diff --git a/apps/cic-eth/cic_eth/eth/tx.py b/apps/cic-eth/cic_eth/eth/tx.py index 74c6c165..2e43f5f6 100644 --- a/apps/cic-eth/cic_eth/eth/tx.py +++ b/apps/cic-eth/cic_eth/eth/tx.py @@ -118,7 +118,7 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non safe_gas = c.safe_threshold_amount() if balance < safe_gas: s_nonce = celery.signature( - 'cic_eth.eth.tx.reserve_nonce' + 'cic_eth.eth.tx.reserve_nonce', [ address, c.gas_provider(), @@ -304,7 +304,6 @@ class ParityNodeHandler: s_debug = celery.signature( 'cic_eth.admin.debug.alert', [ - tx_hash_hex, tx_hash_hex, debugstr, ], @@ -337,7 +336,6 @@ class ParityNodeHandler: s_debug = celery.signature( 'cic_eth.admin.debug.alert', [ - tx_hash_hex, tx_hash_hex, debugstr, ], @@ -446,7 +444,7 @@ def refill_gas(self, recipient_address, chain_str): if c > 0: #session.close() #raise AlreadyFillingGasError(recipient_address) - logg.warning(str(AlreadyFillingGasError(recipient_address))) + logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address)))) zero_amount = True session.flush() diff --git a/apps/cic-eth/cic_eth/eth/util.py b/apps/cic-eth/cic_eth/eth/util.py index 71683d82..b1bce1e5 100644 --- a/apps/cic-eth/cic_eth/eth/util.py +++ b/apps/cic-eth/cic_eth/eth/util.py @@ -3,10 +3,11 @@ import logging import sha3 import web3 -# third-party imports +# external imports from rlp import decode as rlp_decode from rlp import encode as rlp_encode from eth_keys import KeyAPI +from chainlib.eth.tx import unpack logg = logging.getLogger() @@ -22,64 +23,65 @@ field_debugs = [ 's', ] +unpack_signed_raw_tx = unpack -def unpack_signed_raw_tx(tx_raw_bytes, chain_id): - d = rlp_decode(tx_raw_bytes) - - logg.debug('decoding using chain id {}'.format(chain_id)) - j = 0 - for i in d: - logg.debug('decoded {}: {}'.format(field_debugs[j], i.hex())) - j += 1 - vb = chain_id - if chain_id != 0: - v = int.from_bytes(d[6], 'big') - vb = v - (chain_id * 2) - 35 - while len(d[7]) < 32: - d[7] = b'\x00' + d[7] - while len(d[8]) < 32: - d[8] = b'\x00' + d[8] - s = b''.join([d[7], d[8], bytes([vb])]) - so = KeyAPI.Signature(signature_bytes=s) - - h = sha3.keccak_256() - h.update(rlp_encode(d)) - signed_hash = h.digest() - - d[6] = chain_id - d[7] = b'' - d[8] = b'' - - h = sha3.keccak_256() - h.update(rlp_encode(d)) - unsigned_hash = h.digest() - - p = so.recover_public_key_from_msg_hash(unsigned_hash) - a = p.to_checksum_address() - logg.debug('decoded recovery byte {}'.format(vb)) - logg.debug('decoded address {}'.format(a)) - logg.debug('decoded signed hash {}'.format(signed_hash.hex())) - logg.debug('decoded unsigned hash {}'.format(unsigned_hash.hex())) - - to = d[3].hex() or None - if to != None: - to = web3.Web3.toChecksumAddress('0x' + to) - - return { - 'from': a, - 'nonce': int.from_bytes(d[0], 'big'), - 'gasPrice': int.from_bytes(d[1], 'big'), - 'gas': int.from_bytes(d[2], 'big'), - 'to': to, - 'value': int.from_bytes(d[4], 'big'), - 'data': '0x' + d[5].hex(), - 'v': chain_id, - 'r': '0x' + s[:32].hex(), - 's': '0x' + s[32:64].hex(), - 'chainId': chain_id, - 'hash': '0x' + signed_hash.hex(), - 'hash_unsigned': '0x' + unsigned_hash.hex(), - } +#def unpack_signed_raw_tx(tx_raw_bytes, chain_id): +# d = rlp_decode(tx_raw_bytes) +# +# logg.debug('decoding {} using chain id {}'.format(tx_raw_bytes.hex(), chain_id)) +# j = 0 +# for i in d: +# logg.debug('decoded {}: {}'.format(field_debugs[j], i.hex())) +# j += 1 +# vb = chain_id +# if chain_id != 0: +# v = int.from_bytes(d[6], 'big') +# vb = v - (chain_id * 2) - 35 +# while len(d[7]) < 32: +# d[7] = b'\x00' + d[7] +# while len(d[8]) < 32: +# d[8] = b'\x00' + d[8] +# s = b''.join([d[7], d[8], bytes([vb])]) +# so = KeyAPI.Signature(signature_bytes=s) +# +# h = sha3.keccak_256() +# h.update(rlp_encode(d)) +# signed_hash = h.digest() +# +# d[6] = chain_id +# d[7] = b'' +# d[8] = b'' +# +# h = sha3.keccak_256() +# h.update(rlp_encode(d)) +# unsigned_hash = h.digest() +# +# p = so.recover_public_key_from_msg_hash(unsigned_hash) +# a = p.to_checksum_address() +# logg.debug('decoded recovery byte {}'.format(vb)) +# logg.debug('decoded address {}'.format(a)) +# logg.debug('decoded signed hash {}'.format(signed_hash.hex())) +# logg.debug('decoded unsigned hash {}'.format(unsigned_hash.hex())) +# +# to = d[3].hex() or None +# if to != None: +# to = web3.Web3.toChecksumAddress('0x' + to) +# +# return { +# 'from': a, +# 'nonce': int.from_bytes(d[0], 'big'), +# 'gasPrice': int.from_bytes(d[1], 'big'), +# 'gas': int.from_bytes(d[2], 'big'), +# 'to': to, +# 'value': int.from_bytes(d[4], 'big'), +# 'data': '0x' + d[5].hex(), +# 'v': chain_id, +# 'r': '0x' + s[:32].hex(), +# 's': '0x' + s[32:64].hex(), +# 'chainId': chain_id, +# 'hash': '0x' + signed_hash.hex(), +# 'hash_unsigned': '0x' + unsigned_hash.hex(), +# } def unpack_signed_raw_tx_hex(tx_raw_hex, chain_id): diff --git a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py index a4d695b8..7f5cf7c2 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py @@ -15,6 +15,8 @@ import web3 from web3 import HTTPProvider, WebsocketProvider from cic_registry import CICRegistry from cic_registry.chain import ChainSpec +from chainlib.eth.tx import unpack +from hexathon import strip_0x # local imports import cic_eth @@ -36,7 +38,7 @@ from cic_eth.error import ( TemporaryTxError, NotLocalTxError, ) -from cic_eth.eth.util import unpack_signed_raw_tx_hex +#from cic_eth.eth.util import unpack_signed_raw_tx_hex logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -115,7 +117,9 @@ class DispatchSyncer: chain_str = str(self.chain_spec) for k in txs.keys(): tx_raw = txs[k] - tx = unpack_signed_raw_tx_hex(tx_raw, self.chain_spec.chain_id()) + #tx = unpack_signed_raw_tx_hex(tx_raw, self.chain_spec.chain_id()) + tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw)) + tx = unpack(tx_raw_bytes, self.chain_spec.chain_id()) try: set_dequeue(tx['hash']) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py index 2d0a8349..8d0e3124 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py @@ -26,7 +26,6 @@ class RegistrationFilter(SyncFilter): def filter(self, conn, block, tx, db_session=None): registered_address = None - logg.debug('register filter checking log {}'.format(tx.logs)) for l in tx.logs: event_topic_hex = l['topics'][0] if event_topic_hex == account_registry_add_log_hash: @@ -34,16 +33,23 @@ class RegistrationFilter(SyncFilter): address_hex = strip_0x(l['topics'][1])[64-40:] address = to_checksum(add_0x(address_hex)) - logg.debug('request token gift to {}'.format(address)) - s = celery.signature( - 'cic_eth.eth.account.gift', + logg.info('request token gift to {}'.format(address)) + s_nonce = celery.signature( + 'cic_eth.eth.tx.reserve_nonce', [ address, + ], + queue=self.queue, + ) + s_gift = celery.signature( + 'cic_eth.eth.account.gift', + [ str(self.chain_spec), ], queue=self.queue, ) - s.apply_async() + s_nonce.link(s_gift) + s_nonce.apply_async() def __str__(self): diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py index ad921b47..82ae9a4f 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py @@ -66,14 +66,21 @@ class TransferAuthFilter(SyncFilter): sender = add_0x(to_checksum(o['sender'])) recipient = add_0x(to_checksum(recipient)) token = add_0x(to_checksum(o['token'])) - s = celery.signature( + token_data = { + 'address': token, + } + + s_nonce = celery.signature( + 'cic_eth.eth.tx.reserve_nonce', + [ + [token_data], + sender, + ], + queue=self.queue, + ) + s_approve = celery.signature( 'cic_eth.eth.token.approve', [ - [ - { - 'address': token, - }, - ], sender, recipient, o['value'], @@ -81,7 +88,8 @@ class TransferAuthFilter(SyncFilter): ], queue=self.queue, ) - t = s.apply_async() + s_nonce.link(s_approve) + t = s_nonce.apply_async() return True diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py index dbee5025..dfa97839 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py @@ -30,7 +30,7 @@ class TxFilter(SyncFilter): if otx == None: logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex)) return None - logg.info('local tx match {}'.format(otx.tx_hash)) + logg.info('tx filter match on {}'.format(otx.tx_hash)) SessionBase.release_session(db_session) s = celery.signature( 'cic_eth.queue.tx.set_final_status', diff --git a/apps/cic-eth/cic_eth/runnable/view.py b/apps/cic-eth/cic_eth/runnable/view.py index 40e10cce..144f0940 100644 --- a/apps/cic-eth/cic_eth/runnable/view.py +++ b/apps/cic-eth/cic_eth/runnable/view.py @@ -43,6 +43,7 @@ argparser = argparse.ArgumentParser() argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)') argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address') argparser.add_argument('-f', '--format', dest='f', default='terminal', type=str, help='Output format') +argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output statis bit enum names only') argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') @@ -122,7 +123,7 @@ def render_tx(o, **kwargs): for v in o.get('status_log', []): d = datetime.datetime.fromisoformat(v[0]) - e = status_str(v[1]) + e = status_str(v[1], args.status_raw) content += '{}: {}\n'.format(d, e) return content diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 99c2750f..9b4ed466 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 10, 0, - 'alpha.37', + 'alpha.38', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/docker/start_tasker.sh b/apps/cic-eth/docker/start_tasker.sh index f33098d3..02c36fa0 100644 --- a/apps/cic-eth/docker/start_tasker.sh +++ b/apps/cic-eth/docker/start_tasker.sh @@ -6,7 +6,7 @@ set -e # set CONFINI_ENV_PREFIX to override the env prefix to override env vars echo "!!! starting signer" -python /usr/local/bin/crypto-dev-daemon -vv -c /usr/local/etc/crypto-dev-signer & +python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer & echo "!!! starting tracker" /usr/local/bin/cic-eth-taskerd $@ diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index bf832fbe..96b4c93e 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -19,6 +19,6 @@ eth-gas-proxy==0.0.1a4 websocket-client==0.57.0 moolb~=0.1.1b2 eth-address-index~=0.1.0a8 -chainlib~=0.0.1a19 +chainlib~=0.0.1a20 hexathon~=0.0.1a3 chainsyncer~=0.0.1a19 diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index 1cb6cbb5..fa1cf34d 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,3 +1,3 @@ cic-base[full_graph]==0.1.1a12 -cic-eth==0.10.0a37 +cic-eth==0.10.0a38 cic-types==0.1.0a8 diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index b0ca84d7..dda61b9e 100644 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -31,7 +31,7 @@ set -e set -a # We need to not install these here... -pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a37 chainlib==0.0.1a19 cic-contracts==0.0.2a2 +pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a38 chainlib==0.0.1a19 cic-contracts==0.0.2a2 pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10 >&2 echo "create account for gas gifter" diff --git a/apps/requirements.txt b/apps/requirements.txt index 97ffa2bb..51a413eb 100644 --- a/apps/requirements.txt +++ b/apps/requirements.txt @@ -42,6 +42,6 @@ rlp==2.0.1 cryptocurrency-cli-tools==0.0.4 giftable-erc20-token==0.0.7b12 hexathon==0.0.1a3 -chainlib==0.0.1a19 +chainlib==0.0.1a20 chainsyncer==0.0.1a19 cic-registry==0.5.3.a22 diff --git a/docker-compose.yml b/docker-compose.yml index 3c8b4426..2cc7b178 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: # PGDATA: /tmp/cic/postgres ports: - ${DEV_POSTGRES_PORT:-63432}:5432 + command: [ "-c", "max_connections=200" ] volumes: - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql - ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh @@ -237,7 +238,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tasker.sh -q cic-eth -v + ./start_tasker.sh -q cic-eth # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ] cic-eth-tracker: