Fix signature length error in unpack

apps/cic-eth/cic_eth/api/api_admin.py

@@ -1,5 +1,6 @@
 # standard imports
 import logging
+import sys
 
 # third-party imports
 import celery
@@ -317,6 +318,8 @@ class AdminApi:
         :return: Transaction details
         :rtype: dict
         """
+        problems = []
+
         if tx_hash != None and tx_raw != None:
             ValueError('Specify only one of hash or raw tx')
 
@@ -444,10 +447,12 @@ class AdminApi:
             r = c.w3.eth.getTransactionReceipt(tx_hash)
             if r.status == 1:
                 tx['network_status'] = 'Confirmed'
-                tx['block'] = r.blockNumber
-                tx['tx_index'] = r.transactionIndex
             else:
                 tx['network_status'] = 'Reverted'
+            tx['network_block_number'] = r.blockNumber
+            tx['network_tx_index'] = r.transactionIndex
+            if tx['block_number'] == None:
+                problems.append('Queue is missing block number {} for mined tx'.format(r.blockNumber))
         except web3.exceptions.TransactionNotFound:
             pass
 
@@ -469,4 +474,9 @@ class AdminApi:
         t = s.apply_async()
         tx['status_log'] = t.get()
 
+        if len(problems) > 0:
+            sys.stderr.write('\n')
+            for p in problems:
+                sys.stderr.write('!!!{}\n'.format(p))
+
         return tx

apps/cic-eth/cic_eth/db/enum.py

@@ -103,6 +103,9 @@ def status_str(v, bits_only=False):
         except ValueError:
             pass
 
+    if v == 0:
+        return 'NONE'
+
     for i in range(16):
         b = (1 << i)
         if (b & 0xffff) & v:

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -115,7 +115,6 @@ class Nonce(SessionBase):
         return nonce
 
 
-
 class NonceReservation(SessionBase):
 
     __tablename__ = 'nonce_task_reservation'

apps/cic-eth/cic_eth/db/models/otx.py

@@ -400,7 +400,7 @@ class Otx(SessionBase):
             raise TxStateChangeError('CANCEL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
 
         if confirmed:
-            if not self.status & StatusBits.OBSOLETE:
+            if self.status > 0 and not self.status & StatusBits.OBSOLETE:
                 raise TxStateChangeError('CANCEL can only be set on an entry marked OBSOLETE ({})'.format(status_str(self.status)))
             self.__set_status(StatusEnum.CANCELLED, session)
         else:

apps/cic-eth/cic_eth/eth/tx.py

@@ -118,7 +118,7 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
     safe_gas = c.safe_threshold_amount()
     if balance < safe_gas:
         s_nonce = celery.signature(
-            'cic_eth.eth.tx.reserve_nonce'
+            'cic_eth.eth.tx.reserve_nonce',
             [
                 address,
                 c.gas_provider(),
@@ -304,7 +304,6 @@ class ParityNodeHandler:
         s_debug = celery.signature(
             'cic_eth.admin.debug.alert',
             [
-                tx_hash_hex,
                 tx_hash_hex,
                 debugstr,
                 ],
@@ -337,7 +336,6 @@ class ParityNodeHandler:
         s_debug = celery.signature(
             'cic_eth.admin.debug.alert',
             [
-                tx_hash_hex,
                 tx_hash_hex,
                 debugstr,
                 ],
@@ -446,7 +444,7 @@ def refill_gas(self, recipient_address, chain_str):
     if c > 0:
         #session.close()
         #raise AlreadyFillingGasError(recipient_address)
-        logg.warning(str(AlreadyFillingGasError(recipient_address)))
+        logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address))))
         zero_amount = True
     session.flush()
 

apps/cic-eth/cic_eth/eth/util.py

@@ -3,10 +3,11 @@ import logging
 import sha3
 import web3
 
-# third-party imports
+# external imports
 from rlp import decode as rlp_decode
 from rlp import encode as rlp_encode
 from eth_keys import KeyAPI
+from chainlib.eth.tx import unpack
 
 logg = logging.getLogger()
 
@@ -22,64 +23,65 @@ field_debugs = [
         's',
         ]
 
+unpack_signed_raw_tx = unpack
 
-def unpack_signed_raw_tx(tx_raw_bytes, chain_id):
+#def unpack_signed_raw_tx(tx_raw_bytes, chain_id):
-    d = rlp_decode(tx_raw_bytes)
+#    d = rlp_decode(tx_raw_bytes)
-
+#
-    logg.debug('decoding using chain id {}'.format(chain_id))
+#    logg.debug('decoding {} using chain id {}'.format(tx_raw_bytes.hex(), chain_id))
-    j = 0
+#    j = 0
-    for i in d:
+#    for i in d:
-        logg.debug('decoded {}: {}'.format(field_debugs[j], i.hex()))
+#        logg.debug('decoded {}: {}'.format(field_debugs[j], i.hex()))
-        j += 1
+#        j += 1
-    vb = chain_id
+#    vb = chain_id
-    if chain_id != 0:
+#    if chain_id != 0:
-        v = int.from_bytes(d[6], 'big')
+#        v = int.from_bytes(d[6], 'big')
-        vb = v - (chain_id * 2) - 35
+#        vb = v - (chain_id * 2) - 35
-    while len(d[7]) < 32:
+#    while len(d[7]) < 32:
-        d[7] = b'\x00' + d[7]
+#        d[7] = b'\x00' + d[7]
-    while len(d[8]) < 32:
+#    while len(d[8]) < 32:
-        d[8] = b'\x00' + d[8]
+#        d[8] = b'\x00' + d[8]
-    s = b''.join([d[7], d[8], bytes([vb])])
+#    s = b''.join([d[7], d[8], bytes([vb])])
-    so = KeyAPI.Signature(signature_bytes=s)
+#    so = KeyAPI.Signature(signature_bytes=s)
-
+#
-    h = sha3.keccak_256()
+#    h = sha3.keccak_256()
-    h.update(rlp_encode(d))
+#    h.update(rlp_encode(d))
-    signed_hash = h.digest()
+#    signed_hash = h.digest()
-
+#
-    d[6] = chain_id
+#    d[6] = chain_id
-    d[7] = b''
+#    d[7] = b''
-    d[8] = b''
+#    d[8] = b''
-
+#
-    h = sha3.keccak_256()
+#    h = sha3.keccak_256()
-    h.update(rlp_encode(d))
+#    h.update(rlp_encode(d))
-    unsigned_hash = h.digest()
+#    unsigned_hash = h.digest()
-    
+#    
-    p = so.recover_public_key_from_msg_hash(unsigned_hash)
+#    p = so.recover_public_key_from_msg_hash(unsigned_hash)
-    a = p.to_checksum_address()
+#    a = p.to_checksum_address()
-    logg.debug('decoded recovery byte {}'.format(vb))
+#    logg.debug('decoded recovery byte {}'.format(vb))
-    logg.debug('decoded address {}'.format(a))
+#    logg.debug('decoded address {}'.format(a))
-    logg.debug('decoded signed hash {}'.format(signed_hash.hex()))
+#    logg.debug('decoded signed hash {}'.format(signed_hash.hex()))
-    logg.debug('decoded unsigned hash {}'.format(unsigned_hash.hex()))
+#    logg.debug('decoded unsigned hash {}'.format(unsigned_hash.hex()))
-
+#
-    to = d[3].hex() or None
+#    to = d[3].hex() or None
-    if to != None:
+#    if to != None:
-        to = web3.Web3.toChecksumAddress('0x' + to)
+#        to = web3.Web3.toChecksumAddress('0x' + to)
-
+#
-    return {
+#    return {
-        'from': a,
+#        'from': a,
-        'nonce': int.from_bytes(d[0], 'big'),
+#        'nonce': int.from_bytes(d[0], 'big'),
-        'gasPrice': int.from_bytes(d[1], 'big'),
+#        'gasPrice': int.from_bytes(d[1], 'big'),
-        'gas': int.from_bytes(d[2], 'big'),
+#        'gas': int.from_bytes(d[2], 'big'),
-        'to': to, 
+#        'to': to, 
-        'value': int.from_bytes(d[4], 'big'),
+#        'value': int.from_bytes(d[4], 'big'),
-        'data': '0x' + d[5].hex(),
+#        'data': '0x' + d[5].hex(),
-        'v': chain_id,
+#        'v': chain_id,
-        'r': '0x' + s[:32].hex(),
+#        'r': '0x' + s[:32].hex(),
-        's': '0x' + s[32:64].hex(),
+#        's': '0x' + s[32:64].hex(),
-        'chainId': chain_id,
+#        'chainId': chain_id,
-        'hash': '0x' + signed_hash.hex(),
+#        'hash': '0x' + signed_hash.hex(),
-        'hash_unsigned': '0x' + unsigned_hash.hex(),
+#        'hash_unsigned': '0x' + unsigned_hash.hex(),
-            }
+#            }
 
 
 def unpack_signed_raw_tx_hex(tx_raw_hex, chain_id):

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -15,6 +15,8 @@ import web3
 from web3 import HTTPProvider, WebsocketProvider
 from cic_registry import CICRegistry
 from cic_registry.chain import ChainSpec
+from chainlib.eth.tx import unpack
+from hexathon import strip_0x
 
 # local imports
 import cic_eth
@@ -36,7 +38,7 @@ from cic_eth.error import (
         TemporaryTxError,
         NotLocalTxError,
         )
-from cic_eth.eth.util import unpack_signed_raw_tx_hex
+#from cic_eth.eth.util import unpack_signed_raw_tx_hex
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -115,7 +117,9 @@ class DispatchSyncer:
         chain_str = str(self.chain_spec)
         for k in txs.keys():
             tx_raw = txs[k]
-            tx = unpack_signed_raw_tx_hex(tx_raw, self.chain_spec.chain_id())
+            #tx = unpack_signed_raw_tx_hex(tx_raw, self.chain_spec.chain_id())
+            tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
+            tx = unpack(tx_raw_bytes, self.chain_spec.chain_id())
             
             try:
                 set_dequeue(tx['hash'])

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -26,7 +26,6 @@ class RegistrationFilter(SyncFilter):
 
     def filter(self, conn, block, tx, db_session=None): 
         registered_address = None
-        logg.debug('register filter checking log {}'.format(tx.logs))
         for l in tx.logs:
             event_topic_hex = l['topics'][0]
             if event_topic_hex == account_registry_add_log_hash:
@@ -34,16 +33,23 @@ class RegistrationFilter(SyncFilter):
 
                 address_hex = strip_0x(l['topics'][1])[64-40:]
                 address = to_checksum(add_0x(address_hex))
-                logg.debug('request token gift to {}'.format(address))
+                logg.info('request token gift to {}'.format(address))
-                s = celery.signature(
+                s_nonce = celery.signature(
-                    'cic_eth.eth.account.gift',
+                    'cic_eth.eth.tx.reserve_nonce',
                     [
                         address,
+                        ],
+                    queue=self.queue,
+                    )
+                s_gift = celery.signature(
+                    'cic_eth.eth.account.gift',
+                    [
                         str(self.chain_spec),
                         ],
                     queue=self.queue,
                     )
-                s.apply_async()
+                s_nonce.link(s_gift)
+                s_nonce.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -66,14 +66,21 @@ class TransferAuthFilter(SyncFilter):
         sender = add_0x(to_checksum(o['sender']))
         recipient = add_0x(to_checksum(recipient))
         token = add_0x(to_checksum(o['token']))
-        s = celery.signature(
+        token_data = {
+            'address': token,
+            }
+
+        s_nonce = celery.signature(
+            'cic_eth.eth.tx.reserve_nonce',
+            [
+                [token_data],
+                sender,
+                ],
+            queue=self.queue,
+            )
+        s_approve = celery.signature(
             'cic_eth.eth.token.approve',
             [
-                [
-                    {
-                        'address': token,
-                        },
-                    ],
                 sender,
                 recipient,
                 o['value'],
@@ -81,7 +88,8 @@ class TransferAuthFilter(SyncFilter):
                 ],
             queue=self.queue,
             )
-        t = s.apply_async()
+        s_nonce.link(s_approve)
+        t = s_nonce.apply_async()
         return True
 
 

apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py

@@ -30,7 +30,7 @@ class TxFilter(SyncFilter):
         if otx == None:
             logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
             return None
-        logg.info('local tx match {}'.format(otx.tx_hash))
+        logg.info('tx filter match on {}'.format(otx.tx_hash))
         SessionBase.release_session(db_session)
         s = celery.signature(
                 'cic_eth.queue.tx.set_final_status',

apps/cic-eth/cic_eth/runnable/view.py

@@ -43,6 +43,7 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)')
 argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
 argparser.add_argument('-f', '--format', dest='f', default='terminal', type=str, help='Output format')
+argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output statis bit enum names only')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@@ -122,7 +123,7 @@ def render_tx(o, **kwargs):
     
     for v in o.get('status_log', []):
         d = datetime.datetime.fromisoformat(v[0])
-        e = status_str(v[1])
+        e = status_str(v[1], args.status_raw)
         content += '{}: {}\n'.format(d, e)
 
     return content

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         0,
-        'alpha.37',
+        'alpha.38',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/start_tasker.sh

@@ -6,7 +6,7 @@ set -e
 # set CONFINI_ENV_PREFIX to override the env prefix to override env vars
 
 echo "!!! starting signer"
-python /usr/local/bin/crypto-dev-daemon -vv -c /usr/local/etc/crypto-dev-signer &
+python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer &
 
 echo "!!! starting tracker"
 /usr/local/bin/cic-eth-taskerd $@

apps/cic-eth/requirements.txt

@@ -19,6 +19,6 @@ eth-gas-proxy==0.0.1a4
 websocket-client==0.57.0
 moolb~=0.1.1b2
 eth-address-index~=0.1.0a8
-chainlib~=0.0.1a19
+chainlib~=0.0.1a20
 hexathon~=0.0.1a3
 chainsyncer~=0.0.1a19

apps/contract-migration/scripts/requirements.txt

@@ -1,3 +1,3 @@
 cic-base[full_graph]==0.1.1a12
-cic-eth==0.10.0a37
+cic-eth==0.10.0a38
 cic-types==0.1.0a8

apps/contract-migration/seed_cic_eth.sh

@@ -31,7 +31,7 @@ set -e
 set -a
 
 # We need to not install these here...
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a37 chainlib==0.0.1a19 cic-contracts==0.0.2a2
+pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a38 chainlib==0.0.1a19 cic-contracts==0.0.2a2
 pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10
 
 >&2 echo "create account for gas gifter"

apps/requirements.txt

@@ -42,6 +42,6 @@ rlp==2.0.1
 cryptocurrency-cli-tools==0.0.4
 giftable-erc20-token==0.0.7b12
 hexathon==0.0.1a3
-chainlib==0.0.1a19
+chainlib==0.0.1a20
 chainsyncer==0.0.1a19
 cic-registry==0.5.3.a22

docker-compose.yml

@@ -50,6 +50,7 @@ services:
       # PGDATA: /tmp/cic/postgres
     ports:
       - ${DEV_POSTGRES_PORT:-63432}:5432
+    command: [ "-c", "max_connections=200" ]
     volumes:
       - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql 
       - ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh
@@ -237,7 +238,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_tasker.sh -q cic-eth -v
+        ./start_tasker.sh -q cic-eth 
     # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ]
 
   cic-eth-tracker: