Rehabilitate shift nonce, add test

apps/cic-eth/.coveragerc

@@ -5,4 +5,6 @@ omit =
 	cic_eth/db/migrations/*
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
+	cic_eth/eth/state.py
+	cic_eth/admin/nonce.py
 	*redis*.py

apps/cic-eth/cic_eth/admin/nonce.py

@@ -4,11 +4,18 @@ import logging
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.eth.tx import unpack
+from chainlib.connection import RPCConnection
+from chainlib.eth.tx import (
+        unpack,
+        TxFactory,
+        )
+from chainlib.eth.gas import OverrideGasOracle
 from chainqueue.query import get_tx
 from chainqueue.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
+from hexathon import strip_0x
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -38,25 +45,27 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     :type tx_hash_orig_hex: str, 0x-hex
     :param delta: Amount
     """
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     queue = None
     try:
         queue = self.request.delivery_info.get('routing_key')
     except AttributeError:
         pass
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    session = SessionBase.create_session()
-    tx_brief = get_tx(tx_hash_orig_hex)
+    tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
-    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:]))
+    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
     tx = unpack(tx_raw, chain_spec)
     nonce = tx_brief['nonce']
     address = tx['from']
 
     logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce))
 
-    lock_queue(None, chain_str, address)
+    lock_queue(None, chain_spec.asdict(), address=address)
-    lock_send(None, chain_str, address)
+    lock_send(None, chain_spec.asdict(), address=address)
 
-    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==address)
@@ -69,49 +78,61 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
+        tx_new = snake_and_camel(tx_new)
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']
 
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
+        tx_new['gas_price'] += 1
+        tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
 
-        (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
+        logg.debug('tx_nex {}'.format(tx_new))
+        gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
+        c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
+        #(tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
+        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
         logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
 
+#        otx = Otx(
+#            nonce=tx_new['nonce'],
+#            address=tx_new['from'],
+#            tx_hash=tx_hash_hex,
+#            signed_tx=tx_signed_raw_hex,
+#                )
         otx = Otx(
-            nonce=tx_new['nonce'],
+            tx_new['nonce'],
-            address=tx_new['from'],
+            tx_hash_hex,
-            tx_hash=tx_hash_hex,
+            tx_signed_raw_hex,
-            signed_tx=tx_signed_raw_hex,
             )
         session.add(otx)
         session.commit()
 
         # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
-        set_cancel(tx_previous_hash_hex, True)
+        set_cancel(chain_spec, tx_previous_hash_hex, manual=True, session=session)
 
-        TxCache.clone(tx_previous_hash_hex, tx_hash_hex)
+        TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
 
         tx_hashes.append(tx_hash_hex)
         txs.append(tx_signed_raw_hex)
 
     session.close()
 
-    s = create_check_gas_and_send_task(
+    s = create_check_gas_task(
          txs, 
-         chain_str,
+         chain_spec,
          tx_new['from'],
-         tx_new['gas'],
+         gas=tx_new['gas'],
-         tx_hashes, 
+         tx_hashes_hex=tx_hashes, 
-         queue,
+         queue=queue,
         )
 
     s_unlock_send = celery.signature(
         'cic_eth.admin.ctrl.unlock_send',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,
@@ -119,7 +140,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     s_unlock_direct = celery.signature(
         'cic_eth.admin.ctrl.unlock_queue',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,

apps/cic-eth/tests/task/test_task_admin.py

@@ -0,0 +1,88 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.tx import (
+        create as queue_create,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.tx import TxFormat
+from chainqueue.query import get_nonce_tx_cache
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from hexathon import add_0x
+
+# local imports
+from cic_eth.admin.nonce import shift_nonce
+from cic_eth.eth.gas import cache_gas_data
+
+logg = logging.getLogger()
+
+
+def test_shift_nonce(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                42+i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.admin.nonce.shift_nonce',
+            [
+                str(default_chain_spec),
+                tx_hashes[3],
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+
+    for i in range(42+4, 42+10):
+        txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database)
+        for k in txs.keys():
+            hsh = add_0x(k)
+            otx = Otx.load(hsh, session=init_database)
+            logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status))
+            if add_0x(k) == tx_hashes[i-42]:
+                assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+            else:
+                assert otx.status == 1

apps/cic-eth/tests/task/test_task_tx.py

@@ -33,7 +33,6 @@ from cic_eth.eth.gas import cache_gas_data
 logg = logging.getLogger()
 
 
-@pytest.mark.skip()
 def test_tx_send(
         init_database,
         default_chain_spec,
@@ -127,7 +126,6 @@ def test_sync_tx(
     assert o.status & StatusBits.FINAL == StatusBits.FINAL
 
 
-@pytest.mark.skip()
 def test_resend_with_higher_gas(
         init_database,
         default_chain_spec,

apps/cic-eth/tests/task/test_task_tx_misc.py

@@ -6,7 +6,10 @@ import logging
 import pytest
 import celery
 from chainqueue.tx import create as queue_create
-from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.nonce import (
+        RPCNonceOracle,
+        OverrideNonceOracle,
+        )
 from chainlib.eth.gas import (
         OverrideGasOracle,
         Gas,
@@ -28,7 +31,6 @@ from chainqueue.state import (
 logg = logging.getLogger()
 
 
-@pytest.mark.skip()
 def test_hashes_to_txs(
         init_database,
         default_chain_spec,
@@ -99,7 +101,7 @@ def test_hashes_to_txs(
 
 
 
-def test_hashes_to_txs(
+def test_double_send(
         init_database,
         default_chain_spec,
         agent_roles,