Rehabilitate admin api tests (except nonce manip)

apps/cic-eth/cic_eth/api/api_admin.py

@@ -6,10 +6,18 @@ import sys
 import celery
 from chainlib.eth.constant import (
         ZERO_ADDRESS,
-        ZERO_CONTENT,
         )
+from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.eth.address import to_checksum_address
+from chainlib.eth.contract import code
+from chainlib.eth.tx import (
+        transaction,
+        receipt,
+        unpack,
+        )
+from hexathon import strip_0x
+from chainlib.eth.gas import balance
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -39,18 +47,20 @@ class AdminApi:
     :param queue: Name of worker queue to submit tasks to
     :type queue: str
     """
-    def __init__(self, rpc, queue='cic-eth'):
+    def __init__(self, rpc, queue='cic-eth', call_address=ZERO_ADDRESS):
         self.rpc = rpc
         self.queue = queue
+        self.call_address = call_address
 
 
     def unlock(self, chain_spec, address, flags=None):
         s_unlock = celery.signature(
             'cic_eth.admin.ctrl.unlock',
             [
-                str(chain_spec),
+                None,
-                flags,
+                chain_spec.asdict(),
                 address,
+                flags,
                 ],
             queue=self.queue,
             )
@@ -61,9 +71,10 @@ class AdminApi:
         s_lock = celery.signature(
             'cic_eth.admin.ctrl.lock',
             [
-                str(chain_spec),
+                None,
-                flags,
+                chain_spec.asdict(),
                 address,
+                flags,
                 ],
             queue=self.queue,
             )
@@ -76,7 +87,7 @@ class AdminApi:
             [],
             queue=self.queue,
             )
-        return s_lock.apply_async().get()
+        return s_lock.apply_async()
 
 
     def tag_account(self, tag, address_hex, chain_spec):
@@ -99,22 +110,19 @@ class AdminApi:
                 ],
             queue=self.queue,
             )
-        t = s_tag.apply_async()
+        return s_tag.apply_async()
-        logg.debug('taaag {}'.format(t))
-        return t.get()
 
 
-    def have_account(self, address_hex, chain_str):
+    def have_account(self, address_hex, chain_spec):
         s_have = celery.signature(
             'cic_eth.eth.account.have',
             [
                 address_hex,
-                chain_str,
+                chain_spec.asdict(),
                 ],
             queue=self.queue,
             )
-        t = s_have.apply_async()
+        return s_have.apply_async()
-        return t.get()
 
 
     def resend(self, tx_hash_hex, chain_str, in_place=True, unlock=False):
@@ -215,7 +223,7 @@ class AdminApi:
             }
 
 
-    def fix_nonce(self, address, nonce):
+    def fix_nonce(self, address, nonce, chain_spec):
         s = celery.signature(
                 'cic_eth.queue.tx.get_account_tx',
                 [
@@ -236,7 +244,7 @@ class AdminApi:
         s_nonce = celery.signature(
                 'cic_eth.admin.nonce.shift_nonce',
                 [
-                    str(self.rpc_client.chain_spec),
+                    self.rpc.chain_spec.asdict(),
                     tx_hash_hex, 
                 ],
                 queue=self.queue
@@ -244,18 +252,18 @@ class AdminApi:
         return s_nonce.apply_async()
 
 
-    # TODO: this is a stub, complete all checks
+#    # TODO: this is a stub, complete all checks
-    def ready(self):
+#    def ready(self):
-        """Checks whether all required initializations have been performed.
+#        """Checks whether all required initializations have been performed.
-
+#
-        :raises cic_eth.error.InitializationError: At least one setting pre-requisite has not been met.
+#        :raises cic_eth.error.InitializationError: At least one setting pre-requisite has not been met.
-        :raises KeyError: An address provided for initialization is not known by the keystore.
+#        :raises KeyError: An address provided for initialization is not known by the keystore.
-        """
+#        """
-        addr = AccountRole.get_address('ETH_GAS_PROVIDER_ADDRESS')
+#        addr = AccountRole.get_address('ETH_GAS_PROVIDER_ADDRESS')
-        if addr == zero_address:
+#        if addr == ZERO_ADDRESS:
-            raise InitializationError('missing account ETH_GAS_PROVIDER_ADDRESS')
+#            raise InitializationError('missing account ETH_GAS_PROVIDER_ADDRESS')
-
+#
-        self.w3.eth.sign(addr, text='666f6f')
+#        self.w3.eth.sign(addr, text='666f6f')
 
 
     def account(self, chain_spec, address, cols=['tx_hash', 'sender', 'recipient', 'nonce', 'block', 'tx_index', 'status', 'network_status', 'date_created'], include_sender=True, include_recipient=True):
@@ -337,7 +345,7 @@ class AdminApi:
         tx = s.apply_async().get()
   
         source_token = None
-        if tx['source_token'] != zero_address:
+        if tx['source_token'] != ZERO_ADDRESS:
             try:
                 source_token = CICRegistry.get_address(chain_spec, tx['source_token']).contract
             except UnknownContractError:
@@ -346,7 +354,7 @@ class AdminApi:
                 logg.warning('unknown source token contract {}'.format(tx['source_token']))
 
         destination_token = None
-        if tx['source_token'] != zero_address:
+        if tx['source_token'] != ZERO_ADDRESS:
             try:
                 destination_token = CICRegistry.get_address(chain_spec, tx['destination_token'])
             except UnknownContractError:
@@ -357,10 +365,13 @@ class AdminApi:
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
 
-        c = RpcClient(chain_spec)
+        registry = CICRegistry(chain_spec, self.rpc)
-        if len(c.w3.eth.getCode(tx['sender'])) > 0:
+        o = code(tx['sender'])
+        r = self.rpc.do(o)
+        if len(strip_0x(r, allow_empty=True)) > 0:
             try:
-                sender_contract = CICRegistry.get_address(chain_spec, tx['sender'])
+                #sender_contract = CICRegistry.get_address(chain_spec, tx['sender'])
+                sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
                 tx['sender_description'] = 'Contract {}'.format(sender_contract.identifier())
             except UnknownContractError:
                 tx['sender_description'] = 'Unknown contract'
@@ -371,7 +382,7 @@ class AdminApi:
                     'cic_eth.eth.account.have',
                     [
                         tx['sender'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -384,7 +395,7 @@ class AdminApi:
                     'cic_eth.eth.account.role',
                     [
                         tx['sender'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -393,8 +404,9 @@ class AdminApi:
                 if role != None:
                     tx['sender_description'] = role
 
-
+        o = code(tx['recipient'])
-        if len(c.w3.eth.getCode(tx['recipient'])) > 0:
+        r = self.rpc.do(o)
+        if len(strip_0x(r, allow_empty=True)) > 0:
             try:
                 recipient_contract = CICRegistry.get_address(chain_spec, tx['recipient'])
                 tx['recipient_description'] = 'Contract {}'.format(recipient_contract.identifier())
@@ -407,7 +419,7 @@ class AdminApi:
                     'cic_eth.eth.account.have',
                     [
                         tx['recipient'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -420,7 +432,7 @@ class AdminApi:
                     'cic_eth.eth.account.role',
                     [
                         tx['recipient'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -439,29 +451,39 @@ class AdminApi:
 
         tx['network_status'] = 'Not submitted'
 
+        r = None
         try:
-            c.w3.eth.getTransaction(tx_hash)
+            o = transaction(tx_hash)
+            r = self.rpc.do(o)
+        except Exception as e:
+            logg.warning('too permissive exception handler, please fix!')
             tx['network_status'] = 'Mempool'
-        except web3.exceptions.TransactionNotFound:
-            pass
 
-        try:
+        if r != None:
-            r = c.w3.eth.getTransactionReceipt(tx_hash)
+            try:
-            if r.status == 1:
+                o = receipt(tx_hash)
-                tx['network_status'] = 'Confirmed'
+                r = self.rpc.do(o)
-            else:
+                if r['status'] == 1:
-                tx['network_status'] = 'Reverted'
+                    tx['network_status'] = 'Confirmed'
-            tx['network_block_number'] = r.blockNumber
+                else:
-            tx['network_tx_index'] = r.transactionIndex
+                    tx['network_status'] = 'Reverted'
-            if tx['block_number'] == None:
+                tx['network_block_number'] = r.blockNumber
-                problems.append('Queue is missing block number {} for mined tx'.format(r.blockNumber))
+                tx['network_tx_index'] = r.transactionIndex
-        except web3.exceptions.TransactionNotFound:
+                if tx['block_number'] == None:
-            pass
+                    problems.append('Queue is missing block number {} for mined tx'.format(r.blockNumber))
+            except Exception as e:
+                logg.warning('too permissive exception handler, please fix!')
+                pass
 
-        tx['sender_gas_balance'] = c.w3.eth.getBalance(tx['sender'])
+        o = balance(tx['sender'])
-        tx['recipient_gas_balance'] = c.w3.eth.getBalance(tx['recipient'])
+        r = self.rpc.do(o)
+        tx['sender_gas_balance'] = r
 
-        tx_unpacked = unpack_signed_raw_tx(bytes.fromhex(tx['signed_tx'][2:]), chain_spec.chain_id())
+        o = balance(tx['recipient'])
+        r = self.rpc.do(o)
+        tx['recipient_gas_balance'] = r
+
+        tx_unpacked = unpack(bytes.fromhex(tx['signed_tx'][2:]), chain_spec.chain_id())
         tx['gas_price'] = tx_unpacked['gasPrice']
         tx['gas_limit'] = tx_unpacked['gas']
         tx['data'] = tx_unpacked['data']

apps/cic-eth/cic_eth/eth/account.py

@@ -234,7 +234,6 @@ def have(self, account, chain_spec_dict):
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):
-    logg.debug('foo fooofoo')
     if not to_checksum_address(address):
         raise ValueError('invalid checksum address {}'.format(address))
     session = SessionBase.create_session()

apps/cic-eth/requirements.txt

@@ -18,7 +18,7 @@ semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
 eth-address-index~=0.1.1a4
-chainlib~=0.0.1a32
+chainlib~=0.0.1a33
 hexathon~=0.0.1a5
 chainsyncer~=0.0.1a20
 pysha3==1.0.2

apps/cic-eth/test_requirements.txt

@@ -4,3 +4,4 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
+giftable-erc20-token==0.0.8a4

apps/cic-eth/tests/fixtures_role.py

@@ -37,6 +37,7 @@ def custodial_roles(
     token_roles,
     agent_roles,
     eth_accounts,
+    eth_keystore,
     init_database,
     ):
     r = {}

apps/cic-eth/tests/task/api/test_admin.py

@@ -0,0 +1,294 @@
+# standard imports
+import os
+import logging
+
+# external imports
+import celery
+import pytest
+from chainlib.eth.tx import (
+        unpack,
+        TxFormat,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import Gas
+from chainlib.eth.address import to_checksum_address
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+
+# local imports
+from cic_eth.api import AdminApi
+from cic_eth.db.models.role import AccountRole
+from cic_eth.db.models.otx import Otx
+from cic_eth.db.models.tx import TxCache
+from cic_eth.db.enum import (
+        StatusEnum,
+        StatusBits,
+        status_str,
+        LockEnum,
+        )
+from cic_eth.error import InitializationError
+from cic_eth.eth.tx import (
+        cache_gas_data,
+        )
+#from cic_eth.eth.gas import cache_gas_tx
+from cic_eth.queue.tx import (
+        create as queue_create,
+        get_tx,
+        )
+
+logg = logging.getLogger()
+
+
+#def test_resend_inplace(
+#    default_chain_spec,
+#    init_database,
+#    init_w3,
+#    celery_session_worker,
+#    ):
+#    
+#    chain_str = str(default_chain_spec)
+#    c = RpcClient(default_chain_spec)
+#    
+#    sigs = []
+#
+#    gas_provider = c.gas_provider()
+#
+#    s_nonce = celery.signature(
+#        'cic_eth.eth.tx.reserve_nonce',
+#        [
+#            init_w3.eth.accounts[0],
+#            gas_provider,
+#            ],
+#        queue=None,
+#        )
+#    s_refill = celery.signature(
+#        'cic_eth.eth.tx.refill_gas',
+#        [
+#            chain_str,
+#            ],
+#        queue=None,
+#            )
+#    s_nonce.link(s_refill)
+#    t = s_nonce.apply_async()
+#    t.get()
+#    for r in t.collect():
+#        pass
+#    assert t.successful()
+#
+#    q = init_database.query(Otx)
+#    q = q.join(TxCache)
+#    q = q.filter(TxCache.recipient==init_w3.eth.accounts[0])
+#    o = q.first()
+#    tx_raw = o.signed_tx
+#
+#    tx_dict = unpack_signed_raw_tx(bytes.fromhex(tx_raw[2:]), default_chain_spec.chain_id())
+#    gas_price_before = tx_dict['gasPrice'] 
+#
+#    s = celery.signature(
+#        'cic_eth.admin.ctrl.lock_send',
+#        [
+#            chain_str,
+#            init_w3.eth.accounts[0],
+#            ],
+#        queue=None,
+#            )
+#    t = s.apply_async()
+#    t.get()
+#    assert t.successful()
+#
+#    api = AdminApi(c, queue=None)
+#    t = api.resend(tx_dict['hash'], chain_str, unlock=True)
+#    t.get()
+#    i = 0
+#    tx_hash_new_hex = None
+#    for r in t.collect():
+#        tx_hash_new_hex = r[1]
+#    assert t.successful()
+#  
+#    tx_raw_new = get_tx(tx_hash_new_hex) 
+#    logg.debug('get {}'.format(tx_raw_new))
+#    tx_dict_new = unpack_signed_raw_tx(bytes.fromhex(tx_raw_new['signed_tx'][2:]), default_chain_spec.chain_id())
+#    assert tx_hash_new_hex != tx_dict['hash']
+#    assert tx_dict_new['gasPrice'] > gas_price_before
+#
+#    tx_dict_after = get_tx(tx_dict['hash'])
+#
+#    logg.debug('logggg {}'.format(status_str(tx_dict_after['status'])))
+#    assert tx_dict_after['status'] & StatusBits.MANUAL
+
+
+#def test_check_fix_nonce(
+#    default_chain_spec,
+#    init_database,
+#    init_eth_account_roles,
+#    init_w3,
+#    eth_empty_accounts,
+#    celery_session_worker,
+#    ):
+#
+#    chain_str = str(default_chain_spec)
+#    
+#    sigs = []
+#    for i in range(5):
+#        s = celery.signature(
+#            'cic_eth.eth.tx.refill_gas',
+#            [
+#                eth_empty_accounts[i],
+#                chain_str,
+#                ],
+#            queue=None,
+#                )
+#        sigs.append(s)
+#
+#    t = celery.group(sigs)()
+#    txs = t.get()
+#    assert t.successful()
+#
+#    tx_hash = web3.Web3.keccak(hexstr=txs[2])
+#    c = RpcClient(default_chain_spec)
+#    api = AdminApi(c, queue=None)
+#    address = init_eth_account_roles['eth_account_gas_provider']
+#    nonce_spec = api.check_nonce(address)
+#    assert nonce_spec['nonce']['network'] == 0
+#    assert nonce_spec['nonce']['queue'] == 4
+#    assert nonce_spec['nonce']['blocking'] == None
+#
+#    s_set = celery.signature(
+#            'cic_eth.queue.tx.set_rejected',
+#            [
+#                tx_hash.hex(),
+#                ],
+#            queue=None,
+#            )
+#    t = s_set.apply_async()
+#    t.get()
+#    t.collect()
+#    assert t.successful()
+#
+#
+#    nonce_spec = api.check_nonce(address)
+#    assert nonce_spec['nonce']['blocking'] == 2
+#    assert nonce_spec['tx']['blocking'] == tx_hash.hex()
+#
+#    t = api.fix_nonce(address, nonce_spec['nonce']['blocking'])
+#    t.get()
+#    t.collect()
+#    assert t.successful()
+#
+#    for tx in txs[3:]:
+#        tx_hash = web3.Web3.keccak(hexstr=tx)
+#        tx_dict = get_tx(tx_hash.hex())
+#        assert tx_dict['status'] == StatusEnum.OVERRIDDEN
+#
+#
+
+
+def test_have_account(
+    default_chain_spec,
+    custodial_roles,
+    init_celery_tasks,
+    eth_rpc,
+    celery_session_worker,
+    ):
+
+    api = AdminApi(None, queue=None)
+    t = api.have_account(custodial_roles['ALICE'], default_chain_spec)
+    assert t.get() != None 
+
+    bogus_address = add_0x(to_checksum_address(os.urandom(20).hex()))
+    api = AdminApi(None, queue=None)
+    t = api.have_account(bogus_address, default_chain_spec)
+    assert t.get() == None
+
+
+def test_locking(
+    default_chain_spec,
+    init_database,
+    agent_roles,
+    init_celery_tasks,
+    celery_session_worker,
+    ):
+
+    api = AdminApi(None, queue=None)
+
+    t = api.lock(default_chain_spec, agent_roles['ALICE'], LockEnum.SEND)
+    t.get()
+    t = api.get_lock()
+    r = t.get()
+    assert len(r) == 1
+
+    t = api.unlock(default_chain_spec, agent_roles['ALICE'], LockEnum.SEND)
+    t.get()
+    t = api.get_lock()
+    r = t.get()
+    assert len(r) == 0
+
+
+def test_tag_account(
+    default_chain_spec,
+    init_database,
+    agent_roles,
+    eth_rpc,
+    init_celery_tasks,
+    celery_session_worker,
+    ):
+
+    api = AdminApi(eth_rpc, queue=None)
+
+    t = api.tag_account('foo', agent_roles['ALICE'], default_chain_spec)
+    t.get()
+    t = api.tag_account('bar', agent_roles['BOB'], default_chain_spec)
+    t.get()
+    t = api.tag_account('bar', agent_roles['CAROL'], default_chain_spec)
+    t.get()
+
+    assert AccountRole.get_address('foo', init_database) == agent_roles['ALICE']
+    assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
+
+
+#def test_ready(
+#    init_database,
+#    agent_roles,
+#    eth_rpc,
+#    ):
+#
+#    api = AdminApi(eth_rpc)
+#   
+#    with pytest.raises(InitializationError):
+#        api.ready()
+#
+#    bogus_account = os.urandom(20)
+#    bogus_account_hex = '0x' + bogus_account.hex()
+#
+#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex))
+#    with pytest.raises(KeyError):
+#        api.ready()
+#
+#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0])
+#    api.ready()
+
+
+def test_tx(
+    default_chain_spec,
+    cic_registry,
+    init_database,
+    eth_rpc,
+    eth_signer,
+    agent_roles,
+    contract_roles,
+    celery_session_worker,
+    ):
+
+    chain_id = default_chain_spec.chain_id()
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
+    c = Gas(signer=eth_signer, nonce_oracle=nonce_oracle, chain_id=chain_id)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
+    tx = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), chain_id)
+    queue_create(tx['nonce'], agent_roles['ALICE'], tx_hash_hex, tx_signed_raw_hex, default_chain_spec, session=init_database)
+    cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
+    logg.warning('code missing to verify tx contents {}'.format(tx))