Remove eth-abi dep in data seeding

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         0,
-        'alpha.2',
+        'alpha.3',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/tools_requirements.txt

@@ -1,6 +1,6 @@
 crypto-dev-signer~=0.4.14b6
 chainqueue~=0.0.2b5
-confini~=0.3.6rc4
+confini~=0.3.6
 cic-eth-registry~=0.5.6a1
 redis==3.5.3
 hexathon~=0.0.1a7

apps/data-seeding/cic_eth/import_balance.py

@@ -11,7 +11,6 @@ import csv
 import json
 
 # external imports
-import eth_abi
 import confini
 from hexathon import (
         strip_0x,
@@ -29,7 +28,10 @@ from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
 from chainlib.jsonrpc import JSONRPCRequest
-from chainlib.eth.error import EthException
+from chainlib.eth.error import (
+        EthException,
+        RequestMismatchException,
+        )
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
@@ -37,6 +39,9 @@ from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
 from cic_base.eth.syncer import chain_interface
+from eth_accounts_index import AccountsIndex
+from eth_contract_registry import Registry
+from eth_token_index import TokenUniqueSymbolIndex
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -131,58 +136,52 @@ class Handler:
             logg.debug('no payload, skipping {}'.format(tx))
             return
 
-        if tx.payload[:8] == self.account_index_add_signature:
+        recipient = None
-            recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
+        try:
-            #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
+            r = AccountsIndex.parse_add_request(tx.payload)
-            user_file = 'new/{}/{}/{}.json'.format(
+        except RequestMismatchException:
-                    recipient[2:4].upper(),
+            return
-                    recipient[4:6].upper(),
+        recipient = r[0]
-                    recipient[2:].upper(),
+        
-                    )
+        user_file = 'new/{}/{}/{}.json'.format(
-            filepath = os.path.join(self.user_dir, user_file)
+                recipient[2:4].upper(),
-            o = None
+                recipient[4:6].upper(),
-            try:
+                recipient[2:].upper(),
-                f = open(filepath, 'r')
+                )
-                o = json.load(f)
+        filepath = os.path.join(self.user_dir, user_file)
-                f.close()
+        o = None
-            except FileNotFoundError:
+        try:
-                logg.error('no import record of address {}'.format(recipient))
+            f = open(filepath, 'r')
-                return
+            o = json.load(f)
-            u = Person.deserialize(o)
-            original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
-            try:
-                balance = self.balances[original_address]
-            except KeyError as e:
-                logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
-                return
-
-            # TODO: store token object in handler ,get decimals from there
-            multiplier = 10**6
-            balance_full = balance * multiplier
-            logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
-
-            (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
-            logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
-            r = conn.do(o)
-
-            tx_path = os.path.join(
-                    user_dir,
-                    'txs',
-                    strip_0x(tx_hash_hex),
-                    )
-            f = open(tx_path, 'w')
-            f.write(strip_0x(o['params'][0]))
             f.close()
-#        except TypeError as e:
+        except FileNotFoundError:
-#            logg.warning('typerror {}'.format(e))
+            logg.error('no import record of address {}'.format(recipient))
-#            pass
+            return
-#        except IndexError as e:
+        u = Person.deserialize(o)
-#            logg.warning('indexerror {}'.format(e))
+        original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
-#            pass
+        try:
-#        except EthException as e:
+            balance = self.balances[original_address]
-#            logg.error('send error {}'.format(e).ljust(200))
+        except KeyError as e:
-        #except KeyError as e:
+            logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
-        #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
+            return
+
+        # TODO: store token object in handler ,get decimals from there
+        multiplier = 10**6
+        balance_full = balance * multiplier
+        logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
+
+        (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
+        logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
+        r = conn.do(o)
+
+        tx_path = os.path.join(
+                user_dir,
+                'txs',
+                strip_0x(tx_hash_hex),
+                )
+        f = open(tx_path, 'w')
+        f.write(strip_0x(o['params'][0]))
+        f.close()
 
 
 def progress_callback(block_number, tx_index):
@@ -198,49 +197,26 @@ def main():
     nonce_oracle = RPCNonceOracle(signer_address, conn)
 
     # Get Token registry address
-    txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
+    registry = Registry(chain_spec)
-    tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
+    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
-
-    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
-    data = add_0x(registry_addressof_method)
-    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
-    txf.set_code(tx, data)
-   
-    j = JSONRPCRequest()
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
-    token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    token_index_address = registry.parse_address_of(r)
+    token_index_address = to_checksum_address(token_index_address)
     logg.info('found token index address {}'.format(token_index_address))
-
+    
-
     # Get Sarafu token address
-    tx = txf.template(signer_address, token_index_address)
+    token_index = TokenUniqueSymbolIndex(chain_spec)
-    data = add_0x(registry_addressof_method)
+    o = token_index.address_of(token_index_address, token_symbol)
-    h = hashlib.new('sha256')
-    h.update(token_symbol.encode('utf-8'))
-    z = h.digest()
-    data += eth_abi.encode_single('bytes32', z).hex()
-    txf.set_code(tx, data)
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
+    token_address = token_index.parse_address_of(r)
     try:
-        sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+        token_address = to_checksum_address(token_address)
     except ValueError as e:
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
-
+    logg.info('found token address {}'.format(token_address))
-    if sarafu_token_address == ZERO_ADDRESS:
+    
-        raise KeyError('token address for symbol {} is zero'.format(token_symbol))
+    sys.exit(0)
-
-    logg.info('found token address {}'.format(sarafu_token_address))
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -248,22 +224,6 @@ def main():
         o = block_latest()
         r = conn.do(o)
         block_offset = int(strip_0x(r), 16) + 1
-#
-#    addresses = {}
-#    f = open('{}/addresses.csv'.format(user_dir, 'r'))
-#    while True:
-#        l = f.readline()
-#        if l == None:
-#            break
-#        r = l.split(',')
-#        try:
-#            k = r[0]
-#            v = r[1].rstrip()
-#            addresses[k] = v
-#            sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
-#        except IndexError as e:
-#            break
-#    f.close()
 
     # TODO get decimals from token
     balances = {}

apps/data-seeding/eth/import_balance.py

@@ -11,7 +11,6 @@ import csv
 import json
 
 # external imports
-import eth_abi
 import confini
 from hexathon import (
         strip_0x,
@@ -29,13 +28,19 @@ from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
 from chainlib.jsonrpc import JSONRPCRequest
-from chainlib.eth.error import EthException
+from chainlib.eth.error import (
+        EthException,
+        RequestMismatchException,
+        )
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
 from cic_base.eth.syncer import chain_interface
+from eth_accounts_index import AccountsIndex
+from eth_contract_registry import Registry
+from eth_token_index import TokenUniqueSymbolIndex
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -129,58 +134,52 @@ class Handler:
             logg.debug('no payload, skipping {}'.format(tx))
             return
 
-        if tx.payload[:8] == self.account_index_add_signature:
+        recipient = None
-            recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
+        try:
-            #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
+            r = AccountsIndex.parse_add_request(tx.payload)
-            user_file = 'new/{}/{}/{}.json'.format(
+        except RequestMismatchException:
-                    recipient[2:4].upper(),
+            return
-                    recipient[4:6].upper(),
+        recipient = r[0]
-                    recipient[2:].upper(),
+        
-                    )
+        user_file = 'new/{}/{}/{}.json'.format(
-            filepath = os.path.join(self.user_dir, user_file)
+                recipient[2:4].upper(),
-            o = None
+                recipient[4:6].upper(),
-            try:
+                recipient[2:].upper(),
-                f = open(filepath, 'r')
+                )
-                o = json.load(f)
+        filepath = os.path.join(self.user_dir, user_file)
-                f.close()
+        o = None
-            except FileNotFoundError:
+        try:
-                logg.error('no import record of address {}'.format(recipient))
+            f = open(filepath, 'r')
-                return
+            o = json.load(f)
-            u = Person.deserialize(o)
-            original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
-            try:
-                balance = self.balances[original_address]
-            except KeyError as e:
-                logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
-                return
-
-            # TODO: store token object in handler ,get decimals from there
-            multiplier = 10**6
-            balance_full = balance * multiplier
-            logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
-
-            (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
-            logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
-            r = conn.do(o)
-
-            tx_path = os.path.join(
-                    user_dir,
-                    'txs',
-                    strip_0x(tx_hash_hex),
-                    )
-            f = open(tx_path, 'w')
-            f.write(strip_0x(o['params'][0]))
             f.close()
-#        except TypeError as e:
+        except FileNotFoundError:
-#            logg.warning('typerror {}'.format(e))
+            logg.error('no import record of address {}'.format(recipient))
-#            pass
+            return
-#        except IndexError as e:
+        u = Person.deserialize(o)
-#            logg.warning('indexerror {}'.format(e))
+        original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
-#            pass
+        try:
-#        except EthException as e:
+            balance = self.balances[original_address]
-#            logg.error('send error {}'.format(e).ljust(200))
+        except KeyError as e:
-        #except KeyError as e:
+            logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
-        #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
+            return
+
+        # TODO: store token object in handler ,get decimals from there
+        multiplier = 10**6
+        balance_full = balance * multiplier
+        logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
+
+        (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
+        logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
+        r = conn.do(o)
+
+        tx_path = os.path.join(
+                user_dir,
+                'txs',
+                strip_0x(tx_hash_hex),
+                )
+        f = open(tx_path, 'w')
+        f.write(strip_0x(o['params'][0]))
+        f.close()
 
 
 def progress_callback(block_number, tx_index):
@@ -196,45 +195,24 @@ def main():
     nonce_oracle = RPCNonceOracle(signer_address, conn)
 
     # Get Token registry address
-    txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
+    registry = Registry(chain_spec)
-    tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
+    o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
-
-    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
-    data = add_0x(registry_addressof_method)
-    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
-    txf.set_code(tx, data)
-   
-    j = JSONRPCRequest()
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
-    token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    token_index_address = registry.parse_address_of(r)
+    token_index_address = to_checksum_address(token_index_address)
     logg.info('found token index address {}'.format(token_index_address))
-
+    
-
     # Get Sarafu token address
-    tx = txf.template(signer_address, token_index_address)
+    token_index = TokenUniqueSymbolIndex(chain_spec)
-    data = add_0x(registry_addressof_method)
+    o = token_index.address_of(token_index_address, token_symbol)
-    h = hashlib.new('sha256')
-    h.update(token_symbol.encode('utf-8'))
-    z = h.digest()
-    data += eth_abi.encode_single('bytes32', z).hex()
-    txf.set_code(tx, data)
-    o = j.template()
-    o['method'] = 'eth_call'
-    o['params'].append(txf.normalize(tx))
-    o['params'].append('latest')
-    o = j.finalize(o)
     r = conn.do(o)
+    token_address = token_index.parse_address_of(r)
     try:
-        sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+        token_address = to_checksum_address(token_address)
     except ValueError as e:
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
-    logg.info('found token address {}'.format(sarafu_token_address))
+    logg.info('found token address {}'.format(token_address))
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -242,22 +220,6 @@ def main():
         o = block_latest()
         r = conn.do(o)
         block_offset = int(strip_0x(r), 16) + 1
-#
-#    addresses = {}
-#    f = open('{}/addresses.csv'.format(user_dir, 'r'))
-#    while True:
-#        l = f.readline()
-#        if l == None:
-#            break
-#        r = l.split(',')
-#        try:
-#            k = r[0]
-#            v = r[1].rstrip()
-#            addresses[k] = v
-#            sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
-#        except IndexError as e:
-#            break
-#    f.close()
 
     # TODO get decimals from token
     balances = {}
@@ -282,7 +244,7 @@ def main():
 
     syncer_backend.set(block_offset, 0)
     syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
-    handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
+    handler = Handler(conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)
     

apps/data-seeding/requirements.txt

@@ -1,4 +1,11 @@
-sarafu-faucet==0.0.4a1
+sarafu-faucet~=0.0.4a2
-cic-eth[tools]==0.12.0a1
+cic-eth[tools]~=0.12.0a3
-cic-types==0.1.0a13
+cic-base[base]~=0.2.0a1
-crypto-dev-signer==0.4.14b6
+cic-types~=0.1.0a13
+crypto-dev-signer~=0.4.14b6
+chainsyncer~=0.0.3a3
+chainlib-eth~=0.0.5a1
+eth-address-index~=0.1.2a1
+eth-contract-registry~=0.5.6a1
+eth-accounts-index~=0.0.12a1
+eth-erc20~=0.0.10a3