From ff9075f2391ab5c3b051ef4e7a25ac9a3c50169a Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 12 Jul 2021 20:22:09 +0200 Subject: [PATCH] Remove eth-abi dep in data seeding --- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/tools_requirements.txt | 2 +- apps/data-seeding/cic_eth/import_balance.py | 168 ++++++++------------ apps/data-seeding/eth/import_balance.py | 164 ++++++++----------- apps/data-seeding/requirements.txt | 15 +- 5 files changed, 140 insertions(+), 211 deletions(-) diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 8f85db25..a60d8726 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 12, 0, - 'alpha.2', + 'alpha.3', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index 67ab3257..503e55b5 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,6 +1,6 @@ crypto-dev-signer~=0.4.14b6 chainqueue~=0.0.2b5 -confini~=0.3.6rc4 +confini~=0.3.6 cic-eth-registry~=0.5.6a1 redis==3.5.3 hexathon~=0.0.1a7 diff --git a/apps/data-seeding/cic_eth/import_balance.py b/apps/data-seeding/cic_eth/import_balance.py index 99e8eef4..7f7c840c 100644 --- a/apps/data-seeding/cic_eth/import_balance.py +++ b/apps/data-seeding/cic_eth/import_balance.py @@ -11,7 +11,6 @@ import csv import json # external imports -import eth_abi import confini from hexathon import ( strip_0x, @@ -29,7 +28,10 @@ from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.tx import TxFactory from chainlib.jsonrpc import JSONRPCRequest -from chainlib.eth.error import EthException +from chainlib.eth.error import ( + EthException, + RequestMismatchException, + ) from chainlib.chain import ChainSpec from chainlib.eth.constant import ZERO_ADDRESS from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer @@ -37,6 +39,9 @@ from crypto_dev_signer.keystore.dict import DictKeystore from cic_types.models.person import Person from eth_erc20 import ERC20 from cic_base.eth.syncer import chain_interface +from eth_accounts_index import AccountsIndex +from eth_contract_registry import Registry +from eth_token_index import TokenUniqueSymbolIndex logging.basicConfig(level=logging.WARNING) @@ -131,58 +136,52 @@ class Handler: logg.debug('no payload, skipping {}'.format(tx)) return - if tx.payload[:8] == self.account_index_add_signature: - recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:])) - #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)]) - user_file = 'new/{}/{}/{}.json'.format( - recipient[2:4].upper(), - recipient[4:6].upper(), - recipient[2:].upper(), - ) - filepath = os.path.join(self.user_dir, user_file) - o = None - try: - f = open(filepath, 'r') - o = json.load(f) - f.close() - except FileNotFoundError: - logg.error('no import record of address {}'.format(recipient)) - return - u = Person.deserialize(o) - original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0] - try: - balance = self.balances[original_address] - except KeyError as e: - logg.error('balance get fail orig {} new {}'.format(original_address, recipient)) - return - - # TODO: store token object in handler ,get decimals from there - multiplier = 10**6 - balance_full = balance * multiplier - logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full)) - - (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full) - logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient)) - r = conn.do(o) - - tx_path = os.path.join( - user_dir, - 'txs', - strip_0x(tx_hash_hex), - ) - f = open(tx_path, 'w') - f.write(strip_0x(o['params'][0])) + recipient = None + try: + r = AccountsIndex.parse_add_request(tx.payload) + except RequestMismatchException: + return + recipient = r[0] + + user_file = 'new/{}/{}/{}.json'.format( + recipient[2:4].upper(), + recipient[4:6].upper(), + recipient[2:].upper(), + ) + filepath = os.path.join(self.user_dir, user_file) + o = None + try: + f = open(filepath, 'r') + o = json.load(f) f.close() -# except TypeError as e: -# logg.warning('typerror {}'.format(e)) -# pass -# except IndexError as e: -# logg.warning('indexerror {}'.format(e)) -# pass -# except EthException as e: -# logg.error('send error {}'.format(e).ljust(200)) - #except KeyError as e: - # logg.error('key record not found in imports: {}'.format(e).ljust(200)) + except FileNotFoundError: + logg.error('no import record of address {}'.format(recipient)) + return + u = Person.deserialize(o) + original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0] + try: + balance = self.balances[original_address] + except KeyError as e: + logg.error('balance get fail orig {} new {}'.format(original_address, recipient)) + return + + # TODO: store token object in handler ,get decimals from there + multiplier = 10**6 + balance_full = balance * multiplier + logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full)) + + (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full) + logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient)) + r = conn.do(o) + + tx_path = os.path.join( + user_dir, + 'txs', + strip_0x(tx_hash_hex), + ) + f = open(tx_path, 'w') + f.write(strip_0x(o['params'][0])) + f.close() def progress_callback(block_number, tx_index): @@ -198,49 +197,26 @@ def main(): nonce_oracle = RPCNonceOracle(signer_address, conn) # Get Token registry address - txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None) - tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS')) - - registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8] - data = add_0x(registry_addressof_method) - data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex() - txf.set_code(tx, data) - - j = JSONRPCRequest() - o = j.template() - o['method'] = 'eth_call' - o['params'].append(txf.normalize(tx)) - o['params'].append('latest') - o = j.finalize(o) + registry = Registry(chain_spec) + o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry') r = conn.do(o) - token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) + token_index_address = registry.parse_address_of(r) + token_index_address = to_checksum_address(token_index_address) logg.info('found token index address {}'.format(token_index_address)) - - + # Get Sarafu token address - tx = txf.template(signer_address, token_index_address) - data = add_0x(registry_addressof_method) - h = hashlib.new('sha256') - h.update(token_symbol.encode('utf-8')) - z = h.digest() - data += eth_abi.encode_single('bytes32', z).hex() - txf.set_code(tx, data) - o = j.template() - o['method'] = 'eth_call' - o['params'].append(txf.normalize(tx)) - o['params'].append('latest') - o = j.finalize(o) + token_index = TokenUniqueSymbolIndex(chain_spec) + o = token_index.address_of(token_index_address, token_symbol) r = conn.do(o) + token_address = token_index.parse_address_of(r) try: - sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) + token_address = to_checksum_address(token_address) except ValueError as e: logg.critical('lookup failed for token {}: {}'.format(token_symbol, e)) sys.exit(1) - - if sarafu_token_address == ZERO_ADDRESS: - raise KeyError('token address for symbol {} is zero'.format(token_symbol)) - - logg.info('found token address {}'.format(sarafu_token_address)) + logg.info('found token address {}'.format(token_address)) + + sys.exit(0) syncer_backend = MemBackend(chain_str, 0) @@ -248,22 +224,6 @@ def main(): o = block_latest() r = conn.do(o) block_offset = int(strip_0x(r), 16) + 1 -# -# addresses = {} -# f = open('{}/addresses.csv'.format(user_dir, 'r')) -# while True: -# l = f.readline() -# if l == None: -# break -# r = l.split(',') -# try: -# k = r[0] -# v = r[1].rstrip() -# addresses[k] = v -# sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r") -# except IndexError as e: -# break -# f.close() # TODO get decimals from token balances = {} diff --git a/apps/data-seeding/eth/import_balance.py b/apps/data-seeding/eth/import_balance.py index 2d436a66..9c9816b5 100644 --- a/apps/data-seeding/eth/import_balance.py +++ b/apps/data-seeding/eth/import_balance.py @@ -11,7 +11,6 @@ import csv import json # external imports -import eth_abi import confini from hexathon import ( strip_0x, @@ -29,13 +28,19 @@ from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.tx import TxFactory from chainlib.jsonrpc import JSONRPCRequest -from chainlib.eth.error import EthException +from chainlib.eth.error import ( + EthException, + RequestMismatchException, + ) from chainlib.chain import ChainSpec from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from crypto_dev_signer.keystore.dict import DictKeystore from cic_types.models.person import Person from eth_erc20 import ERC20 from cic_base.eth.syncer import chain_interface +from eth_accounts_index import AccountsIndex +from eth_contract_registry import Registry +from eth_token_index import TokenUniqueSymbolIndex logging.basicConfig(level=logging.WARNING) @@ -129,58 +134,52 @@ class Handler: logg.debug('no payload, skipping {}'.format(tx)) return - if tx.payload[:8] == self.account_index_add_signature: - recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:])) - #original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)]) - user_file = 'new/{}/{}/{}.json'.format( - recipient[2:4].upper(), - recipient[4:6].upper(), - recipient[2:].upper(), - ) - filepath = os.path.join(self.user_dir, user_file) - o = None - try: - f = open(filepath, 'r') - o = json.load(f) - f.close() - except FileNotFoundError: - logg.error('no import record of address {}'.format(recipient)) - return - u = Person.deserialize(o) - original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0] - try: - balance = self.balances[original_address] - except KeyError as e: - logg.error('balance get fail orig {} new {}'.format(original_address, recipient)) - return - - # TODO: store token object in handler ,get decimals from there - multiplier = 10**6 - balance_full = balance * multiplier - logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full)) - - (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full) - logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient)) - r = conn.do(o) - - tx_path = os.path.join( - user_dir, - 'txs', - strip_0x(tx_hash_hex), - ) - f = open(tx_path, 'w') - f.write(strip_0x(o['params'][0])) + recipient = None + try: + r = AccountsIndex.parse_add_request(tx.payload) + except RequestMismatchException: + return + recipient = r[0] + + user_file = 'new/{}/{}/{}.json'.format( + recipient[2:4].upper(), + recipient[4:6].upper(), + recipient[2:].upper(), + ) + filepath = os.path.join(self.user_dir, user_file) + o = None + try: + f = open(filepath, 'r') + o = json.load(f) f.close() -# except TypeError as e: -# logg.warning('typerror {}'.format(e)) -# pass -# except IndexError as e: -# logg.warning('indexerror {}'.format(e)) -# pass -# except EthException as e: -# logg.error('send error {}'.format(e).ljust(200)) - #except KeyError as e: - # logg.error('key record not found in imports: {}'.format(e).ljust(200)) + except FileNotFoundError: + logg.error('no import record of address {}'.format(recipient)) + return + u = Person.deserialize(o) + original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0] + try: + balance = self.balances[original_address] + except KeyError as e: + logg.error('balance get fail orig {} new {}'.format(original_address, recipient)) + return + + # TODO: store token object in handler ,get decimals from there + multiplier = 10**6 + balance_full = balance * multiplier + logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full)) + + (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full) + logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient)) + r = conn.do(o) + + tx_path = os.path.join( + user_dir, + 'txs', + strip_0x(tx_hash_hex), + ) + f = open(tx_path, 'w') + f.write(strip_0x(o['params'][0])) + f.close() def progress_callback(block_number, tx_index): @@ -196,45 +195,24 @@ def main(): nonce_oracle = RPCNonceOracle(signer_address, conn) # Get Token registry address - txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None) - tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS')) - - registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8] - data = add_0x(registry_addressof_method) - data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex() - txf.set_code(tx, data) - - j = JSONRPCRequest() - o = j.template() - o['method'] = 'eth_call' - o['params'].append(txf.normalize(tx)) - o['params'].append('latest') - o = j.finalize(o) + registry = Registry(chain_spec) + o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry') r = conn.do(o) - token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) + token_index_address = registry.parse_address_of(r) + token_index_address = to_checksum_address(token_index_address) logg.info('found token index address {}'.format(token_index_address)) - - + # Get Sarafu token address - tx = txf.template(signer_address, token_index_address) - data = add_0x(registry_addressof_method) - h = hashlib.new('sha256') - h.update(token_symbol.encode('utf-8')) - z = h.digest() - data += eth_abi.encode_single('bytes32', z).hex() - txf.set_code(tx, data) - o = j.template() - o['method'] = 'eth_call' - o['params'].append(txf.normalize(tx)) - o['params'].append('latest') - o = j.finalize(o) + token_index = TokenUniqueSymbolIndex(chain_spec) + o = token_index.address_of(token_index_address, token_symbol) r = conn.do(o) + token_address = token_index.parse_address_of(r) try: - sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) + token_address = to_checksum_address(token_address) except ValueError as e: logg.critical('lookup failed for token {}: {}'.format(token_symbol, e)) sys.exit(1) - logg.info('found token address {}'.format(sarafu_token_address)) + logg.info('found token address {}'.format(token_address)) syncer_backend = MemBackend(chain_str, 0) @@ -242,22 +220,6 @@ def main(): o = block_latest() r = conn.do(o) block_offset = int(strip_0x(r), 16) + 1 -# -# addresses = {} -# f = open('{}/addresses.csv'.format(user_dir, 'r')) -# while True: -# l = f.readline() -# if l == None: -# break -# r = l.split(',') -# try: -# k = r[0] -# v = r[1].rstrip() -# addresses[k] = v -# sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r") -# except IndexError as e: -# break -# f.close() # TODO get decimals from token balances = {} @@ -282,7 +244,7 @@ def main(): syncer_backend.set(block_offset, 0) syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback) - handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) + handler = Handler(conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index ab043327..f1350e97 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,4 +1,11 @@ -sarafu-faucet==0.0.4a1 -cic-eth[tools]==0.12.0a1 -cic-types==0.1.0a13 -crypto-dev-signer==0.4.14b6 +sarafu-faucet~=0.0.4a2 +cic-eth[tools]~=0.12.0a3 +cic-base[base]~=0.2.0a1 +cic-types~=0.1.0a13 +crypto-dev-signer~=0.4.14b6 +chainsyncer~=0.0.3a3 +chainlib-eth~=0.0.5a1 +eth-address-index~=0.1.2a1 +eth-contract-registry~=0.5.6a1 +eth-accounts-index~=0.0.12a1 +eth-erc20~=0.0.10a3