kamikazechaser
/
cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove eth-abi dep in data seeding

This commit is contained in:
nolash 2021-07-12 20:22:09 +02:00
parent c52885a016
commit ff9075f239
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
5 changed files with 140 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-eth/cic_eth/version.py
View File

@ -10,7 +10,7 @@ version = (
0,
12,
0,
'alpha.2',
'alpha.3',
)
version_object = semver.VersionInfo(

2
apps/cic-eth/tools_requirements.txt
View File

@ -1,6 +1,6 @@
crypto-dev-signer~=0.4.14b6
chainqueue~=0.0.2b5
confini~=0.3.6rc4
confini~=0.3.6
cic-eth-registry~=0.5.6a1
redis==3.5.3
hexathon~=0.0.1a7

168
apps/data-seeding/cic_eth/import_balance.py
View File

@ -11,7 +11,6 @@ import csv
import json
# external imports
import eth_abi
import confini
from hexathon import (
strip_0x,
@ -29,7 +28,10 @@ from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import TxFactory
from chainlib.jsonrpc import JSONRPCRequest
from chainlib.eth.error import EthException
from chainlib.eth.error import (
EthException,
RequestMismatchException,
)
from chainlib.chain import ChainSpec
from chainlib.eth.constant import ZERO_ADDRESS
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
@ -37,6 +39,9 @@ from crypto_dev_signer.keystore.dict import DictKeystore
from cic_types.models.person import Person
from eth_erc20 import ERC20
from cic_base.eth.syncer import chain_interface
from eth_accounts_index import AccountsIndex
from eth_contract_registry import Registry
from eth_token_index import TokenUniqueSymbolIndex
logging.basicConfig(level=logging.WARNING)
@ -131,58 +136,52 @@ class Handler:
logg.debug('no payload, skipping {}'.format(tx))
return
if tx.payload[:8] == self.account_index_add_signature:
recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
#original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
user_file = 'new/{}/{}/{}.json'.format(
recipient[2:4].upper(),
recipient[4:6].upper(),
recipient[2:].upper(),
)
filepath = os.path.join(self.user_dir, user_file)
o = None
try:
f = open(filepath, 'r')
o = json.load(f)
f.close()
except FileNotFoundError:
logg.error('no import record of address {}'.format(recipient))
return
u = Person.deserialize(o)
original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
try:
balance = self.balances[original_address]
except KeyError as e:
logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
return
# TODO: store token object in handler ,get decimals from there
multiplier = 10**6
balance_full = balance * multiplier
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
recipient = None
try:
r = AccountsIndex.parse_add_request(tx.payload)
except RequestMismatchException:
return
recipient = r[0]
user_file = 'new/{}/{}/{}.json'.format(
recipient[2:4].upper(),
recipient[4:6].upper(),
recipient[2:].upper(),
)
filepath = os.path.join(self.user_dir, user_file)
o = None
try:
f = open(filepath, 'r')
o = json.load(f)
f.close()
# except TypeError as e:
# logg.warning('typerror {}'.format(e))
# pass
# except IndexError as e:
# logg.warning('indexerror {}'.format(e))
# pass
# except EthException as e:
# logg.error('send error {}'.format(e).ljust(200))
#except KeyError as e:
# logg.error('key record not found in imports: {}'.format(e).ljust(200))
except FileNotFoundError:
logg.error('no import record of address {}'.format(recipient))
return
u = Person.deserialize(o)
original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
try:
balance = self.balances[original_address]
except KeyError as e:
logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
return
# TODO: store token object in handler ,get decimals from there
multiplier = 10**6
balance_full = balance * multiplier
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close()
def progress_callback(block_number, tx_index):
@ -198,49 +197,26 @@ def main():
nonce_oracle = RPCNonceOracle(signer_address, conn)
# Get Token registry address
txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
data = add_0x(registry_addressof_method)
data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
txf.set_code(tx, data)
j = JSONRPCRequest()
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
registry = Registry(chain_spec)
o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
r = conn.do(o)
token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
token_index_address = registry.parse_address_of(r)
token_index_address = to_checksum_address(token_index_address)
logg.info('found token index address {}'.format(token_index_address))
# Get Sarafu token address
tx = txf.template(signer_address, token_index_address)
data = add_0x(registry_addressof_method)
h = hashlib.new('sha256')
h.update(token_symbol.encode('utf-8'))
z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data)
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
token_index = TokenUniqueSymbolIndex(chain_spec)
o = token_index.address_of(token_index_address, token_symbol)
r = conn.do(o)
token_address = token_index.parse_address_of(r)
try:
sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
token_address = to_checksum_address(token_address)
except ValueError as e:
logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
sys.exit(1)
if sarafu_token_address == ZERO_ADDRESS:
raise KeyError('token address for symbol {} is zero'.format(token_symbol))
logg.info('found token address {}'.format(sarafu_token_address))
logg.info('found token address {}'.format(token_address))
sys.exit(0)
syncer_backend = MemBackend(chain_str, 0)
@ -248,22 +224,6 @@ def main():
o = block_latest()
r = conn.do(o)
block_offset = int(strip_0x(r), 16) + 1
#
# addresses = {}
# f = open('{}/addresses.csv'.format(user_dir, 'r'))
# while True:
# l = f.readline()
# if l == None:
# break
# r = l.split(',')
# try:
# k = r[0]
# v = r[1].rstrip()
# addresses[k] = v
# sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
# except IndexError as e:
# break
# f.close()
# TODO get decimals from token
balances = {}

164
apps/data-seeding/eth/import_balance.py
View File

@ -11,7 +11,6 @@ import csv
import json
# external imports
import eth_abi
import confini
from hexathon import (
strip_0x,
@ -29,13 +28,19 @@ from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import TxFactory
from chainlib.jsonrpc import JSONRPCRequest
from chainlib.eth.error import EthException
from chainlib.eth.error import (
EthException,
RequestMismatchException,
)
from chainlib.chain import ChainSpec
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
from crypto_dev_signer.keystore.dict import DictKeystore
from cic_types.models.person import Person
from eth_erc20 import ERC20
from cic_base.eth.syncer import chain_interface
from eth_accounts_index import AccountsIndex
from eth_contract_registry import Registry
from eth_token_index import TokenUniqueSymbolIndex
logging.basicConfig(level=logging.WARNING)
@ -129,58 +134,52 @@ class Handler:
logg.debug('no payload, skipping {}'.format(tx))
return
if tx.payload[:8] == self.account_index_add_signature:
recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
#original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)])
user_file = 'new/{}/{}/{}.json'.format(
recipient[2:4].upper(),
recipient[4:6].upper(),
recipient[2:].upper(),
)
filepath = os.path.join(self.user_dir, user_file)
o = None
try:
f = open(filepath, 'r')
o = json.load(f)
f.close()
except FileNotFoundError:
logg.error('no import record of address {}'.format(recipient))
return
u = Person.deserialize(o)
original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
try:
balance = self.balances[original_address]
except KeyError as e:
logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
return
# TODO: store token object in handler ,get decimals from there
multiplier = 10**6
balance_full = balance * multiplier
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
recipient = None
try:
r = AccountsIndex.parse_add_request(tx.payload)
except RequestMismatchException:
return
recipient = r[0]
user_file = 'new/{}/{}/{}.json'.format(
recipient[2:4].upper(),
recipient[4:6].upper(),
recipient[2:].upper(),
)
filepath = os.path.join(self.user_dir, user_file)
o = None
try:
f = open(filepath, 'r')
o = json.load(f)
f.close()
# except TypeError as e:
# logg.warning('typerror {}'.format(e))
# pass
# except IndexError as e:
# logg.warning('indexerror {}'.format(e))
# pass
# except EthException as e:
# logg.error('send error {}'.format(e).ljust(200))
#except KeyError as e:
# logg.error('key record not found in imports: {}'.format(e).ljust(200))
except FileNotFoundError:
logg.error('no import record of address {}'.format(recipient))
return
u = Person.deserialize(o)
original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
try:
balance = self.balances[original_address]
except KeyError as e:
logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
return
# TODO: store token object in handler ,get decimals from there
multiplier = 10**6
balance_full = balance * multiplier
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close()
def progress_callback(block_number, tx_index):
@ -196,45 +195,24 @@ def main():
nonce_oracle = RPCNonceOracle(signer_address, conn)
# Get Token registry address
txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None)
tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
data = add_0x(registry_addressof_method)
data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
txf.set_code(tx, data)
j = JSONRPCRequest()
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
registry = Registry(chain_spec)
o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
r = conn.do(o)
token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
token_index_address = registry.parse_address_of(r)
token_index_address = to_checksum_address(token_index_address)
logg.info('found token index address {}'.format(token_index_address))
# Get Sarafu token address
tx = txf.template(signer_address, token_index_address)
data = add_0x(registry_addressof_method)
h = hashlib.new('sha256')
h.update(token_symbol.encode('utf-8'))
z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data)
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
token_index = TokenUniqueSymbolIndex(chain_spec)
o = token_index.address_of(token_index_address, token_symbol)
r = conn.do(o)
token_address = token_index.parse_address_of(r)
try:
sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
token_address = to_checksum_address(token_address)
except ValueError as e:
logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
sys.exit(1)
logg.info('found token address {}'.format(sarafu_token_address))
logg.info('found token address {}'.format(token_address))
syncer_backend = MemBackend(chain_str, 0)
@ -242,22 +220,6 @@ def main():
o = block_latest()
r = conn.do(o)
block_offset = int(strip_0x(r), 16) + 1
#
# addresses = {}
# f = open('{}/addresses.csv'.format(user_dir, 'r'))
# while True:
# l = f.readline()
# if l == None:
# break
# r = l.split(',')
# try:
# k = r[0]
# v = r[1].rstrip()
# addresses[k] = v
# sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
# except IndexError as e:
# break
# f.close()
# TODO get decimals from token
balances = {}
@ -282,7 +244,7 @@ def main():
syncer_backend.set(block_offset, 0)
syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
handler = Handler(conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle)
syncer.add_filter(handler)
syncer.loop(1, conn)

15
apps/data-seeding/requirements.txt
View File

@ -1,4 +1,11 @@
sarafu-faucet==0.0.4a1
cic-eth[tools]==0.12.0a1
cic-types==0.1.0a13
crypto-dev-signer==0.4.14b6
sarafu-faucet~=0.0.4a2
cic-eth[tools]~=0.12.0a3
cic-base[base]~=0.2.0a1
cic-types~=0.1.0a13
crypto-dev-signer~=0.4.14b6
chainsyncer~=0.0.3a3
chainlib-eth~=0.0.5a1
eth-address-index~=0.1.2a1
eth-contract-registry~=0.5.6a1
eth-accounts-index~=0.0.12a1
eth-erc20~=0.0.10a3