Remove eth-abi dep in data seeding

This commit is contained in:
Louis Holbrook 2021-07-14 14:43:32 +00:00
parent d6782abbcc
commit f9594b766a
20 changed files with 234 additions and 317 deletions

View File

@ -1,13 +1,13 @@
cic-base==0.1.3a3+build.984b5cff cic-base~=0.2.0a2
alembic==1.4.2 alembic==1.4.2
confini~=0.3.6rc3 confini>=0.3.6rc3,<0.5.0
uwsgi==2.0.19.1 uwsgi==2.0.19.1
moolb~=0.1.0 moolb~=0.1.0
cic-eth-registry~=0.5.6a1 cic-eth-registry~=0.5.6a2
SQLAlchemy==1.3.20 SQLAlchemy==1.3.20
semver==2.13.0 semver==2.13.0
psycopg2==2.8.6 psycopg2==2.8.6
celery==4.4.7 celery==4.4.7
redis==3.5.3 redis==3.5.3
chainsyncer[sql]~=0.0.3a3 chainsyncer[sql]~=0.0.3a4
erc20-faucet~=0.2.2a1 erc20-faucet~=0.2.2a2

View File

@ -1,5 +1,5 @@
SQLAlchemy==1.3.20 SQLAlchemy==1.3.20
cic-eth-registry~=0.5.6a1 cic-eth-registry~=0.5.6a2
hexathon~=0.0.1a7 hexathon~=0.0.1a7
chainqueue~=0.0.2b5 chainqueue~=0.0.2b6
eth-erc20==0.0.10a2 eth-erc20~=0.0.10a3

View File

@ -10,7 +10,7 @@ version = (
0, 0,
12, 12,
0, 0,
'alpha.2', 'alpha.3',
) )
version_object = semver.VersionInfo( version_object = semver.VersionInfo(

View File

@ -1,3 +1,3 @@
celery==4.4.7 celery==4.4.7
chainlib~=0.0.5a1 chainlib~=0.0.5a2
semver==2.13.0 semver==2.13.0

View File

@ -1,16 +1,15 @@
chainsyncer[sql]~=0.0.3a3 chainsyncer[sql]~=0.0.3a4
chainqueue~=0.0.2b5 chainqueue~=0.0.2b6
alembic==1.4.2 alembic==1.4.2
confini~=0.3.6rc4 confini>=0.3.6rc4,<0.5.0
redis==3.5.3 redis==3.5.3
hexathon~=0.0.1a7 hexathon~=0.0.1a7
pycryptodome==3.10.1 pycryptodome==3.10.1
liveness~=0.0.1a7 liveness~=0.0.1a7
eth-address-index~=0.1.2a1 eth-address-index~=0.1.2a2
eth-accounts-index~=0.0.12a1 eth-accounts-index~=0.0.12a2
cic-eth-registry~=0.5.6a1 cic-eth-registry~=0.5.6a2
erc20-faucet~=0.2.2a1 erc20-faucet~=0.2.2a2
erc20-transfer-authorization~=0.3.2a1 erc20-transfer-authorization~=0.3.2a2
sarafu-faucet~=0.0.4a1 sarafu-faucet~=0.0.4a3
moolb~=0.1.1b2 moolb~=0.1.1b2
erc20-transfer-authorization~=0.3.2a1

View File

@ -6,4 +6,4 @@ pytest-redis==2.0.0
redis==3.5.3 redis==3.5.3
eth-tester==0.5.0b3 eth-tester==0.5.0b3
py-evm==0.3.0a20 py-evm==0.3.0a20
eth-erc20~=0.0.10a2 eth-erc20~=0.0.10a3

View File

@ -1,7 +1,7 @@
crypto-dev-signer~=0.4.14b6 crypto-dev-signer~=0.4.14b7
chainqueue~=0.0.2b5 chainqueue~=0.0.2b5
confini~=0.3.6rc4 confini>=0.3.6rc4,<0.5.0
cic-eth-registry~=0.5.6a1 cic-eth-registry~=0.5.6a2
redis==3.5.3 redis==3.5.3
hexathon~=0.0.1a7 hexathon~=0.0.1a7
pycryptodome==3.10.1 pycryptodome==3.10.1

View File

@ -9,7 +9,7 @@ import semver
logg = logging.getLogger() logg = logging.getLogger()
version = (0, 4, 0, 'alpha.7') version = (0, 4, 0, 'alpha.8')
version_object = semver.VersionInfo( version_object = semver.VersionInfo(
major=version[0], major=version[0],

View File

@ -1 +1 @@
cic_base[full_graph]==0.1.3a3+build.984b5cff cic_base[full_graph]~=0.2.0a3

View File

@ -1,7 +1,7 @@
[app] [app]
ALLOWED_IP=0.0.0.0/0 ALLOWED_IP=0.0.0.0/0
LOCALE_FALLBACK=en LOCALE_FALLBACK=en
LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/ LOCALE_PATH=var/lib/locale/
MAX_BODY_LENGTH=1024 MAX_BODY_LENGTH=1024
PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I= PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
SERVICE_CODE=*483*46#,*483*061#,*384*96# SERVICE_CODE=*483*46#,*483*061#,*384*96#
@ -11,13 +11,13 @@ SUPPORT_PHONE_NUMBER=0757628885
REGION=KE REGION=KE
[ussd] [ussd]
MENU_FILE=/usr/src/data/ussd_menu.json MENU_FILE=data/ussd_menu.json
user = user =
pass = pass =
[statemachine] [statemachine]
STATES=/usr/src/cic-ussd/states/ STATES=states/
TRANSITIONS=/usr/src/cic-ussd/transitions/ TRANSITIONS=transitions/
[client] [client]
host = host =

View File

@ -1,5 +1,5 @@
[pgp] [pgp]
export_dir = /usr/src/pgp/keys/ export_dir = pgp/keys/
keys_path = /usr/src/secrets/ keys_path = /usr/src/secrets/
private_keys = privatekeys_meta.asc private_keys = privatekeys_meta.asc
passphrase = passphrase =

View File

@ -4,4 +4,4 @@
user_server_port=${SERVER_PORT:-9500} user_server_port=${SERVER_PORT:-9500}
/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@" /usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

View File

@ -4,4 +4,4 @@
user_ussd_server_port=${SERVER_PORT:-9000} user_ussd_server_port=${SERVER_PORT:-9000}
/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@" /usr/local/bin/uwsgi --wsgi-file cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

View File

@ -1,4 +1,4 @@
cic_base[full_graph]==0.1.3a3+build.984b5cff cic_base[full_graph]==0.2.0a3
cic-eth~=0.12.0a1 cic-eth~=0.12.0a1
cic-notify~=0.4.0a7 cic-notify~=0.4.0a8
cic-types~=0.1.0a11 cic-types~=0.1.0a11

View File

@ -165,4 +165,3 @@ set +e
echo -n 2 > $init_level_file echo -n 2 > $init_level_file
exec "$@" exec "$@"
l:83

View File

@ -11,7 +11,6 @@ import csv
import json import json
# external imports # external imports
import eth_abi
import confini import confini
from hexathon import ( from hexathon import (
strip_0x, strip_0x,
@ -29,7 +28,10 @@ from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import TxFactory from chainlib.eth.tx import TxFactory
from chainlib.jsonrpc import JSONRPCRequest from chainlib.jsonrpc import JSONRPCRequest
from chainlib.eth.error import EthException from chainlib.eth.error import (
EthException,
RequestMismatchException,
)
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.constant import ZERO_ADDRESS
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
@ -37,6 +39,9 @@ from crypto_dev_signer.keystore.dict import DictKeystore
from cic_types.models.person import Person from cic_types.models.person import Person
from eth_erc20 import ERC20 from eth_erc20 import ERC20
from cic_base.eth.syncer import chain_interface from cic_base.eth.syncer import chain_interface
from eth_accounts_index import AccountsIndex
from eth_contract_registry import Registry
from eth_token_index import TokenUniqueSymbolIndex
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
@ -131,58 +136,52 @@ class Handler:
logg.debug('no payload, skipping {}'.format(tx)) logg.debug('no payload, skipping {}'.format(tx))
return return
if tx.payload[:8] == self.account_index_add_signature: recipient = None
recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:])) try:
#original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)]) r = AccountsIndex.parse_add_request(tx.payload)
user_file = 'new/{}/{}/{}.json'.format( except RequestMismatchException:
recipient[2:4].upper(), return
recipient[4:6].upper(), recipient = r[0]
recipient[2:].upper(),
)
filepath = os.path.join(self.user_dir, user_file)
o = None
try:
f = open(filepath, 'r')
o = json.load(f)
f.close()
except FileNotFoundError:
logg.error('no import record of address {}'.format(recipient))
return
u = Person.deserialize(o)
original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
try:
balance = self.balances[original_address]
except KeyError as e:
logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
return
# TODO: store token object in handler ,get decimals from there user_file = 'new/{}/{}/{}.json'.format(
multiplier = 10**6 recipient[2:4].upper(),
balance_full = balance * multiplier recipient[4:6].upper(),
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full)) recipient[2:].upper(),
)
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full) filepath = os.path.join(self.user_dir, user_file)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient)) o = None
r = conn.do(o) try:
f = open(filepath, 'r')
tx_path = os.path.join( o = json.load(f)
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close() f.close()
# except TypeError as e: except FileNotFoundError:
# logg.warning('typerror {}'.format(e)) logg.error('no import record of address {}'.format(recipient))
# pass return
# except IndexError as e: u = Person.deserialize(o)
# logg.warning('indexerror {}'.format(e)) original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
# pass try:
# except EthException as e: balance = self.balances[original_address]
# logg.error('send error {}'.format(e).ljust(200)) except KeyError as e:
#except KeyError as e: logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
# logg.error('key record not found in imports: {}'.format(e).ljust(200)) return
# TODO: store token object in handler ,get decimals from there
multiplier = 10**6
balance_full = balance * multiplier
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close()
def progress_callback(block_number, tx_index): def progress_callback(block_number, tx_index):
@ -198,49 +197,26 @@ def main():
nonce_oracle = RPCNonceOracle(signer_address, conn) nonce_oracle = RPCNonceOracle(signer_address, conn)
# Get Token registry address # Get Token registry address
txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None) registry = Registry(chain_spec)
tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS')) o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
data = add_0x(registry_addressof_method)
data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
txf.set_code(tx, data)
j = JSONRPCRequest()
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) token_index_address = registry.parse_address_of(r)
token_index_address = to_checksum_address(token_index_address)
logg.info('found token index address {}'.format(token_index_address)) logg.info('found token index address {}'.format(token_index_address))
# Get Sarafu token address # Get Sarafu token address
tx = txf.template(signer_address, token_index_address) token_index = TokenUniqueSymbolIndex(chain_spec)
data = add_0x(registry_addressof_method) o = token_index.address_of(token_index_address, token_symbol)
h = hashlib.new('sha256')
h.update(token_symbol.encode('utf-8'))
z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data)
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
token_address = token_index.parse_address_of(r)
try: try:
sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) token_address = to_checksum_address(token_address)
except ValueError as e: except ValueError as e:
logg.critical('lookup failed for token {}: {}'.format(token_symbol, e)) logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
sys.exit(1) sys.exit(1)
logg.info('found token address {}'.format(token_address))
if sarafu_token_address == ZERO_ADDRESS: sys.exit(0)
raise KeyError('token address for symbol {} is zero'.format(token_symbol))
logg.info('found token address {}'.format(sarafu_token_address))
syncer_backend = MemBackend(chain_str, 0) syncer_backend = MemBackend(chain_str, 0)
@ -248,22 +224,6 @@ def main():
o = block_latest() o = block_latest()
r = conn.do(o) r = conn.do(o)
block_offset = int(strip_0x(r), 16) + 1 block_offset = int(strip_0x(r), 16) + 1
#
# addresses = {}
# f = open('{}/addresses.csv'.format(user_dir, 'r'))
# while True:
# l = f.readline()
# if l == None:
# break
# r = l.split(',')
# try:
# k = r[0]
# v = r[1].rstrip()
# addresses[k] = v
# sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
# except IndexError as e:
# break
# f.close()
# TODO get decimals from token # TODO get decimals from token
balances = {} balances = {}

View File

@ -17,7 +17,7 @@ default_config_dir = '/usr/local/etc/cic'
arg_parser = argparse.ArgumentParser() arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file') arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file')
arg_parser.add_argument('-q', type=str, default='cic-eth', help='Task queue') arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='Task queue')
arg_parser.add_argument('-v', action='store_true', help='Be verbose') arg_parser.add_argument('-v', action='store_true', help='Be verbose')
arg_parser.add_argument('-vv', action='store_true', help='Be more verbose') arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree') arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree')

View File

@ -11,7 +11,6 @@ import csv
import json import json
# external imports # external imports
import eth_abi
import confini import confini
from hexathon import ( from hexathon import (
strip_0x, strip_0x,
@ -29,13 +28,20 @@ from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import TxFactory from chainlib.eth.tx import TxFactory
from chainlib.jsonrpc import JSONRPCRequest from chainlib.jsonrpc import JSONRPCRequest
from chainlib.eth.error import EthException from chainlib.eth.error import (
EthException,
RequestMismatchException,
)
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
from crypto_dev_signer.keystore.dict import DictKeystore from crypto_dev_signer.keystore.dict import DictKeystore
from cic_types.models.person import Person from cic_types.models.person import Person
from eth_erc20 import ERC20 from eth_erc20 import ERC20
from cic_base.eth.syncer import chain_interface from cic_base.eth.syncer import chain_interface
from eth_accounts_index import AccountsIndex
from eth_contract_registry import Registry
from eth_token_index import TokenUniqueSymbolIndex
from erc20_faucet import Faucet
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
@ -50,7 +56,7 @@ argparser.add_argument('-c', type=str, default=config_dir, help='config root to
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('--head', action='store_true', help='start at current block height (overrides --offset)') argparser.add_argument('--head', action='store_true', help='start at current block height (overrides --offset)')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@ -112,12 +118,16 @@ class Handler:
account_index_add_signature = keccak256_string_to_hex('add(address)')[:8] account_index_add_signature = keccak256_string_to_hex('add(address)')[:8]
def __init__(self, conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle): def __init__(self, conn, chain_spec, user_dir, balances, token_address, faucet_address, signer_address, signer, gas_oracle, nonce_oracle):
self.token_address = token_address self.token_address = token_address
self.faucet_address = faucet_address
self.user_dir = user_dir self.user_dir = user_dir
self.balances = balances self.balances = balances
self.chain_spec = chain_spec self.chain_spec = chain_spec
self.tx_factory = ERC20(chain_spec, signer, gas_oracle, nonce_oracle) self.nonce_oracle = nonce_oracle
self.gas_oracle = gas_oracle
self.signer_address = signer_address
self.signer = signer
def name(self): def name(self):
@ -129,58 +139,59 @@ class Handler:
logg.debug('no payload, skipping {}'.format(tx)) logg.debug('no payload, skipping {}'.format(tx))
return return
if tx.payload[:8] == self.account_index_add_signature: recipient = None
recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:])) try:
#original_address = to_checksum_address(self.addresses[to_checksum_address(recipient)]) r = AccountsIndex.parse_add_request(tx.payload)
user_file = 'new/{}/{}/{}.json'.format( except RequestMismatchException:
recipient[2:4].upper(), return
recipient[4:6].upper(), recipient = r[0]
recipient[2:].upper(),
)
filepath = os.path.join(self.user_dir, user_file)
o = None
try:
f = open(filepath, 'r')
o = json.load(f)
f.close()
except FileNotFoundError:
logg.error('no import record of address {}'.format(recipient))
return
u = Person.deserialize(o)
original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
try:
balance = self.balances[original_address]
except KeyError as e:
logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
return
# TODO: store token object in handler ,get decimals from there user_file = 'new/{}/{}/{}.json'.format(
multiplier = 10**6 recipient[2:4].upper(),
balance_full = balance * multiplier recipient[4:6].upper(),
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full)) recipient[2:].upper(),
)
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full) filepath = os.path.join(self.user_dir, user_file)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient)) o = None
r = conn.do(o) try:
f = open(filepath, 'r')
tx_path = os.path.join( o = json.load(f)
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close() f.close()
# except TypeError as e: except FileNotFoundError:
# logg.warning('typerror {}'.format(e)) logg.error('no import record of address {}'.format(recipient))
# pass return
# except IndexError as e: u = Person.deserialize(o)
# logg.warning('indexerror {}'.format(e)) original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0]
# pass try:
# except EthException as e: balance = self.balances[original_address]
# logg.error('send error {}'.format(e).ljust(200)) except KeyError as e:
#except KeyError as e: logg.error('balance get fail orig {} new {}'.format(original_address, recipient))
# logg.error('key record not found in imports: {}'.format(e).ljust(200)) return
# TODO: store token object in handler ,get decimals from there
erc20 = ERC20(self.chain_spec, signer=self.signer, gas_oracle=self.gas_oracle, nonce_oracle=self.nonce_oracle)
o = erc20.decimals(self.token_address)
r = conn.do(o)
decimals = erc20.parse_decimals(r)
multiplier = 10 ** decimals
balance_full = balance * multiplier
logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
(tx_hash_hex, o) = erc20.transfer(self.token_address, self.signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close()
faucet = Faucet(self.chain_spec, signer=self.signer, gas_oracle=self.gas_oracle, nonce_oracle=self.nonce_oracle)
(tx_hash, o) = faucet.give_to(self.faucet_address, self.signer_address, recipient)
r = conn.do(o)
def progress_callback(block_number, tx_index): def progress_callback(block_number, tx_index):
@ -196,45 +207,31 @@ def main():
nonce_oracle = RPCNonceOracle(signer_address, conn) nonce_oracle = RPCNonceOracle(signer_address, conn)
# Get Token registry address # Get Token registry address
txf = TxFactory(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=None) registry = Registry(chain_spec)
tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS')) o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
data = add_0x(registry_addressof_method)
data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
txf.set_code(tx, data)
j = JSONRPCRequest()
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) token_index_address = registry.parse_address_of(r)
token_index_address = to_checksum_address(token_index_address)
logg.info('found token index address {}'.format(token_index_address)) logg.info('found token index address {}'.format(token_index_address))
# Get Faucet address
o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'Faucet')
r = conn.do(o)
faucet_address = registry.parse_address_of(r)
faucet_address = to_checksum_address(faucet_address)
logg.info('found faucet {}'.format(faucet_address))
# Get Sarafu token address # Get Sarafu token address
tx = txf.template(signer_address, token_index_address) token_index = TokenUniqueSymbolIndex(chain_spec)
data = add_0x(registry_addressof_method) o = token_index.address_of(token_index_address, token_symbol)
h = hashlib.new('sha256')
h.update(token_symbol.encode('utf-8'))
z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data)
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
token_address = token_index.parse_address_of(r)
try: try:
sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) token_address = to_checksum_address(token_address)
except ValueError as e: except ValueError as e:
logg.critical('lookup failed for token {}: {}'.format(token_symbol, e)) logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
sys.exit(1) sys.exit(1)
logg.info('found token address {}'.format(sarafu_token_address)) logg.info('found token address {}'.format(token_address))
syncer_backend = MemBackend(chain_str, 0) syncer_backend = MemBackend(chain_str, 0)
@ -242,22 +239,6 @@ def main():
o = block_latest() o = block_latest()
r = conn.do(o) r = conn.do(o)
block_offset = int(strip_0x(r), 16) + 1 block_offset = int(strip_0x(r), 16) + 1
#
# addresses = {}
# f = open('{}/addresses.csv'.format(user_dir, 'r'))
# while True:
# l = f.readline()
# if l == None:
# break
# r = l.split(',')
# try:
# k = r[0]
# v = r[1].rstrip()
# addresses[k] = v
# sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
# except IndexError as e:
# break
# f.close()
# TODO get decimals from token # TODO get decimals from token
balances = {} balances = {}
@ -282,7 +263,7 @@ def main():
syncer_backend.set(block_offset, 0) syncer_backend.set(block_offset, 0)
syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback) syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) handler = Handler(conn, chain_spec, user_dir, balances, token_address, faucet_address, signer_address, signer, gas_oracle, nonce_oracle)
syncer.add_filter(handler) syncer.add_filter(handler)
syncer.loop(1, conn) syncer.loop(1, conn)

View File

@ -1,4 +1,12 @@
sarafu-faucet==0.0.4a1 sarafu-faucet==0.0.4a3
cic-eth[tools]==0.12.0a1 cic-eth[tools]==0.12.1a1
cic-types==0.1.0a13 cic-types==0.1.0a13
crypto-dev-signer==0.4.14b6 crypto-dev-signer==0.4.14b7
faker==4.17.1
chainsyncer~=0.0.3a3
chainlib-eth~=0.0.5a1
eth-address-index~=0.1.2a1
eth-contract-registry~=0.5.6a1
eth-accounts-index~=0.0.12a1
eth-erc20~=0.0.10a3
erc20-faucet~=0.2.2a1

View File

@ -14,7 +14,6 @@ import urllib.parse
# external imports # external imports
import celery import celery
import confini import confini
import eth_abi
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainlib.eth.address import to_checksum_address from chainlib.eth.address import to_checksum_address
from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.connection import EthHTTPConnection
@ -33,6 +32,9 @@ from cic_types.models.person import (
from erc20_faucet import Faucet from erc20_faucet import Faucet
from eth_erc20 import ERC20 from eth_erc20 import ERC20
from hexathon.parse import strip_0x, add_0x from hexathon.parse import strip_0x, add_0x
from eth_contract_registry import Registry
from eth_accounts_index import AccountsIndex
from eth_token_index import TokenUniqueSymbolIndex
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
@ -43,7 +45,8 @@ custodial_tests = [
'local_key', 'local_key',
'gas', 'gas',
'faucet', 'faucet',
'ussd' 'ussd',
'ussd_pins',
] ]
metadata_tests = [ metadata_tests = [
@ -71,6 +74,7 @@ argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spe
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url') argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification')
argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
@ -130,6 +134,11 @@ if args.skip_custodial:
for t in custodial_tests: for t in custodial_tests:
if t not in exclude: if t not in exclude:
exclude.append(t) exclude.append(t)
if args.skip_metadata:
logg.info('will skip all metadata verifications ({})'.format(','.join(metadata_tests)))
for t in metadata_tests:
if t not in exclude:
exclude.append(t)
for t in include: for t in include:
if t not in all_tests: if t not in all_tests:
raise ValueError('Cannot include unknown verification "{}"'.format(t)) raise ValueError('Cannot include unknown verification "{}"'.format(t))
@ -140,7 +149,7 @@ for t in include:
api = None api = None
for t in custodial_tests: for t in custodial_tests:
if t in active_tests: if t in active_tests:
from cic_eth.api.api_admin import AdminApi from cic_eth.api.admin import AdminApi
api = AdminApi(None) api = AdminApi(None)
logg.info('activating custodial module'.format(t)) logg.info('activating custodial module'.format(t))
break break
@ -263,19 +272,11 @@ class Verifier:
def verify_accounts_index(self, address, balance=None): def verify_accounts_index(self, address, balance=None):
tx = self.tx_factory.template(ZERO_ADDRESS, self.index_address) accounts_index = AccountsIndex(self.chain_spec)
data = keccak256_string_to_hex('have(address)')[:8] o = accounts_index.have(self.index_address, address)
data += eth_abi.encode_single('address', address).hex()
tx = self.tx_factory.set_code(tx, data)
tx = self.tx_factory.normalize(tx)
j = JSONRPCRequest()
o = j.template()
o['method'] = 'eth_call'
o['params'].append(tx)
o = j.finalize(o)
r = self.conn.do(o) r = self.conn.do(o)
logg.debug('index check for {}: {}'.format(address, r)) n = accounts_index.parse_have(r)
n = eth_abi.decode_single('uint256', bytes.fromhex(strip_0x(r))) logg.debug('index check for {}: {}'.format(address, n))
if n != 1: if n != 1:
raise VerifierError(n, 'accounts index') raise VerifierError(n, 'accounts index')
@ -427,69 +428,38 @@ def main():
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000) gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
# Get Token registry address # Get Token registry address
txf = TxFactory(chain_spec, signer=None, gas_oracle=gas_oracle, nonce_oracle=None) registry = Registry(chain_spec)
tx = txf.template(ZERO_ADDRESS, config.get('CIC_REGISTRY_ADDRESS')) o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
# TODO: replace with cic-eth-registry
registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
data = add_0x(registry_addressof_method)
data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
txf.set_code(tx, data)
j = JSONRPCRequest()
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) token_index_address = registry.parse_address_of(r)
token_index_address = to_checksum_address(token_index_address)
logg.info('found token index address {}'.format(token_index_address)) logg.info('found token index address {}'.format(token_index_address))
data = add_0x(registry_addressof_method) # Get Account registry address
data += eth_abi.encode_single('bytes32', b'AccountRegistry').hex() o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'AccountRegistry')
txf.set_code(tx, data)
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
account_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) account_index_address = registry.parse_address_of(r)
account_index_address = to_checksum_address(account_index_address)
logg.info('found account index address {}'.format(account_index_address)) logg.info('found account index address {}'.format(account_index_address))
data = add_0x(registry_addressof_method) # Get Faucet address
data += eth_abi.encode_single('bytes32', b'Faucet').hex() o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'Faucet')
txf.set_code(tx, data)
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
faucet_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) faucet_address = registry.parse_address_of(r)
faucet_index_address = to_checksum_address(token_index_address)
logg.info('found faucet {}'.format(faucet_address)) logg.info('found faucet {}'.format(faucet_address))
# Get Sarafu token address
token_index = TokenUniqueSymbolIndex(chain_spec)
# Get Sarafu token address o = token_index.address_of(token_index_address, token_symbol)
tx = txf.template(ZERO_ADDRESS, token_index_address)
data = add_0x(registry_addressof_method)
h = hashlib.new('sha256')
h.update(token_symbol.encode('utf-8'))
z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data)
o = j.template()
o['method'] = 'eth_call'
o['params'].append(txf.normalize(tx))
o['params'].append('latest')
o = j.finalize(o)
r = conn.do(o) r = conn.do(o)
sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r)))) token_address = token_index.parse_address_of(r)
logg.info('found token address {}'.format(sarafu_token_address)) try:
token_address = to_checksum_address(token_address)
except ValueError as e:
logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
sys.exit(1)
logg.info('found token address {}'.format(token_address))
balances = {} balances = {}
f = open('{}/balances.csv'.format(user_dir, 'r')) f = open('{}/balances.csv'.format(user_dir, 'r'))
@ -511,7 +481,7 @@ def main():
f.close() f.close()
verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, sarafu_token_address, faucet_address, user_dir, exit_on_error) verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, token_address, faucet_address, user_dir, exit_on_error)
user_new_dir = os.path.join(user_dir, 'new') user_new_dir = os.path.join(user_dir, 'new')
i = 0 i = 0