cic-batch/scripts/parse_csv.py

# standard imports
import csv
import os
import logging
import argparse
import uuid
import sys
import tempfile
import stat
import shutil

# external imports
import csv
import confini
from chainlib.eth.address import is_checksum_address
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.tx import TxFormat
from chainlib.chain import ChainSpec
from chainlib.eth.connection import EthHTTPConnection
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
from crypto_dev_signer.keystore.dict import DictKeystore
from eth_erc20 import ERC20
from cic_eth_registry.erc20 import ERC20Token

logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()

required_fields = [
    'to',
    'token',
    'value',
        ]

config_dir = os.environ.get('CONFINI_DIR', '.')

argparser = argparse.ArgumentParser('chainqueue transaction submission and trigger server')
argparser.add_argument('-c', '--config', dest='c', type=str, default=config_dir, help='configuration directory')
argparser.add_argument('-p', type=str, help='rpc endpoint')
argparser.add_argument('-i', type=str, help='chain spec')
argparser.add_argument('--session-id', dest='session_id', type=str, default=str(uuid.uuid4()), help='session id to use for session')
argparser.add_argument('--gas-limit', dest='gas_limit', type=int, default=8000000, help='override gas limit')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be very verbose')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-o', '--output', type=str, help='output directory')
argparser.add_argument('input_data', type=str, help='input file')
args = argparser.parse_args(sys.argv[1:])

if args.vv:
    logg.setLevel(logging.DEBUG)
elif args.v:
    logg.setLevel(logging.INFO)

config = confini.Config(args.c)
config.process()
args_override = {
            'SESSION_CHAIN_SPEC': getattr(args, 'i'),
            'RPC_ENDPOINT': getattr(args, 'p'),
        }
config.dict_override(args_override, 'cli args')
config.add(getattr(args, 'input_data'), '_INPUT_FILE', True)
config.add(getattr(args, 'session_id'), '_SESSION_ID', True)
config.add(getattr(args, 'gas_limit'), '_GAS_LIMIT', True)

output_dir = getattr(args, 'output')
if not output_dir:
    output_dir = os.path.join('.', 'output', config.get('_SESSION_ID'))
config.add(output_dir, '_OUTPUT_DIR', True)
logg.debug('config loaded:\n{}'.format(config))
logg.info('using session id {}'.format(config.get('_SESSION_ID')))

passphrase_env = 'ETH_PASSPHRASE'
if args.env_prefix != None:
    passphrase_env = args.env_prefix + '_' + passphrase_env
passphrase = os.environ.get(passphrase_env)
if passphrase == None:
    logg.warning('no passphrase given')
    passphrase=''

signer_address = None
keystore = DictKeystore()
if args.y == None:
    sys.stderr.write('keyfile missing\n')
    sys.exit(1)

logg.debug('loading keystore file {}'.format(args.y))
signer_address = keystore.import_keystore_file(args.y, password=passphrase)
logg.debug('now have key for signer address {}'.format(signer_address))
signer = EIP155Signer(keystore)

rpc = EthHTTPConnection(config.get('RPC_ENDPOINT'))

nonce_oracle = RPCNonceOracle(signer_address, conn=rpc)

gas_oracle = OverrideGasOracle(limit=config.get('_GAS_LIMIT'), conn=rpc)

chain_spec = ChainSpec.from_chain_str(config.get('SESSION_CHAIN_SPEC'))

try:
    os.stat(config.get('_OUTPUT_DIR'))
    sys.stderr.write('output directory {} already exists\n'.format(config.get('_OUTPUT_DIR')))
    sys.exit(1)
except FileNotFoundError:
    pass


class InvalidData(Exception):
    pass
        

token_cache = {}
erc20 = ERC20(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)

def validate(row):
    if len(row) != 3:
        raise InvalidData('row length {} found, need {}'.format(len(row), 3))
    if not is_checksum_address(row[0]):
        raise InvalidData('invalid recipient address {}'.format(row[0]))
    if not is_checksum_address(row[1]):
        raise InvalidData('invalid token address {}'.format(row[1]))

    if token_cache.get(row[1]) == None:
        token_cache[row[1]] = ERC20Token(chain_spec, rpc, row[1])
        
    try:
        value = float(row[2])
    except ValueError:
        raise InvalidData('invalid token address {}'.format(row[1]))

    return token_cache[row[1]]


def main():
    tmp_out = tempfile.mkdtemp()
    logg.debug('using tmp dir {}'.format(tmp_out))

    f = open(config.get('_INPUT_FILE'), 'r')
    cr = csv.reader(f)
    i = 1

    for row in cr:
        token_symbol = None
        try:
            token = validate(row)
        except InvalidData as e:
            sys.stderr.write(str(e) + ' in line {}\n'.format(i))
            shutil.rmtree(tmp_out)
            sys.exit(1)
        multiplier = 10 ** token.decimals
        value = int(multiplier * float(row[2]))
        (tx_hash_hex, o) = erc20.transfer(row[1], signer_address, row[0], value, tx_format=TxFormat.RLP_SIGNED)
        fname = '{}_{}'.format(i, row[0])
        fpath = os.path.join(tmp_out, fname)
        f = open(fpath, 'x')
        f.write(o)
        f.close()
        logg.info('tx {}: {} ({} * 10^{}) {} {} -> {}'.format(tx_hash_hex, value, row[2], token.decimals, token.symbol, signer_address, row[0]))
        i += 1
    f.close()
    shutil.copytree(tmp_out, config.get('_OUTPUT_DIR'))
    logg.debug('files moved from tmp dir {} to {}'.format(tmp_out, config.get('_OUTPUT_DIR')))
    shutil.rmtree(tmp_out)
    

if __name__ == '__main__':
    main()