cic-batch/scripts/parse_csv.py

# standard imports
import csv
import os
import logging
import argparse
import uuid
import sys
import tempfile
import stat
import shutil

# external imports
import csv
import confini
from chainlib.eth.address import is_checksum_address
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.tx import TxFormat
from chainlib.chain import ChainSpec
from chainlib.eth.connection import EthHTTPConnection
from funga.eth.signer import EIP155Signer
from funga.eth.keystore.dict import DictKeystore
from eth_erc20 import ERC20
from cic_eth_registry.erc20 import ERC20Token

logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()

required_fields = [
    'to',
    'token',
    'value',
        ]

config_dir = os.environ.get('CONFINI_DIR', './config')

argparser = argparse.ArgumentParser('chainqueue transaction submission and trigger server')
argparser.add_argument('-c', '--config', dest='c', type=str, help='configuration directory')
argparser.add_argument('-p', type=str, help='rpc endpoint')
argparser.add_argument('-i', type=str, help='chain spec')
argparser.add_argument('--session-id', dest='session_id', type=str, default=str(uuid.uuid4()), help='session id to use for session')
argparser.add_argument('--fee-limit', dest='fee_limit', type=int, default=8000000, help='override gas limit')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be very verbose')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-o', '--output', type=str, help='output directory')
argparser.add_argument('input_data', type=str, help='input file')
args = argparser.parse_args(sys.argv[1:])

if args.vv:
    logg.setLevel(logging.DEBUG)
elif args.v:
    logg.setLevel(logging.INFO)

config = confini.Config(config_dir, override_dirs=args.c)
config.process()
args_override = {
            'SESSION_CHAIN_SPEC': getattr(args, 'i'),
            'RPC_ENDPOINT': getattr(args, 'p'),
        }
config.dict_override(args_override, 'cli args')
config.add(getattr(args, 'input_data'), '_INPUT_FILE', True)
config.add(getattr(args, 'session_id'), '_SESSION_ID', True)
config.add(getattr(args, 'fee_limit'), '_FEE_LIMIT', True)

output_dir = getattr(args, 'output')
if not output_dir:
    output_dir = os.path.join('.', 'output', config.get('_SESSION_ID'))
config.add(output_dir, '_OUTPUT_DIR', True)
logg.debug('config loaded:\n{}'.format(config))
logg.info('using session id {}'.format(config.get('_SESSION_ID')))

passphrase_env = 'ETH_PASSPHRASE'
if args.env_prefix != None:
    passphrase_env = args.env_prefix + '_' + passphrase_env
passphrase = os.environ.get(passphrase_env)
if passphrase == None:
    logg.warning('no passphrase given')
    passphrase=''

signer_address = None
keystore = DictKeystore()
if args.y == None:
    sys.stderr.write('keyfile missing\n')
    sys.exit(1)

logg.debug('loading keystore file {}'.format(args.y))
signer_address = keystore.import_keystore_file(args.y, password=passphrase)
logg.debug('now have key for signer address {}'.format(signer_address))
signer = EIP155Signer(keystore)

rpc = EthHTTPConnection(config.get('RPC_ENDPOINT'))

nonce_oracle = RPCNonceOracle(signer_address, conn=rpc)

gas_oracle = OverrideGasOracle(limit=config.get('_FEE_LIMIT'), conn=rpc)

chain_spec = ChainSpec.from_chain_str(config.get('SESSION_CHAIN_SPEC'))

try:
    os.stat(config.get('_OUTPUT_DIR'))
    sys.stderr.write('output directory {} already exists\n'.format(config.get('_OUTPUT_DIR')))
    sys.exit(1)
except FileNotFoundError:
    pass


class InvalidData(Exception):
    pass
        

token_cache = {}
erc20 = ERC20(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)

def validate(row):
    if len(row) != 3:
        raise InvalidData('row length {} found, need {}'.format(len(row), 3))
    if not is_checksum_address(row[0]):
        raise InvalidData('invalid recipient address {}'.format(row[0]))
    if not is_checksum_address(row[1]):
        raise InvalidData('invalid token address {}'.format(row[1]))

    if token_cache.get(row[1]) == None:
        token_cache[row[1]] = ERC20Token(chain_spec, rpc, row[1])
        
    try:
        value = float(row[2])
    except ValueError:
        raise InvalidData('invalid token address {}'.format(row[1]))

    return token_cache[row[1]]


def main():
    tmp_out = tempfile.mkdtemp()
    logg.debug('using tmp dir {}'.format(tmp_out))

    f = open(config.get('_INPUT_FILE'), 'r')
    cr = csv.reader(f)
    i = 1

    for row in cr:
        token_symbol = None
        try:
            token = validate(row)
        except InvalidData as e:
            sys.stderr.write(str(e) + ' in line {}\n'.format(i))
            shutil.rmtree(tmp_out)
            sys.exit(1)
        multiplier = 10 ** token.decimals
        value = int(multiplier * float(row[2]))
        (tx_hash_hex, o) = erc20.transfer(row[1], signer_address, row[0], value, tx_format=TxFormat.RLP_SIGNED)
        fname = '{}_{}'.format(i, row[0])
        fpath = os.path.join(tmp_out, fname)
        f = open(fpath, 'x')
        f.write(o)
        f.close()
        logg.info('tx {}: {} ({} * 10^{}) {} {} -> {}'.format(tx_hash_hex, value, row[2], token.decimals, token.symbol, signer_address, row[0]))
        i += 1
    f.close()
    shutil.copytree(tmp_out, config.get('_OUTPUT_DIR'))
    logg.debug('files moved from tmp dir {} to {}'.format(tmp_out, config.get('_OUTPUT_DIR')))
    shutil.rmtree(tmp_out)
    

if __name__ == '__main__':
    main()