Add readme

README.md

@@ -0,0 +1,43 @@
+# cic-batch
+
+Helper scripts for submitting erc20 transfers to chaind-eth queuer.
+
+Currently it contains two scripts:
+
+* `scripts/parse_csv.py` - Generate signed ethereum transactions from input data in csv format
+* `scripts/submit.py` - Submit signed transactions to the chaind-eth socket server
+
+
+## Parsing
+
+The expected CSV input format is:
+
+```
+recipient_address,token_address,value
+```
+
+The `value` can be an integer or a floating-point value.
+
+Signed transactions will be written as individual files, as hex-encode byte literals. The file name will be prefixed with the transaction's position in the csv. 
+
+The output directory will be the concatenation of the `--output` and `--session-id` argument flags. If `--output` is not given, the default folder name will be `./output`. If `--session-id` is not specified, a new `uuid` will be generated and used as the session id.
+
+To verify the details of the generated transactions, the contents of the generated files can be piped to `eth-decode`. Note that the same fee price will be used for all generated transactions.
+
+
+## Submitting
+
+Invoke `scripts/submit.py` on the output of `parse_csv.py`.
+
+You may specify the input directory (that is, the output location used by `parse_csv.py` as a single, optional positional argument to `submit.py`). If omitted, the default value `./output` will be used.
+
+The `--session-id` argument is used both to create the input data path, aswell as the socket file for the chaind-eth server.
+
+For example: If you invoke `submit.py [...] --session-id foo ./bar` then any files in `./bar/foo` will be attempted processed as signed transactions. However, if you invoke `submit.py [...] --session-id foo` (omitting the positional argument), files in `./output/foo` will be parsed.
+
+
+## Caveats
+
+The `--session-id` argument value should be the same as the corresponding value used when starting the `chaind-eth` socket server.
+
+The default value of `--fee-limit` in `parse_csv.py` is `8000000`. If you are issuing transactions against a erc20 contract you do not fully know or trust, you may want to explicitly set this to a lower value.

example.csv

@@ -1,3 +1,3 @@
-0x7Ad7a61539De44a2Cf296dE0aEfe37F6c4bbdA7E,0xb708175e3f6Cd850643aAF7B32212AFad50e2549,100
-0x6336e1b77b7106C7F63aE870df846619AFEe51C4,0xb708175e3f6Cd850643aAF7B32212AFad50e2549,12012
-0xDfB7B9DbdC9398ff95738aa27ba180aD03dDa024,0xb708175e3f6Cd850643aAF7B32212AFad50e2549,25530.939
+0x7Ad7a61539De44a2Cf296dE0aEfe37F6c4bbdA7E,0xD7332B13Ed7d1eE01ababD63A2c8F285b0a4F5F1,100
+0x6336e1b77b7106C7F63aE870df846619AFEe51C4,0xD7332B13Ed7d1eE01ababD63A2c8F285b0a4F5F1,12012
+0xDfB7B9DbdC9398ff95738aa27ba180aD03dDa024,0xD7332B13Ed7d1eE01ababD63A2c8F285b0a4F5F1,25530.939

requirements.txt

@@ -1,7 +1,7 @@
 jsonschema~=3.2.0
-chainlib~=0.0.3a1
-confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a7
-eth-erc20~=0.0.9a3
+chainlib~=0.0.12
+confini~=0.5.1
+cic-eth-registry~=0.6.2
+eth-erc20~=0.1.2
 pycryptodome==3.10.1
 pyxdg==0.27

scripts/parse_csv.py

@@ -18,8 +18,8 @@ from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.tx import TxFormat
 from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
+from funga.eth.signer import EIP155Signer
+from funga.eth.keystore.dict import DictKeystore
 from eth_erc20 import ERC20
 from cic_eth_registry.erc20 import ERC20Token
 
@@ -32,14 +32,14 @@ required_fields = [
     'value',
         ]
 
-config_dir = os.environ.get('CONFINI_DIR', '.')
+config_dir = os.environ.get('CONFINI_DIR', './config')
 
 argparser = argparse.ArgumentParser('chainqueue transaction submission and trigger server')
-argparser.add_argument('-c', '--config', dest='c', type=str, default=config_dir, help='configuration directory')
+argparser.add_argument('-c', '--config', dest='c', type=str, help='configuration directory')
 argparser.add_argument('-p', type=str, help='rpc endpoint')
 argparser.add_argument('-i', type=str, help='chain spec')
 argparser.add_argument('--session-id', dest='session_id', type=str, default=str(uuid.uuid4()), help='session id to use for session')
-argparser.add_argument('--gas-limit', dest='gas_limit', type=int, default=8000000, help='override gas limit')
+argparser.add_argument('--fee-limit', dest='fee_limit', type=int, default=8000000, help='override gas limit')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be very verbose')
@@ -53,7 +53,7 @@ if args.vv:
 elif args.v:
     logg.setLevel(logging.INFO)
 
-config = confini.Config(args.c)
+config = confini.Config(config_dir, override_dirs=args.c)
 config.process()
 args_override = {
             'SESSION_CHAIN_SPEC': getattr(args, 'i'),
@@ -62,7 +62,7 @@ args_override = {
 config.dict_override(args_override, 'cli args')
 config.add(getattr(args, 'input_data'), '_INPUT_FILE', True)
 config.add(getattr(args, 'session_id'), '_SESSION_ID', True)
-config.add(getattr(args, 'gas_limit'), '_GAS_LIMIT', True)
+config.add(getattr(args, 'fee_limit'), '_FEE_LIMIT', True)
 
 output_dir = getattr(args, 'output')
 if not output_dir:
@@ -94,7 +94,7 @@ rpc = EthHTTPConnection(config.get('RPC_ENDPOINT'))
 
 nonce_oracle = RPCNonceOracle(signer_address, conn=rpc)
 
-gas_oracle = OverrideGasOracle(limit=config.get('_GAS_LIMIT'), conn=rpc)
+gas_oracle = OverrideGasOracle(limit=config.get('_FEE_LIMIT'), conn=rpc)
 
 chain_spec = ChainSpec.from_chain_str(config.get('SESSION_CHAIN_SPEC'))
 

scripts/submit.py

@@ -14,17 +14,17 @@ from xdg.BaseDirectory import get_runtime_dir
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = os.environ.get('CONFINI_DIR', '.')
-default_queue_runtime_path = os.path.join(get_runtime_dir(), 'chainqueue')
+config_dir = os.environ.get('CONFINI_DIR', './config')
+default_queue_runtime_path = os.path.join(get_runtime_dir(), 'chaind')
 
 argparser = argparse.ArgumentParser('chainqueue transaction submission and trigger server')
-argparser.add_argument('-c', '--config', dest='c', type=str, default=config_dir, help='configuration directory')
+argparser.add_argument('-c', '--config', dest='c', type=str, help='configuration directory')
 argparser.add_argument('--session-id', dest='session_id', type=str, default=str(uuid.uuid4()), help='session id to use for session')
 argparser.add_argument('-s', '--socket-path', dest='s', type=str, help='socket path')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be very verbose')
-argparser.add_argument('input_dir', type=str, help='directory with transaction files')
+argparser.add_argument('input_dir', type=str, nargs='?', default='output', help='directory with transaction files')
 args = argparser.parse_args(sys.argv[1:])
 
 if args.vv:
@@ -32,17 +32,22 @@ if args.vv:
 elif args.v:
     logg.setLevel(logging.INFO)
 
-config = confini.Config(args.c)
+config = confini.Config(config_dir, override_dirs=args.c)
 socket_path = getattr(args, 's')
+config.process()
 args_override = {
         'QUEUE_SOCKET_PATH': getattr(args, 's'),
         }
 config.dict_override(args_override, 'cli args')
-config.process()
-config.add(getattr(args, 'input_dir'), '_INPUT_DIR', True)
+
 config.add(getattr(args, 'session_id'), '_SESSION_ID', True)
+
+input_basedir = getattr(args, 'input_dir')
+input_dir = os.path.join(input_basedir, config.get('_SESSION_ID'))
+config.add(input_dir, '_INPUT_DIR', True)
+
 if not config.get('QUEUE_SOCKET_PATH'):
-    config.add(os.path.join(default_queue_runtime_path, config.get('_SESSION_ID'), 'chainqueue.sock'), 'QUEUE_SOCKET_PATH', True)
+    config.add(os.path.join(default_queue_runtime_path, 'eth', config.get('_SESSION_ID'), 'chaind.sock'), 'QUEUE_SOCKET_PATH', True)
 logg.debug('config loaded:\n{}'.format(config))