diff --git a/README.md b/README.md index 158727c..0138c83 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ It capabilities are (unchecked box means feature not yet completed): - [x] unix socket server to accept raw, signed RLP evm transactions - [x] stateful queueing system following full local and remote lifecycle of the transaction - [x] transaction dispatcher unit -- [ ] transaction retry unit (for errored or suspended transactions) +- [x] transaction retry unit (for errored or suspended transactions) - [x] blockchain listener that updates state of transactions in queue - [x] CLI transaction listing tool, filterable by: * [x] transaction range with lower and/or upper bound @@ -38,7 +38,7 @@ For any python command / executable used below: ## setting up the database backend -Currently there is no more practical way of setting up the database backend :/ +Currently there is no more practical way of setting up the database backend than to pull the repository and run a database migration script :/ ``` git clone https://gitlab.com/chaintool/chaind @@ -66,7 +66,7 @@ d=$(mktemp -d) && cd $d ``` python -m venv .venv . .venv/bin/activate -pip install --extra-index-url https://pip.grassrootseconomics.net:8433 "chaind-eth>=0.0.1a2" +pip install --extra-index-url https://pip.grassrootseconomics.net:8433 "chaind-eth>=0.0.3a5" ``` ### start the services @@ -100,7 +100,7 @@ Create two transactions from sender in keyfile (which needs to have gas balance) ``` export WALLET_KEY_FILE= export WALLET_PASSWORD= -export RPC_HTTP_PROVIDER= +export RPC_PROVIDER= export CHAIN_SPEC= # create new account and store address in variable @@ -116,23 +116,141 @@ eth-gas --raw -a $recipient 4096 > tx3.txt ### send test transactions to queue ``` -cat tx1.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock -cat tx2.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock -cat tx3.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock +cat tx1.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock - +cat tx2.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock - +cat tx3.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock - ``` ### check status of transactions +`chainqueue-list` outputs details about transactions in the queue: + ``` export DATABASE_ENGINE=sqlite sender=$(eth-keyfile -d $WALLET_KEY_FILE) -DATABASE_NAME=$HOME/.local/share/chaind/eth/chaind.sqlite chainqueue-list $sender -# to show a summary only instead all transactions -DATABASE_NAME=$HOME/.local/share/chaind/eth/chaind.sqlite chainqueue-list --summary $sender +chainqueue-list $sender ``` -The `chainqueue-list` tool provides some basic filtering. Use `chainqueue-list --help` to see what they are. +To show a summary only instead all transactions: + +``` +chainqueue-list --summary $sender +``` + +The `chaind-list` tool can be used to list by session id. Following the above examples: + +``` +chaind-list testsession +``` + +The `chainqueue-list` and `chaind-list` tools both provides the same basic filtering. Use `--help` to see the details. + + +### Retrieve transaction by hash + +The socket server returns the transaction hash when a transaction is submitted. + +If a socket server is given a transaction hash, it will return the transaction data for that hash (if it exists). + +Extending the previous examples, this will output the original signed transaction: + +``` +eth-gas --raw -a $recipient 1024 > tx1.txt +cat tx1.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock - | cut -b 4- > hash1.txt +cat hash1.tx | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock - | cut -b 4- > tx1_recovered.txt +diff tx1_recovered.txt tx1.txt +# should output 0 +echo $? +``` + +The first 4 bytes of the data returned from the socket is a 32-bit big-endian result code. The data payload follows from the 5th byte. + + +## Batch processing + +The `chaind-eth-send` executable generates signed transactions with data from a csv file. + +The data columns must be in the following order: + +1. receipient address +2. transaction value +3. token specifier (optional, network fee token if not given) +4. network fee token value (optional) + + +If the gas token value (4) is not given for a gas token transaction, the transaction value (2) will be used. + +By default the signed transactions are output as hex to stdout, each on a separate line. + +If a valid `--socket` is given (i.e. the socket of the `chaind-eth-server`) the transactions will be send to the socket instead. The hash of the transaction will be output to standard output. + + +### Using token symbols + +If token symols are to be used in some or all values of column 3, then a valid `--token-index` executable address is required (in this case, a smart contract implementing the [`registry`](https://gitlab.com/grassrootseconomics/cic-contracts/-/blob/master/solidity/Registry.sol) contract interface). + + +### Input validity checks + +The validity of the input data is verified _before_ actual execution takes place. + +These checks include: + +- The token can be made sense of. +- The values can be parsed to integer amounts. +- The recipient address is a valid checksum address. + +The checks do however _not_ include whether the token balances of the signer are sufficient to successfully execute the transactions on the network. + + +### CSV input example + +``` +0x72B70906fD07c72f2d96aAA250C2D31662D0d809,10,0xb708175e3f6Cd850643aAF7B32212AFad50e2549 +0xD536CB6d1d9B8d33875E0ba0Aa3515eD7478f889,0x2a,GFT,100 +0xeE08b59a95E822AE346489038D25750C8EdfcC25,0x029a +``` + +This will result in the following transactions: + +1. send 10 tokens from token contract `0xb708175e3f6Cd850643aAF7B32212AFad50e2549` to recipient `0x72B70906fD07c72f2d96aAA250C2D31662D0d809`. +2. send 42 `GFT` tokens along with 100 network gas tokens to recipient `0xD536CB6d1d9B8d33875E0ba0Aa3515eD7478f889` +3. send 666 network gas tokens to recipient `0xeE08b59a95E822AE346489038D25750C8EdfcC25` + + +### Resending transactions + +Since the `chaind-eth-server` does not have access to signing keys, resending stalled transactions is also a separate external action. + +The `chaind-eth-resend` executable takes a list of signed transactions (e.g. as output from `chaind-eth-send` using the socket) and automatically increases the fee price of the transaction to create a replacement. + +As with `chaind-eth-send`, the resend executable optionally takes a socket argument that sends the transaction directly to a socket. Otherwise, the signed transactions are send to standard output. + +For example, the following will output details of the transaction generated by `chaind-eth-resend`, in which the fee price has been slightly incremented: + +``` +eth-gas --raw --fee-price 100000000 -a $recipient 1024 > tx1.txt +chaind-eth-resend tx1.txt > tx1_bump.txt +cat tx1_bump.txt | eth-decode +``` + + +### Retrieving transactions for resend + +The `chaind-list` tool can be used to retrieve transactions with the same filters as `chainqueue-list`, but also allowing results limited a specific session id. + +As with `chainqueue-list`, which column to output can be customized. This enables creation of signed transaction lists in the format accepted by `chaind-eth-resent`. + +One examples of criteria for transactions due to be resent may be: + +``` +# get any pending transaction in session "testsession" +export DATABASE_ENGINE=sqlite +chaind-list -o signedtx --pending testsession +``` + +Note that the `chaind-list` tool requires a connection to the queueing backend. ## systemd diff --git a/chaind_eth/cli/csv.py b/chaind_eth/cli/csv.py index f70d2d7..75a56ca 100644 --- a/chaind_eth/cli/csv.py +++ b/chaind_eth/cli/csv.py @@ -16,10 +16,10 @@ class CSVProcessor: import csv # only import if needed fr = csv.reader(f) - f.close() for r in fr: contents.append(r) + f.close() l = len(contents) logg.info('successfully parsed source as csv, found {} records'.format(l)) return contents diff --git a/chaind_eth/cli/resolver.py b/chaind_eth/cli/resolver.py index 618ee2e..b7119f2 100644 --- a/chaind_eth/cli/resolver.py +++ b/chaind_eth/cli/resolver.py @@ -3,10 +3,50 @@ import logging # external imports from chainlib.eth.constant import ZERO_ADDRESS +from chainlib.eth.address import is_checksum_address +from hexathon import strip_0x +from eth_token_index.index import TokenUniqueSymbolIndex logg = logging.getLogger(__name__) +class LookNoop: + + def get(self, k, rpc=None): + try: + if not is_checksum_address(k): + raise ValueError('not valid checksum address {}'.format(k)) + except ValueError: + raise ValueError('not valid checksum address {}'.format(k)) + return strip_0x(k) + + + def __str__(self): + return 'checksum address shortcircuit' + + +class TokenIndexLookup(TokenUniqueSymbolIndex): + + + def __init__(self, chain_spec, signer, gas_oracle, nonce_oracle, address, sender_address=ZERO_ADDRESS): + super(TokenIndexLookup, self).__init__(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) + self.local_address = address + self.sender_address = sender_address + + + def get(self, k, rpc=None): + o = self.address_of(self.local_address, k, sender_address=self.sender_address) + r = rpc.do(o) + address = self.parse_address_of(r) + if address != ZERO_ADDRESS: + return address + raise FileNotFoundError(address) + + + def __str__(self): + return 'token symbol index' + + class DefaultResolver: def __init__(self, chain_spec, rpc, sender_address=ZERO_ADDRESS): @@ -18,19 +58,20 @@ class DefaultResolver: self.sender_address = sender_address - def add_lookup(self, lookup, address): + def add_lookup(self, lookup, reverse): self.lookups.append(lookup) - self.lookup_pointers.append(address) + self.lookup_pointers.append(reverse) def lookup(self, k): if k == '' or k == None: return None - for i, lookup in enumerate(self.lookups): - address = self.lookup_pointers[i] - o = lookup.address_of(address, k, sender_address=self.sender_address) - r = self.rpc.do(o) - address = lookup.parse_address_of(r) - if address != ZERO_ADDRESS: + for lookup in self.lookups: + try: + address = lookup.get(k, rpc=self.rpc) + logg.debug('resolved token {} to {} with lookup {}'.format(k, address, lookup)) return address + except Exception as e: + logg.debug('lookup {} failed for {}: {}'.format(lookup, k, e)) + raise FileNotFoundError(k) diff --git a/chaind_eth/runnable/resend.py b/chaind_eth/runnable/resend.py index fc8011b..86feb4f 100644 --- a/chaind_eth/runnable/resend.py +++ b/chaind_eth/runnable/resend.py @@ -33,14 +33,12 @@ config_dir = os.path.join(script_dir, '..', 'data', 'config') arg_flags = chainlib.eth.cli.argflag_std_write argparser = chainlib.eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--socket', dest='socket', type=str, help='Socket to send transactions to') -argparser.add_argument('--token-index', dest='token_index', type=str, help='Token resolver index') argparser.add_positional('source', required=False, type=str, help='Transaction source file') args = argparser.parse_args() extra_args = { 'socket': None, 'source': None, - 'token_index': None, } env = Environment(domain='eth', env=os.environ) diff --git a/chaind_eth/runnable/send.py b/chaind_eth/runnable/send.py index ff6ce71..e2db5f8 100644 --- a/chaind_eth/runnable/send.py +++ b/chaind_eth/runnable/send.py @@ -14,13 +14,16 @@ from chaind import Environment from chainlib.eth.gas import price from chainlib.chain import ChainSpec from hexathon import strip_0x -from eth_token_index.index import TokenUniqueSymbolIndex # local imports from chaind_eth.cli.process import Processor from chaind_eth.cli.csv import CSVProcessor from chaind.error import TxSourceError -from chaind_eth.cli.resolver import DefaultResolver +from chaind_eth.cli.resolver import ( + DefaultResolver, + LookNoop, + TokenIndexLookup, + ) logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -83,18 +86,6 @@ if config.get('_SOURCE') == None: sys.exit(1) -class TokenIndexLookupAdapter(TokenUniqueSymbolIndex): - - def __init__(self, sender, address, chain_spec, signer=None, gas_oracle=None, nonce_oracle=None): - super(TokenIndexLookupAdapter, self).__init__(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) - self.index_address = address - self.sender = sender - - - def resolve(self, v): - return self.address_of(self.index_address, v, sender_address=sender) - - class Outputter: def __init__(self, mode): @@ -123,10 +114,16 @@ class Outputter: def main(): signer = rpc.get_signer() + # TODO: make resolvers pluggable token_resolver = DefaultResolver(chain_spec, conn, sender_address=rpc.get_sender_address()) - token_index_lookup = TokenUniqueSymbolIndex(chain_spec, signer=signer, gas_oracle=rpc.get_gas_oracle(), nonce_oracle=rpc.get_nonce_oracle()) - token_resolver.add_lookup(token_index_lookup, config.get('_TOKEN_INDEX')) + + noop_lookup = LookNoop() + token_resolver.add_lookup(noop_lookup, 'noop') + + if config.get('_TOKEN_INDEX') != None: + token_index_lookup = TokenIndexLookup(chain_spec, signer, rpc.get_gas_oracle(), rpc.get_nonce_oracle(), config.get('_TOKEN_INDEX')) + token_resolver.add_lookup(token_index_lookup, reverse=config.get('_TOKEN_INDEX')) processor = Processor(wallet.get_signer_address(), wallet.get_signer(), config.get('_SOURCE'), chain_spec, rpc.get_gas_oracle(), rpc.get_nonce_oracle(), resolver=token_resolver) processor.add_processor(CSVProcessor()) diff --git a/requirements.txt b/requirements.txt index 33a6678..dc573a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ chaind<=0.0.3,>=0.0.3a5 hexathon~=0.0.1a8 -chainlib-eth<=0.1.0,>=0.0.9a9 -eth-address-index<=0.3.0,>=0.2.3a4 +chainlib-eth<=0.1.0,>=0.0.9a10 +eth-address-index<=0.3.0,>=0.2.3a5