Add checksum address token resolver as default

README.md

@@ -7,7 +7,7 @@ It capabilities are (unchecked box means feature not yet completed):
 - [x] unix socket server to accept raw, signed RLP evm transactions
 - [x] stateful queueing system following full local and remote lifecycle of the transaction
 - [x] transaction dispatcher unit
-- [ ] transaction retry unit (for errored or suspended transactions)
+- [x] transaction retry unit (for errored or suspended transactions)
 - [x] blockchain listener that updates state of transactions in queue
 - [x] CLI transaction listing tool, filterable by:
 	* [x] transaction range with lower and/or upper bound
@@ -38,7 +38,7 @@ For any python command / executable used below:
 
 ## setting up the database backend
 
-Currently there is no more practical way of setting up the database backend :/
+Currently there is no more practical way of setting up the database backend than to pull the repository and run a database migration script :/
 
 ```
 git clone https://gitlab.com/chaintool/chaind
@@ -66,7 +66,7 @@ d=$(mktemp -d) && cd $d
 ```
 python -m venv .venv
 . .venv/bin/activate
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 "chaind-eth>=0.0.1a2"
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 "chaind-eth>=0.0.3a5"
 ```
 
 ### start the services
@@ -100,7 +100,7 @@ Create two transactions from sender in keyfile (which needs to have gas balance)
 ```
 export WALLET_KEY_FILE=<path_to_keyfile>
 export WALLET_PASSWORD=<keyfile_password_if_needed>
-export RPC_HTTP_PROVIDER=<your_provider>
+export RPC_PROVIDER=<your_provider>
 export CHAIN_SPEC=<chain_spec_of_provider>
 
 # create new account and store address in variable
@@ -116,23 +116,141 @@ eth-gas --raw -a $recipient 4096 > tx3.txt
 ### send test transactions to queue
 
 ```
-cat tx1.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock
-cat tx2.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock
-cat tx3.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock
+cat tx1.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock -
+cat tx2.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock -
+cat tx3.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock -
 ```
 
 ### check status of transactions
 
 
+`chainqueue-list` outputs details about transactions in the queue:
+
 ```
 export DATABASE_ENGINE=sqlite
 sender=$(eth-keyfile -d $WALLET_KEY_FILE)
-DATABASE_NAME=$HOME/.local/share/chaind/eth/chaind.sqlite chainqueue-list $sender
-# to show a summary only instead all transactions
-DATABASE_NAME=$HOME/.local/share/chaind/eth/chaind.sqlite chainqueue-list --summary $sender
+chainqueue-list $sender
 ```
 
-The `chainqueue-list` tool provides some basic filtering. Use `chainqueue-list --help` to see what they are.
+To show a summary only instead all transactions:
+
+```
+chainqueue-list --summary $sender
+```
+
+The `chaind-list` tool can be used to list by session id. Following the above examples:
+
+```
+chaind-list testsession
+```
+
+The `chainqueue-list` and `chaind-list` tools both provides the same basic filtering. Use `--help` to see the details.
+
+
+### Retrieve transaction by hash
+
+The socket server returns the transaction hash when a transaction is submitted.
+
+If a socket server is given a transaction hash, it will return the transaction data for that hash (if it exists).
+
+Extending the previous examples, this will output the original signed transaction:
+
+```
+eth-gas --raw -a $recipient 1024 > tx1.txt
+cat tx1.txt | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock - | cut -b 4- > hash1.txt 
+cat hash1.tx | socat UNIX-CLIENT=/run/user/$UID/chaind/eth/testsession/chaind.sock - | cut -b 4- > tx1_recovered.txt
+diff tx1_recovered.txt tx1.txt
+# should output 0
+echo $?
+```
+
+The first 4 bytes of the data returned from the socket is a 32-bit big-endian result code. The data payload follows from the 5th byte.
+
+
+## Batch processing
+
+The `chaind-eth-send` executable generates signed transactions with data from a csv file.
+
+The data columns must be in the following order:
+
+1. receipient address
+2. transaction value
+3. token specifier (optional, network fee token if not given)
+4. network fee token value (optional)
+
+
+If the gas token value (4) is not given for a gas token transaction, the transaction value (2) will be used.
+
+By default the signed transactions are output as hex to stdout, each on a separate line.
+
+If a valid `--socket` is given (i.e. the socket of the `chaind-eth-server`) the transactions will be send to the socket instead. The hash of the transaction will be output to standard output.
+
+
+### Using token symbols
+
+If token symols are to be used in some or all values of column 3, then a valid `--token-index` executable address is required (in this case, a smart contract implementing the [`registry`](https://gitlab.com/grassrootseconomics/cic-contracts/-/blob/master/solidity/Registry.sol) contract interface).
+
+
+### Input validity checks
+
+The validity of the input data is verified _before_ actual execution takes place.
+
+These checks include:
+
+- The token can be made sense of.
+- The values can be parsed to integer amounts.
+- The recipient address is a valid checksum address.
+
+The checks do however _not_ include whether the token balances of the signer are sufficient to successfully execute the transactions on the network.
+
+
+### CSV input example
+
+```
+0x72B70906fD07c72f2d96aAA250C2D31662D0d809,10,0xb708175e3f6Cd850643aAF7B32212AFad50e2549
+0xD536CB6d1d9B8d33875E0ba0Aa3515eD7478f889,0x2a,GFT,100
+0xeE08b59a95E822AE346489038D25750C8EdfcC25,0x029a
+```
+
+This will result in the following transactions:
+
+1. send 10 tokens from token contract `0xb708175e3f6Cd850643aAF7B32212AFad50e2549` to recipient `0x72B70906fD07c72f2d96aAA250C2D31662D0d809`.
+2. send 42 `GFT` tokens along with 100 network gas tokens to recipient `0xD536CB6d1d9B8d33875E0ba0Aa3515eD7478f889`
+3. send 666 network gas tokens to recipient `0xeE08b59a95E822AE346489038D25750C8EdfcC25`
+
+
+### Resending transactions
+
+Since the `chaind-eth-server` does not have access to signing keys, resending stalled transactions is also a separate external action.
+
+The `chaind-eth-resend` executable takes a list of signed transactions (e.g. as output from `chaind-eth-send` using the socket) and automatically increases the fee price of the transaction to create a replacement.
+
+As with `chaind-eth-send`, the resend executable optionally takes a socket argument that sends the transaction directly to a socket. Otherwise, the signed transactions are send to standard output.
+
+For example, the following will output details of the transaction generated by `chaind-eth-resend`, in which the fee price has been slightly incremented:
+
+```
+eth-gas --raw --fee-price 100000000 -a $recipient 1024 > tx1.txt
+chaind-eth-resend tx1.txt > tx1_bump.txt
+cat tx1_bump.txt | eth-decode
+```
+
+
+### Retrieving transactions for resend
+
+The `chaind-list` tool can be used to retrieve transactions with the same filters as `chainqueue-list`, but also allowing results limited a specific session id.
+
+As with `chainqueue-list`, which column to output can be customized. This enables creation of signed transaction lists in the format accepted by `chaind-eth-resent`.
+
+One examples of criteria for transactions due to be resent may be:
+
+```
+# get any pending transaction in session "testsession"
+export DATABASE_ENGINE=sqlite
+chaind-list -o signedtx --pending testsession
+```
+
+Note that the `chaind-list` tool requires a connection to the queueing backend.
 
 
 ## systemd

chaind_eth/cli/csv.py

@@ -16,10 +16,10 @@ class CSVProcessor:
 
         import csv # only import if needed
         fr = csv.reader(f)
-        f.close()
 
         for r in fr:
             contents.append(r)
+        f.close()
         l = len(contents)
         logg.info('successfully parsed source as csv, found {} records'.format(l))
         return contents

chaind_eth/cli/resolver.py

@@ -3,10 +3,50 @@ import logging
 
 # external imports
 from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.address import is_checksum_address
+from hexathon import strip_0x
+from eth_token_index.index import TokenUniqueSymbolIndex
 
 logg = logging.getLogger(__name__)
 
 
+class LookNoop:
+
+    def get(self, k, rpc=None):
+        try:
+            if not is_checksum_address(k):
+                raise ValueError('not valid checksum address {}'.format(k))
+        except ValueError:
+            raise ValueError('not valid checksum address {}'.format(k))
+        return strip_0x(k)
+
+
+    def __str__(self):
+        return 'checksum address shortcircuit'
+
+
+class TokenIndexLookup(TokenUniqueSymbolIndex):
+
+
+    def __init__(self, chain_spec, signer, gas_oracle, nonce_oracle, address, sender_address=ZERO_ADDRESS):
+        super(TokenIndexLookup, self).__init__(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+        self.local_address = address
+        self.sender_address = sender_address
+
+
+    def get(self, k, rpc=None):
+        o = self.address_of(self.local_address, k, sender_address=self.sender_address)
+        r = rpc.do(o)
+        address = self.parse_address_of(r)
+        if address != ZERO_ADDRESS:
+            return address
+        raise FileNotFoundError(address)
+
+
+    def __str__(self):
+        return 'token symbol index'
+
+
 class DefaultResolver:
 
     def __init__(self, chain_spec, rpc, sender_address=ZERO_ADDRESS):
@@ -18,19 +58,20 @@ class DefaultResolver:
         self.sender_address = sender_address
 
 
-    def add_lookup(self, lookup, address):
+    def add_lookup(self, lookup, reverse):
         self.lookups.append(lookup)
-        self.lookup_pointers.append(address)
+        self.lookup_pointers.append(reverse)
 
 
     def lookup(self, k):
         if k == '' or k == None:
             return None
-        for i, lookup in enumerate(self.lookups):
-            address = self.lookup_pointers[i]
-            o = lookup.address_of(address, k, sender_address=self.sender_address)
-            r = self.rpc.do(o)
-            address = lookup.parse_address_of(r)
-            if address != ZERO_ADDRESS:
+        for lookup in self.lookups:
+            try:
+                address = lookup.get(k, rpc=self.rpc)
+                logg.debug('resolved token {} to {} with lookup {}'.format(k, address, lookup))
                 return address
+            except Exception as e:
+                logg.debug('lookup {} failed for {}: {}'.format(lookup, k, e))
+
         raise FileNotFoundError(k)

chaind_eth/runnable/resend.py

@@ -33,14 +33,12 @@ config_dir = os.path.join(script_dir, '..', 'data', 'config')
 arg_flags = chainlib.eth.cli.argflag_std_write
 argparser = chainlib.eth.cli.ArgumentParser(arg_flags)
 argparser.add_argument('--socket', dest='socket', type=str, help='Socket to send transactions to')
-argparser.add_argument('--token-index', dest='token_index', type=str, help='Token resolver index')
 argparser.add_positional('source', required=False, type=str, help='Transaction source file')
 args = argparser.parse_args()
 
 extra_args = {
         'socket': None,
         'source': None,
-        'token_index': None,
         }
 
 env = Environment(domain='eth', env=os.environ)

chaind_eth/runnable/send.py

@@ -14,13 +14,16 @@ from chaind import Environment
 from chainlib.eth.gas import price
 from chainlib.chain import ChainSpec
 from hexathon import strip_0x
-from eth_token_index.index import TokenUniqueSymbolIndex
 
 # local imports
 from chaind_eth.cli.process import Processor
 from chaind_eth.cli.csv import CSVProcessor
 from chaind.error import TxSourceError
-from chaind_eth.cli.resolver import DefaultResolver
+from chaind_eth.cli.resolver import (
+        DefaultResolver,
+        LookNoop,
+        TokenIndexLookup,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -83,18 +86,6 @@ if config.get('_SOURCE') == None:
     sys.exit(1)
 
 
-class TokenIndexLookupAdapter(TokenUniqueSymbolIndex):
-
-    def __init__(self, sender, address, chain_spec, signer=None, gas_oracle=None, nonce_oracle=None):
-        super(TokenIndexLookupAdapter, self).__init__(chain_spec, signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-        self.index_address = address
-        self.sender = sender
-
-
-    def resolve(self, v):
-        return self.address_of(self.index_address, v, sender_address=sender)
-
-
 class Outputter:
 
     def __init__(self, mode):
@@ -123,10 +114,16 @@ class Outputter:
 def main():
     signer = rpc.get_signer()
 
+
     # TODO: make resolvers pluggable
     token_resolver = DefaultResolver(chain_spec, conn, sender_address=rpc.get_sender_address())
-    token_index_lookup = TokenUniqueSymbolIndex(chain_spec, signer=signer, gas_oracle=rpc.get_gas_oracle(), nonce_oracle=rpc.get_nonce_oracle())
-    token_resolver.add_lookup(token_index_lookup, config.get('_TOKEN_INDEX'))
+
+    noop_lookup = LookNoop()
+    token_resolver.add_lookup(noop_lookup, 'noop')
+  
+    if config.get('_TOKEN_INDEX') != None:
+        token_index_lookup = TokenIndexLookup(chain_spec, signer, rpc.get_gas_oracle(), rpc.get_nonce_oracle(), config.get('_TOKEN_INDEX'))
+        token_resolver.add_lookup(token_index_lookup, reverse=config.get('_TOKEN_INDEX'))
     
     processor = Processor(wallet.get_signer_address(), wallet.get_signer(), config.get('_SOURCE'), chain_spec, rpc.get_gas_oracle(), rpc.get_nonce_oracle(), resolver=token_resolver)
     processor.add_processor(CSVProcessor())

requirements.txt

@@ -1,4 +1,4 @@
 chaind<=0.0.3,>=0.0.3a5
 hexathon~=0.0.1a8
-chainlib-eth<=0.1.0,>=0.0.9a9
-eth-address-index<=0.3.0,>=0.2.3a4
+chainlib-eth<=0.1.0,>=0.0.9a10
+eth-address-index<=0.3.0,>=0.2.3a5