Bump chaind dep

.gitignore

@@ -6,3 +6,4 @@ build/
 *.egg-info
 *.sqlite
 .coverage
+.chaind

chaind/eth/cache.py

@@ -21,7 +21,6 @@ class Normalizer(TxHexNormalizer, NoopNormalizer):
         hexathon.to_int(v)
 
 
-
 eth_normalizer = Normalizer()
 
 

chaind/eth/data/config/config.ini

@@ -4,3 +4,6 @@ runtime_dir =
 id =
 data_dir =
 dispatch_delay = 0.01
+
+[tx]
+digest_size = 32

chaind/eth/runnable/queuer.py

@@ -5,8 +5,23 @@ import signal
 
 # external imports
 import chainlib.eth.cli
-import chaind.cli
-import chainqueue.cli
+from chainlib.eth.cli.arg import (
+        Arg,
+        ArgFlag,
+        process_args,
+        )
+from chainlib.eth.cli.config import (
+        Config,
+        process_config,
+        )
+from chainqueue.cli.arg import (
+        apply_arg as apply_arg_queue,
+        apply_flag as apply_flag_queue,
+        )
+from chaind.cli.arg import (
+        apply_arg,
+        apply_flag,
+        )
 from chaind.session import SessionController
 from chaind.setup import Environment
 from chaind.error import (
@@ -28,11 +43,22 @@ from chainqueue.cache import CacheTokenTx
 from chainlib.encode import TxHexNormalizer
 from chainlib.chain import ChainSpec
 from chaind.adapters.fs import ChaindFsAdapter
+from chaind.dispatch import DispatchProcessor
+from chainqueue.data import config_dir as chainqueue_config_dir
+from chaind.data import config_dir as chaind_config_dir
+from chainlib.eth.cli.log import process_log
+from chaind.cli.config import process_config as process_config_local
 
 # local imports
-from chaind.eth.dispatch import EthDispatcher
 from chaind.eth.cache import EthCacheTx
-from chaind.eth.settings import ChaindEthSettings
+from chaind.eth.settings import ChaindSettings
+from chaind.eth.dispatch import EthDispatcher
+from chaind.eth.settings import process_settings
+from chaind.settings import (
+        process_queue,
+        process_socket,
+        process_dispatch,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -42,80 +68,63 @@ config_dir = os.path.join(script_dir, '..', 'data', 'config')
 
 env = Environment(domain='eth', env=os.environ)
 
-arg_flags = chainlib.eth.cli.argflag_std_read
-argparser = chainlib.eth.cli.ArgumentParser(arg_flags)
+arg_flags = ArgFlag()
+arg_flags = apply_flag_queue(arg_flags)
+arg_flags = apply_flag(arg_flags)
 
-queue_arg_flags = 0
-chainqueue.cli.process_flags(argparser, queue_arg_flags)
+arg = Arg(arg_flags)
+arg = apply_arg_queue(arg)
+arg = apply_arg(arg)
 
-local_arg_flags = chaind.cli.argflag_local_base | chaind.cli.ChaindFlag.DISPATCH | chaind.cli.ChaindFlag.SOCKET
-chaind.cli.process_flags(argparser, local_arg_flags)
+flags = arg_flags.STD_READ | arg_flags.QUEUE | arg_flags.STATE | arg_flags.SESSION
 
+argparser = chainlib.eth.cli.ArgumentParser()
+argparser = process_args(argparser, arg, flags)
 args = argparser.parse_args()
 
-base_config_dir = [chainqueue.cli.config_dir, chaind.cli.config_dir]
-config = chainlib.eth.cli.Config.from_args(args, arg_flags, base_config_dir=base_config_dir)
-config = chaind.cli.process_config(config, args, local_arg_flags)
-config = chainqueue.cli.process_config(config, args, queue_arg_flags)
+logg = process_log(args, logg)
+
+config = Config()
+config.add_schema_dir(chainqueue_config_dir)
+config.add_schema_dir(chaind_config_dir)
+config = process_config(config, arg, args, flags)
+config = process_config_local(config, arg, args, flags)
 config.add('eth', 'CHAIND_ENGINE', False)
-config.add('queue', 'CHAIND_COMPONENT', False)
+config.add('sync', 'CHAIND_COMPONENT', False)
 logg.debug('config loaded:\n{}'.format(config))
 
-settings = ChaindEthSettings(include_queue=True)
-settings.process(config)
-
-logg.debug('settings:\n{}'.format(settings))
-
-
-def process_outgoing(chain_spec, adapter, rpc, limit=50):
-    adapter = None
-    process_err = None
-    for i in range(2):
-        try:
-            adapter = ChaindFsAdapter(
-                settings.get('CHAIN_SPEC'),
-                settings.dir_for('queue'),
-                EthCacheTx,
-                dispatcher,
-                )
-        except BackendIntegrityError as e:
-            process_err = e
-            continue
-
-    if adapter == None:
-        raise BackendIntegrityError(process_err)
-    
-    upcoming = adapter.upcoming(limit=limit)
-    logg.info('processor has {} candidates for {}, processing with limit {} adapter {} rpc {}'.format(len(upcoming), chain_spec, limit, adapter, rpc))
-    i = 0
-    for tx_hash in upcoming:
-        if adapter.dispatch(tx_hash):
-            i += 1
-    return i
-
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-
-rpc = chainlib.eth.cli.Rpc()
-conn = rpc.connect_by_config(config)
+settings = ChaindSettings(include_sync=True)
+settings = process_settings(settings, config)
+settings = process_queue(settings, config)
+settings = process_socket(settings, config)
+settings = process_dispatch(settings, config)
+logg.debug('settings loaded:\n{}'.format(settings))
 
 tx_normalizer = TxHexNormalizer().tx_hash
-token_cache_store = CacheTokenTx(chain_spec, normalizer=tx_normalizer)
-dispatcher = EthDispatcher(conn)
-queue_adapter = ChaindFsAdapter(
-        settings.get('CHAIN_SPEC'),
-        settings.dir_for('queue'),
-        EthCacheTx,
-        dispatcher,
-        )
+token_cache_store = CacheTokenTx(settings.get('CHAIN_SPEC'), normalizer=tx_normalizer)
+
+dispatcher = EthDispatcher(settings.get('CONN'))
+processor = DispatchProcessor(settings.get('CHAIN_SPEC'), settings.dir_for('queue'), dispatcher)
+ctrl = SessionController(settings, processor.process)
 
-ctrl = SessionController(settings, queue_adapter, process_outgoing)
 signal.signal(signal.SIGINT, ctrl.shutdown)
 signal.signal(signal.SIGTERM, ctrl.shutdown)
 
 logg.info('session id is ' + settings.get('SESSION_ID'))
 logg.info('session socket path is ' + settings.get('SESSION_SOCKET_PATH'))
 
+
 def main():
+    global dispatcher, settings
+
+    queue_adapter = ChaindFsAdapter(
+        settings.get('CHAIN_SPEC'),
+        settings.dir_for('queue'),
+        EthCacheTx,
+        dispatcher,
+        store_sync=False,
+        )
+
     while True:
         v = None
         client_socket = None
@@ -131,7 +140,8 @@ def main():
             pass
 
         if v == None:
-            ctrl.process(conn)
+            ctrl.process(settings.get('CONN'))
+            #queue_adapter = create_adapter(settings, dispatcher)
             continue
 
         result_data = None

chaind/eth/runnable/send.py

@@ -10,15 +10,40 @@ import socket
 
 # external imports
 import chainlib.eth.cli
-import chainqueue.cli
-import chaind.cli
+from chainlib.eth.cli.arg import (
+        Arg,
+        ArgFlag,
+        process_args,
+        )
+from chainlib.eth.cli.config import (
+        Config,
+        process_config,
+        )
 from chaind.setup import Environment
 from chainlib.eth.gas import price
 from chainlib.chain import ChainSpec
 from hexathon import strip_0x
+from chainqueue.cli.arg import (
+        apply_arg as apply_arg_queue,
+        apply_flag as apply_flag_queue,
+        )
+from chainqueue.data import config_dir as chainqueue_config_dir
+from chaind.data import config_dir as chaind_config_dir
+from chaind.cli.arg import (
+        apply_arg,
+        apply_flag,
+        )
+from chainlib.eth.cli.log import process_log
+from chaind.settings import process_queue
+from chaind.settings import ChaindSettings
+from chaind.error import TxSourceError
+from chainlib.error import (
+        InitializationError,
+        SignerMissingException,
+        )
+from chaind.cli.config import process_config as process_config_local
 
 # local imports
-from chaind.error import TxSourceError
 from chaind.eth.token.process import Processor
 from chaind.eth.token.gas import GasTokenResolver
 from chaind.eth.cli.csv import CSVProcessor
@@ -26,45 +51,56 @@ from chaind.eth.cli.output import (
         Outputter,
         OpMode,
         )
-from chaind.eth.settings import ChaindEthSettings
+from chaind.eth.settings import process_settings
 
-logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = chainlib.eth.cli.argflag_std_write
-argparser = chainlib.eth.cli.ArgumentParser(arg_flags, arg_long={'-s': '--send-rpc'})
-argparser.add_positional('source', required=False, type=str, help='Transaction source file')
 
-local_arg_flags = chaind.cli.argflag_local_socket_client | chaind.cli.ChaindFlag.TOKEN
-chaind.cli.process_flags(argparser, local_arg_flags)
+def process_settings_local(settings, config):
+#    if settings.get('SIGNER') == None:
+#        raise SignerMissingException('signer missing')
+    return settings
 
-chainqueue.cli.process_flags(argparser, 0)
-
-args = argparser.parse_args()
 
 env = Environment(domain='eth', env=os.environ)
 
-base_config_dir = [chaind.cli.config_dir, chainqueue.cli.config_dir]
-config = chainlib.eth.cli.Config.from_args(args, arg_flags, base_config_dir=base_config_dir)
-config = chainqueue.cli.process_config(config, args, 0)
-config = chaind.cli.process_config(config, args, local_arg_flags)
+arg_flags = ArgFlag()
+arg_flags = apply_flag_queue(arg_flags)
+arg_flags = apply_flag(arg_flags)
+
+arg = Arg(arg_flags)
+arg = apply_arg_queue(arg)
+arg = apply_arg(arg)
+arg.set_long('s', 'send-rpc')
+
+flags = arg_flags.STD_WRITE | arg_flags.TOKEN | arg_flags.SOCKET_CLIENT | arg_flags.STATE | arg_flags.WALLET | arg_flags.SESSION
+
+argparser = chainlib.eth.cli.ArgumentParser()
+argparser = process_args(argparser, arg, flags)
+argparser.add_argument('source', help='Transaction source file')
+args = argparser.parse_args()
+
+logg = process_log(args, logg)
+
+config = Config()
+config.add_schema_dir(chainqueue_config_dir)
+config.add_schema_dir(chaind_config_dir)
+config = process_config(config, arg, args, flags)
+config = process_config_local(config, arg, args, flags)
 config.add(args.source, '_SOURCE', False)
-config.add('eth', 'CHAIND_ENGINE', False)
 config.add('queue', 'CHAIND_COMPONENT', False)
+config.add('eth', 'CHAIND_ENGINE', False)
 logg.debug('config loaded:\n{}'.format(config))
 
-wallet = chainlib.eth.cli.Wallet()
-wallet.from_config(config)
-
-settings = ChaindEthSettings(include_queue=True)
-settings.process(config)
-
-logg.debug('settings:\n{}'.format(settings))
-
-rpc = chainlib.eth.cli.Rpc(wallet=wallet)
-conn = rpc.connect_by_config(config)
-
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+try:
+    settings = ChaindSettings(include_sync=True)
+    settings = process_settings(settings, config)
+    settings = process_queue(settings, config)
+    settings = process_settings_local(settings, config)
+except InitializationError as e:
+    sys.stderr.write('Initialization error: ' + str(e) + '\n')
+    sys.exit(1)
+logg.debug('settings loaded:\n{}'.format(settings))
 
 mode = OpMode.STDOUT
 
@@ -100,7 +136,14 @@ class SocketSender:
 
     def send(self, tx):
         s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        s.connect(self.path)
+        err = None
+        try:
+            s.connect(self.path)
+        except FileNotFoundError as e:
+            err = e
+        if err != None:
+            s.close()
+            raise err
         s.sendall(tx.encode('utf-8'))
         r = s.recv(68)
         s.close()
@@ -108,6 +151,7 @@ class SocketSender:
 
 
 def main():
+    conn = settings.get('CONN')
     token_resolver = None
     if settings.get('TOKEN_MODULE') != None:
         import importlib
@@ -116,9 +160,15 @@ def main():
     else:
         from chaind.eth.token.gas import GasTokenResolver
         m = GasTokenResolver
-    token_resolver = m(chain_spec, rpc.get_sender_address(), rpc.get_signer(), rpc.get_gas_oracle(), rpc.get_nonce_oracle())
+    token_resolver = m(
+            settings.get('CHAIN_SPEC'),
+            settings.get('SENDER_ADDRESS'),
+            settings.get('SIGNER'),
+            settings.get('GAS_ORACLE'),
+            settings.get('NONCE_ORACLE'),
+            )
     
-    processor = Processor(token_resolver, config.get('_SOURCE'))
+    processor = Processor(token_resolver, config.get('_SOURCE'), use_checksum=not config.get('_UNSAFE'))
     processor.add_processor(CSVProcessor())
 
     sends = None
@@ -143,7 +193,12 @@ def main():
             break
         tx_hex = tx_bytes.hex()
         if sender != None:
-            r = sender.send(tx_hex)
+            r = None
+            try:
+                r = sender.send(tx_hex)
+            except FileNotFoundError as e:
+                sys.stderr.write('send to socket {} failed: {}\n'.format(sender.path, e))
+                sys.exit(1)
             logg.info('sent {} result {}'.format(tx_hex, r))
         print(out.do(tx_hex))
 

chaind/eth/runnable/syncer.py

@@ -3,9 +3,7 @@ import os
 import logging
 
 # external imports
-import chainlib.cli
-import chainsyncer.cli
-import chaind.cli
+import chainlib.eth.cli
 from chaind.setup import Environment
 from chaind.filter import StateFilter
 from chainlib.eth.block import block_latest
@@ -13,15 +11,38 @@ from hexathon import strip_0x
 from chainsyncer.store.fs import SyncFsStore
 from chainsyncer.driver.chain_interface import ChainInterfaceDriver
 from chainsyncer.error import SyncDone
-
-# local imports
-from chaind.eth.settings import ChaindEthSettings
-
+from chainlib.eth.cli.arg import (
+        Arg,
+        ArgFlag,
+        process_args,
+        )
+from chainlib.eth.cli.config import (
+        Config,
+        process_config,
+        )
+from chainsyncer.cli.arg import (
+        apply_arg as apply_arg_sync,
+        apply_flag as apply_flag_sync,
+        )
+from chainsyncer.data import config_dir as chainsyncer_config_dir
+from chaind.data import config_dir as chaind_config_dir
+from chaind.cli.arg import (
+        apply_arg,
+        apply_flag,
+        )
+from chainlib.eth.cli.log import process_log
+from chaind.settings import ChaindSettings
+from chaind.cli.config import process_config as process_config_local
+from chainsyncer.cli.config import process_config as process_config_syncer
 
 # local imports
 from chaind.eth.cache import EthCacheTx
+from chaind.eth.settings import (
+    process_settings,
+    process_sync,
+    )
+
 
-logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
@@ -29,44 +50,50 @@ config_dir = os.path.join(script_dir, '..', 'data', 'config')
 
 env = Environment(domain='eth', env=os.environ)
 
-arg_flags = chainlib.cli.argflag_std_base | chainlib.cli.Flag.CHAIN_SPEC
-argparser = chainlib.cli.ArgumentParser(arg_flags)
+arg_flags = ArgFlag()
+arg_flags = apply_flag_sync(arg_flags)
+arg_flags = apply_flag(arg_flags)
 
-local_arg_flags = chaind.cli.argflag_local_base
-chaind.cli.process_flags(argparser, local_arg_flags)
+arg = Arg(arg_flags)
+arg = apply_arg_sync(arg)
+arg = apply_arg(arg)
 
-sync_flags = chainsyncer.cli.SyncFlag.RANGE | chainsyncer.cli.SyncFlag.HEAD
-chainsyncer.cli.process_flags(argparser, sync_flags)
+flags = arg_flags.STD_BASE | arg_flags.CHAIN_SPEC | arg_flags.PROVIDER | arg_flags.SEQ | arg_flags.STATE
+flags = arg_flags.more(flags, arg_flags.SYNC_RANGE_EXT)
+flags = arg_flags.more(flags, arg_flags.CHAIND_BASE)
 
+argparser = chainlib.eth.cli.ArgumentParser()
+argparser = process_args(argparser, arg, flags)
 args = argparser.parse_args()
 
-base_config_dir = [
-    chainsyncer.cli.config_dir,
-    chaind.cli.config_dir,
-        ]
-config = chainlib.cli.Config.from_args(args, arg_flags, base_config_dir=base_config_dir)
-config = chainsyncer.cli.process_config(config, args, sync_flags)
-config = chaind.cli.process_config(config, args, local_arg_flags)
+logg = process_log(args, logg)
+
+config = Config()
+config.add_schema_dir(chainsyncer_config_dir)
+config.add_schema_dir(chaind_config_dir)
+config = process_config(config, arg, args, flags)
+config = process_config_local(config, arg, args, flags)
+config = process_config_syncer(config, arg, args, flags)
 config.add('eth', 'CHAIND_ENGINE', False)
 config.add('sync', 'CHAIND_COMPONENT', False)
 logg.debug('config loaded:\n{}'.format(config))
 
-settings = ChaindEthSettings(include_sync=True)
-settings.process(config)
-
-logg.debug('settings:\n{}'.format(settings))
+settings = ChaindSettings(include_sync=True)
+settings = process_settings(settings, config)
+settings = process_sync(settings, config)
+logg.debug('settings loaded:\n{}'.format(settings))
 
 
 def main():
     fltr = StateFilter(settings.get('CHAIN_SPEC'), settings.dir_for('queue'), EthCacheTx)
-    sync_store = SyncFsStore(settings.get('SESSION_DATA_DIR'), session_id=settings.get('SESSION_ID'))
+    sync_store = SyncFsStore(settings.get('SESSION_DATA_PATH'), session_id=settings.get('SESSION_ID'))
     sync_store.register(fltr)
 
     logg.debug('session block offset {}'.format(settings.get('SYNCER_OFFSET')))
 
     drv = ChainInterfaceDriver(sync_store, settings.get('SYNCER_INTERFACE'), offset=settings.get('SYNCER_OFFSET'), target=settings.get('SYNCER_LIMIT'))
     try:
-        drv.run(settings.get('RPC'))
+        drv.run(settings.get('CONN'))
     except SyncDone as e:
         logg.info('sync done: {}'.format(e))
    

chaind/eth/settings.py

@@ -1,18 +1,31 @@
 # external imports
 from chainlib.eth.connection import EthHTTPConnection
-from chaind.settings import ChaindSettings
+from chainlib.eth.settings import process_settings as base_process_settings
 from chaind.eth.chain import EthChainInterface
+from chaind.settings import *
+from chainsyncer.settings import process_sync_range
 
 
-class ChaindEthSettings(ChaindSettings):
-
-    def process_sync_interface(self, config):
-        self.o['SYNCER_INTERFACE'] = EthChainInterface()
+def process_common(settings, config):
+    rpc_provider = config.get('RPC_PROVIDER')
+    if rpc_provider == None:
+        rpc_provider = 'http://localhost:8545'
+    conn = EthHTTPConnection(url=rpc_provider, chain_spec=settings.get('CHAIN_SPEC'))
+    settings.set('RPC', conn)
+    return settings
 
 
-    def process_common(self, config):
-        super(ChaindEthSettings, self).process_common(config)
-        rpc_provider = config.get('RPC_PROVIDER')
-        if rpc_provider == None:
-            rpc_provider = 'http://localhost:8545'
-        self.o['RPC'] = EthHTTPConnection(url=rpc_provider, chain_spec=self.o['CHAIN_SPEC'])
+def process_sync(settings, config):
+    settings.set('SYNCER_INTERFACE', EthChainInterface())
+    settings = process_sync_range(settings, config)
+    return settings
+
+
+def process_settings(settings, config):
+    settings = base_process_settings(settings, config)
+    settings = process_common(settings, config)
+    settings = process_backend(settings, config)
+    settings = process_session(settings, config)
+    settings = process_socket(settings, config)
+
+    return settings

chaind/eth/token/process.py

@@ -3,7 +3,10 @@ import logging
 
 # external imports
 from chaind.error import TxSourceError
-from chainlib.eth.address import is_checksum_address
+from chainlib.eth.address import (
+        is_checksum_address,
+        to_checksum_address,
+        )
 from chainlib.eth.tx import unpack
 from chainlib.eth.gas import Gas
 from hexathon import (
@@ -17,10 +20,11 @@ logg = logging.getLogger(__name__)
 
 class Processor:
 
-    def __init__(self, resolver, source):
+    def __init__(self, resolver, source, use_checksum=True):
         self.resolver = resolver
         self.source = source
         self.processor = []
+        self.safe = use_checksum
         self.conn = None
         
 
@@ -50,9 +54,14 @@ class Processor:
         txs = []
         for i, r in enumerate(self.content):
             logg.debug('processing {}'.format(r))
-            if not is_checksum_address(r[0]):
-                raise ValueError('invalid checksum address {} in record {}'.format(r[0], i))
-            self.content[i][0] = add_0x(r[0])
+            address = r[0]
+            if self.safe:
+                if not is_checksum_address(address):
+                    raise ValueError('invalid checksum address {} in record {}'.format(address, i))
+            else:
+                address = to_checksum_address(address)
+
+            self.content[i][0] = add_0x(address)
             try:
                 self.content[i][1] = int(r[1])
             except ValueError:

erc20_requirements.txt

@@ -1 +1 @@
-eth-erc20~=0.3.0
+eth-erc20~=0.3.2

requirements.txt

@@ -1,6 +1,5 @@
-chaind~=0.2.0
-hexathon~=0.1.5
-chainlib-eth~=0.1.1
+chaind~=0.3.0
+hexathon~=0.1.7
+chainlib-eth~=0.3.0
 pyxdg~=0.27
-shep~=0.2.3
-funga-eth~=0.6.0
+funga-eth~=0.6.1

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = chaind-eth
-version = 0.2.0
+version = 0.3.0
 description = Queue server for ethereum
 author = Louis Holbrook
 author_email = dev@holbrook.no

tests/test_tx.py

@@ -11,7 +11,7 @@ from chainlib.chain import ChainSpec
 from chainqueue.cache import CacheTokenTx
 from chainlib.error import RPCException
 from chainlib.status import Status as TxStatus
-from chaind.unittest.common import TestChaindFsBase
+from chaind.unittest.fs import TestChaindFsBase
 from chaind.driver import QueueDriver
 from chaind.filter import StateFilter
 from chainlib.eth.gas import Gas