Merge branch 'lash/http-real'

MANIFEST.in

@@ -0,0 +1 @@
+include *requirements*

TODO

@@ -0,0 +1,6 @@
+Missing tests for:
+
+- crypto_dev_signer/cli/http.py
+- crypto_dev_signer/cli/socket.py
+
+tests/test_keystore_reference.py is dependent on postgres, should use sqlite for tests so that it can run independently

crypto_dev_signer/cli/handle.py

@@ -0,0 +1,115 @@
+# standard imports
+import json
+import logging
+
+# external imports
+from jsonrpc.exceptions import (
+        JSONRPCServerError,
+        JSONRPCParseError,
+        JSONRPCInvalidParams,
+        )
+from hexathon import add_0x
+
+# local imports
+from crypto_dev_signer.eth.transaction import EIP155Transaction
+from crypto_dev_signer.error import (
+        UnknownAccountError,
+        SignerError,
+        )
+from crypto_dev_signer.cli.jsonrpc import jsonrpc_ok
+from .jsonrpc import (
+        jsonrpc_error,
+        is_valid_json,
+        )
+
+logg = logging.getLogger(__name__)
+
+
+class SignRequestHandler:
+
+    keystore = None
+    signer = None
+
+    def process_input(self, j):
+        rpc_id = j['id']
+        m = j['method']
+        p = j['params']
+        return (rpc_id, getattr(self, m)(p))
+
+
+    def handle_jsonrpc(self, d):
+        j = None
+        try:
+            j = json.loads(d)
+            is_valid_json(j)
+            logg.debug('{}'.format(d.decode('utf-8')))
+        except Exception as e:
+            logg.exception('input error {}'.format(e))
+            j = json.dumps(jsonrpc_error(None, JSONRPCParseError)).encode('utf-8')
+            raise SignerError(j)
+
+        try:
+            (rpc_id, r) = self.process_input(j)
+            r = jsonrpc_ok(rpc_id, r)
+            j = json.dumps(r).encode('utf-8')
+        except ValueError as e:
+            # TODO: handle cases to give better error context to caller
+            logg.exception('process error {}'.format(e))
+            j = json.dumps(jsonrpc_error(j['id'], JSONRPCServerError)).encode('utf-8')
+            raise SignerError(j)
+        except UnknownAccountError as e:
+            logg.exception('process unknown account error {}'.format(e))
+            j = json.dumps(jsonrpc_error(j['id'], JSONRPCServerError)).encode('utf-8')
+            raise SignerError(j)
+
+        return j
+
+
+    def personal_newAccount(self, p):
+        password = p
+        if p.__class__.__name__ != 'str':
+            if p.__class__.__name__ != 'list':
+                e = JSONRPCInvalidParams()
+                e.data = 'parameter must be list containing one string'
+                raise ValueError(e)
+            logg.error('foo {}'.format(p))
+            if len(p) != 1:
+                e = JSONRPCInvalidParams()
+                e.data = 'parameter must be list containing one string'
+                raise ValueError(e)
+            if p[0].__class__.__name__ != 'str':
+                e = JSONRPCInvalidParams()
+                e.data = 'parameter must be list containing one string'
+                raise ValueError(e)
+            password = p[0]
+
+        r = self.keystore.new(password)
+                 
+        return add_0x(r)
+
+
+    # TODO: move to translation module ("personal" rpc namespace is node-specific)
+    def personal_signTransaction(self, p):
+        logg.debug('got {} to sign'.format(p[0]))
+        t = EIP155Transaction(p[0], p[0]['nonce'], p[0]['chainId'])
+        raw_signed_tx = self.signer.sign_transaction_to_rlp(t, p[1])
+        o = {
+            'raw': '0x' + raw_signed_tx.hex(),
+            'tx': t.serialize(),
+            }
+        return o
+
+
+    def eth_signTransaction(self, tx):
+        o = self.personal_signTransaction([tx[0], ''])
+        return o['raw']
+
+
+    def eth_sign(self, p):
+        logg.debug('got message {} to sign'.format(p[1]))
+        message_type = type(p[1]).__name__
+        if message_type != 'str':
+            raise ValueError('invalid message format, must be {}, not {}'.format(message_type))
+        z = self.signer.sign_ethereum_message(p[0], p[1][2:])
+        return add_0x(z.hex())
+

crypto_dev_signer/cli/http.py

@@ -0,0 +1,85 @@
+# standard imports
+import logging
+
+# external imports
+from http.server import (
+        HTTPServer,
+        BaseHTTPRequestHandler,
+    )
+
+# local imports
+from .handle import SignRequestHandler
+from crypto_dev_signer.error import SignerError
+
+logg = logging.getLogger(__name__)
+
+
+def start_server_http(spec):
+    httpd = HTTPServer(spec, HTTPSignRequestHandler)
+    logg.debug('starting http server {}'.format(spec))
+    httpd.serve_forever()
+
+
+class HTTPSignRequestHandler(SignRequestHandler, BaseHTTPRequestHandler):
+
+    def do_POST(self):
+        if self.headers.get('Content-Type') != 'application/json':
+            self.send_response(400, 'me read json only')
+            self.end_headers()
+            return
+
+        try:
+            if 'application/json' not in self.headers.get('Accept').split(','):
+                self.send_response(400, 'me json only speak')
+                self.end_headers()
+                return
+        except AttributeError:
+            pass
+
+        l = self.headers.get('Content-Length')
+        try:
+            l = int(l)
+        except ValueError:
+            self.send_response(400, 'content length must be integer')
+            self.end_headers()
+            return
+        if l > 4096:
+            self.send_response(400, 'too much information')
+            self.end_headers()
+            return
+        if l < 0:
+            self.send_response(400, 'you are too negative')
+            self.end_headers()
+            return
+
+        b = b''
+        c = 0
+        while c < l:
+            d = self.rfile.read(l-c)
+            if d == None:
+                break
+            b += d
+            c += len(d)
+            if c > 4096:
+                self.send_response(413, 'i should slap you around for lying about your size')
+                self.end_headers()
+                return
+
+        try:
+            r = self.handle_jsonrpc(d)
+        except SignerError as e:
+            r = e.to_jsonrpc()
+
+        l = len(r)
+        self.send_response(200, 'You are the Keymaster')
+        self.send_header('Content-Length', str(l))
+        self.send_header('Cache-Control', 'no-cache')
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+
+        c = 0
+        while c < l:
+            n = self.wfile.write(r[c:])
+            c += n
+
+

crypto_dev_signer/cli/jsonrpc.py

@@ -0,0 +1,30 @@
+# local imports
+from crypto_dev_signer.error import UnknownAccountError
+
+
+def jsonrpc_error(rpc_id, err):
+    return {
+            'jsonrpc': '2.0',
+            'id': rpc_id,
+            'error': {
+                'code': err.CODE,
+                'message': err.MESSAGE,
+                },
+            }
+
+
+def jsonrpc_ok(rpc_id, response):
+    return {
+            'jsonrpc': '2.0',
+            'id': rpc_id,
+            'result': response,
+            }
+
+
+def is_valid_json(j):
+    if j.get('id') == 'None':
+        raise ValueError('id missing')
+    return True
+
+
+

crypto_dev_signer/cli/socket.py

@@ -0,0 +1,67 @@
+# standard imports
+import os
+import logging 
+import socket
+import stat
+
+# local imports
+from crypto_dev_signer.error import SignerError
+from .handle import SignRequestHandler
+
+logg = logging.getLogger(__name__)
+
+
+class SocketHandler:
+
+    def __init__(self):
+        self.handler = SignRequestHandler()
+
+
+    def process(self, csock):
+        d = csock.recv(4096)
+       
+        r = None
+        try:
+            r = self.handler.handle_jsonrpc(d)
+        except SignerError as e:
+            r = e.to_jsonrpc()
+
+        csock.send(r)
+
+
+def start_server_socket(s):
+    s.listen(10)
+    logg.debug('server started')
+    handler = SocketHandler()
+    while True:
+        (csock, caddr) = s.accept()
+        handler.process(csock)
+        csock.close()
+    s.close()
+    os.unlink(socket_path)
+
+
+def start_server_tcp(spec):
+    s = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
+    s.bind(spec)
+    logg.debug('created tcp socket {}'.format(spec))
+    start_server_socket(s)
+
+
+def start_server_unix(socket_path):
+    socket_dir = os.path.dirname(socket_path)
+    try:
+        fi = os.stat(socket_dir)
+        if not stat.S_ISDIR:
+            RuntimeError('socket path {} is not a directory'.format(socket_dir))
+    except FileNotFoundError:
+        os.mkdir(socket_dir)
+
+    try:
+        os.unlink(socket_path)
+    except FileNotFoundError:
+        pass
+    s = socket.socket(family = socket.AF_UNIX, type = socket.SOCK_STREAM)
+    s.bind(socket_path)
+    logg.debug('created unix ipc socket {}'.format(socket_path))
+    start_server_socket(s)

crypto_dev_signer/error.py

@@ -8,3 +8,17 @@ class TransactionRevertError(Exception):
 
 class NetworkError(Exception):
     pass
+
+
+class SignerError(Exception):
+
+    def __init__(self, s):
+        super(SignerError, self).__init__(s)
+        self.jsonrpc_error = s
+
+
+    def to_jsonrpc(self):
+        return self.jsonrpc_error
+
+
+

crypto_dev_signer/runnable/signer.py

@@ -2,8 +2,6 @@
 import re
 import os
 import sys
-import stat
-import socket
 import json
 import logging 
 import argparse
@@ -12,13 +10,11 @@ from urllib.parse import urlparse
 # external imports
 import confini
 from jsonrpc.exceptions import *
-from hexathon import add_0x
 
 # local imports
 from crypto_dev_signer.eth.signer import ReferenceSigner
-from crypto_dev_signer.eth.transaction import EIP155Transaction
 from crypto_dev_signer.keystore.reference import ReferenceKeystore
-from crypto_dev_signer.error import UnknownAccountError
+from crypto_dev_signer.cli.handle import SignRequestHandler
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -72,171 +68,15 @@ logg.info('using dsn {}'.format(dsn))
 logg.info('using socket {}'.format(config.get('SIGNER_SOCKET_PATH')))
 
 re_http = r'^http'
+re_tcp = r'^tcp'
 re_unix = r'^ipc'
 
-class MissingSecretError(BaseException):
+class MissingSecretError(Exception):
-
-    def __init__(self, message):
-        super(MissingSecretError, self).__init__(message)
-
-
-def personal_new_account(p):
-    password = p
-    if p.__class__.__name__ != 'str':
-        if p.__class__.__name__ != 'list':
-            e = JSONRPCInvalidParams()
-            e.data = 'parameter must be list containing one string'
-            raise ValueError(e)
-        logg.error('foo {}'.format(p))
-        if len(p) != 1:
-            e = JSONRPCInvalidParams()
-            e.data = 'parameter must be list containing one string'
-            raise ValueError(e)
-        if p[0].__class__.__name__ != 'str':
-            e = JSONRPCInvalidParams()
-            e.data = 'parameter must be list containing one string'
-            raise ValueError(e)
-        password = p[0]
-
-    r = db.new(password)
-             
-    return add_0x(r)
-
-
-# TODO: move to translation module ("personal" rpc namespace is node-specific)
-def personal_signTransaction(p):
-    logg.debug('got {} to sign'.format(p[0]))
-    #t = EIP155Transaction(p[0], p[0]['nonce'], 8995)
-    t = EIP155Transaction(p[0], p[0]['nonce'], p[0]['chainId'])
- #   z = signer.sign_transaction(t, p[1])
- #   raw_signed_tx = t.rlp_serialize()
-    raw_signed_tx = signer.sign_transaction_to_rlp(t, p[1])
-    o = {
-        'raw': '0x' + raw_signed_tx.hex(),
-        'tx': t.serialize(),
-        }
-    logg.debug('signed {}'.format(o))
-    return o
-
-
-def eth_signTransaction(tx):
-    o = personal_signTransaction([tx[0], ''])
-    return o['raw']
-
-
-def eth_sign(p):
-    logg.debug('got message {} to sign'.format(p[1]))
-    message_type = type(p[1]).__name__
-    if message_type != 'str':
-        raise ValueError('invalid message format, must be {}, not {}'.format(message_type))
-    z = signer.sign_ethereum_message(p[0], p[1][2:])
-    return str(z)
-
-
-methods = {
-        'personal_newAccount': personal_new_account,
-        'personal_signTransaction': personal_signTransaction,
-        'eth_signTransaction': eth_signTransaction,
-        'eth_sign': eth_sign,
-    }
-
-
-def jsonrpc_error(rpc_id, err):
-    return {
-            'jsonrpc': '2.0',
-            'id': rpc_id,
-            'error': {
-                'code': err.CODE,
-                'message': err.MESSAGE,
-                },
-            }
-
-
-def jsonrpc_ok(rpc_id, response):
-    return {
-            'jsonrpc': '2.0',
-            'id': rpc_id,
-            'result': response,
-            }
-
-
-def is_valid_json(j):
-    if j.get('id') == 'None':
-        raise ValueError('id missing')
-    return True
-
-
-def process_input(j):
-    rpc_id = j['id']
-    m = j['method']
-    p = j['params']
-    return (rpc_id, methods[m](p))
-
-
-def start_server_tcp(spec):
-    s = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
-    s.bind(spec)
-    logg.debug('created tcp socket {}'.format(spec))
-    start_server(s)
-
-
-def start_server_unix(socket_path):
-    socket_dir = os.path.dirname(socket_path)
-    try:
-        fi = os.stat(socket_dir)
-        if not stat.S_ISDIR:
-            RuntimeError('socket path {} is not a directory'.format(socket_dir))
-    except FileNotFoundError:
-        os.mkdir(socket_dir)
-
-    try:
-        os.unlink(socket_path)
-    except FileNotFoundError:
     pass
-    s = socket.socket(family = socket.AF_UNIX, type = socket.SOCK_STREAM)
-    s.bind(socket_path)
-    logg.debug('created unix ipc socket {}'.format(socket_path))
-    start_server(s)
 
 
-def start_server(s):
+def main():
-    s.listen(10)
-    logg.debug('server started')
-    while True:
-        (csock, caddr) = s.accept()
-        d = csock.recv(4096)
-        j = None
-        try:
-            j = json.loads(d)
-            is_valid_json(j)
-            logg.debug('{}'.format(d.decode('utf-8')))
-        except Exception as e:
-            logg.exception('input error {}'.format(e))
-            csock.send(json.dumps(jsonrpc_error(None, JSONRPCParseError)).encode('utf-8'))
-            csock.close()
-            continue
 
-        try:
-            (rpc_id, r) = process_input(j)
-            r = jsonrpc_ok(rpc_id, r)
-            j = json.dumps(r).encode('utf-8')
-            csock.send(j)
-        except ValueError as e:
-            # TODO: handle cases to give better error context to caller
-            logg.exception('process error {}'.format(e))
-            csock.send(json.dumps(jsonrpc_error(j['id'], JSONRPCServerError)).encode('utf-8'))
-        except UnknownAccountError as e:
-            logg.exception('process unknown account error {}'.format(e))
-            csock.send(json.dumps(jsonrpc_error(j['id'], JSONRPCServerError)).encode('utf-8'))
-
-        csock.close()
-    s.close()
-
-    os.unlink(socket_path)
-
-
-def init():
-    global db, signer
     secret_hex = config.get('SIGNER_SECRET')
     if secret_hex == None:
         raise MissingSecretError('please provide a valid hex value for the SIGNER_SECRET configuration variable')
@@ -245,26 +85,31 @@ def init():
     kw = {
             'symmetric_key': secret,
             }
-    db = ReferenceKeystore(dsn, **kw)
+    SignRequestHandler.keystore = ReferenceKeystore(dsn, **kw)
-    signer = ReferenceSigner(db)
+    SignRequestHandler.signer = ReferenceSigner(SignRequestHandler.keystore)
 
-
-def main():
-    init()
     arg = None
     try:
         arg = json.loads(sys.argv[1])
     except:
-        logg.info('no json rpc command detected, starting socket server')
+        logg.info('no json rpc command detected, starting socket server {}'.format(socket_url))
         scheme = 'ipc'
         if socket_url.scheme != '':
             scheme = socket_url.scheme
-        if re.match(re_http, socket_url.scheme):
+        if re.match(re_tcp, socket_url.scheme):
+            from crypto_dev_signer.cli.socket import start_server_tcp
             socket_spec = socket_url.netloc.split(':')
             host = socket_spec[0]
             port = int(socket_spec[1])
             start_server_tcp((host, port))
+        elif re.match(re_http, socket_url.scheme):
+            from crypto_dev_signer.cli.http import start_server_http
+            socket_spec = socket_url.netloc.split(':')
+            host = socket_spec[0]
+            port = int(socket_spec[1])
+            start_server_http((host, port))
         else:
+            from crypto_dev_signer.cli.socket import start_server_unix
             start_server_unix(socket_url.path)
         sys.exit(0)
    

run_tests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+set -x
+#export PYTHONPATH=${PYTHONPATH:.}
+for f in `ls tests/*.py`; do
+	python $f
+done
+set +x
+set +e

setup.py

@@ -33,18 +33,16 @@ f.close()
 
 setup(
         name="crypto-dev-signer",
-        version="0.4.15a1",
+        version="0.4.15a4",
         description="A signer and keystore daemon and library for cryptocurrency software development",
         author="Louis Holbrook",
         author_email="dev@holbrook.no",
         packages=[
             'crypto_dev_signer.eth.signer',
-            #'crypto_dev_signer.eth.web3ext',
-            #'crypto_dev_signer.eth.helper',
             'crypto_dev_signer.eth',
+            'crypto_dev_signer.cli',
             'crypto_dev_signer.keystore',
             'crypto_dev_signer.runnable',
-            #'crypto_dev_signer.helper',
             'crypto_dev_signer',
             ],
         install_requires=requirements,
@@ -54,9 +52,6 @@ setup(
         tests_require=test_requirements,
         long_description=long_description,
         long_description_content_type='text/markdown',
-        #scripts = [
-        #    'scripts/crypto-dev-daemon',
-        #    ],
         entry_points = {
             'console_scripts': [
                 'crypto-dev-daemon=crypto_dev_signer.runnable.signer:main',

tests/test_cli.py

@@ -0,0 +1,88 @@
+# standard imports
+import unittest
+import logging
+import os
+
+# external imports
+from hexathon import strip_0x
+
+# local imports
+from crypto_dev_signer.eth.signer import ReferenceSigner
+from crypto_dev_signer.keystore.dict import DictKeystore
+from crypto_dev_signer.cli.handle import SignRequestHandler
+from crypto_dev_signer.eth.transaction import EIP155Transaction
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+data_dir = os.path.join(script_dir, 'testdata')
+
+class TestCli(unittest.TestCase):
+
+    def setUp(self):
+        #pk = bytes.fromhex('5087503f0a9cc35b38665955eb830c63f778453dd11b8fa5bd04bc41fd2cc6d6')
+        #pk_getter = pkGetter(pk)
+        self.keystore = DictKeystore()
+        SignRequestHandler.keystore = self.keystore
+        self.signer = ReferenceSigner(self.keystore)
+        SignRequestHandler.signer = self.signer
+        self.handler = SignRequestHandler()
+
+
+    def test_new_account(self):
+        q = {
+                'id': 0,
+                'method': 'personal_newAccount',
+                'params': [''],
+                }
+        (rpc_id, result) = self.handler.process_input(q)
+        self.assertTrue(self.keystore.get(result))
+
+
+    def test_sign_tx(self):
+        keystore_file = os.path.join(data_dir, 'UTC--2021-01-08T18-37-01.187235289Z--00a329c0648769a73afac7f9381e08fb43dbea72')
+        sender = self.keystore.import_keystore_file(keystore_file)
+        tx_hexs = {
+            'nonce': '0x',
+            'from': sender,
+            'gasPrice': "0x04a817c800",
+            'gas': "0x5208",
+            'to': '0x3535353535353535353535353535353535353535',
+            'value': "0x03e8",
+            'data': "0xdeadbeef",
+            'chainId': 8995,
+        }
+        tx = EIP155Transaction(tx_hexs, 42, 8995)
+        tx_s = tx.serialize()
+
+        # TODO: move to serialization wrapper for tests
+        tx_s['chainId'] = tx_s['v']
+        tx_s['from'] = sender
+
+        # eth_signTransaction wraps personal_signTransaction, so here we test both already
+        q = {
+                'id': 0,
+                'method': 'eth_signTransaction',
+                'params': [tx_s],
+                }
+        (rpc_id, result) = self.handler.process_input(q)
+        logg.debug('result {}'.format(result))
+
+        self.assertEqual(strip_0x(result), 'f86c2a8504a817c8008252089435353535353535353535353535353535353535358203e884deadbeef82466aa0b7c1bbf52f736ada30fe253c7484176f44d6fd097a9720dc85ae5bbc7f060e54a07afee2563b0cf6d00333df51cc62b0d13c63108b2bce54ce2ad24e26ce7b4f25')
+
+    def test_sign_msg(self):
+        keystore_file = os.path.join(data_dir, 'UTC--2021-01-08T18-37-01.187235289Z--00a329c0648769a73afac7f9381e08fb43dbea72')
+        sender = self.keystore.import_keystore_file(keystore_file)
+        q = {
+                'id': 0,
+                'method': 'eth_sign',
+                'params': [sender, '0xdeadbeef'],
+                }
+        (rpc_id, result) = self.handler.process_input(q)
+        logg.debug('result msg {}'.format(result))
+        self.assertEqual(strip_0x(result), '50320dda75190a121b7b5979de66edadafd02bdfbe4f6d49552e79c01410d2464aae35e385c0e5b61663ff7b44ef65fa0ac7ad8a57472cf405db399b9dba3e1600')
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_keystore_dict.py

@@ -29,7 +29,6 @@ class TestDict(unittest.TestCase):
     db = None
 
     def setUp(self):
-        logg.debug('setup')
         self.db = DictKeystore()
 
         keystore_filepath = os.path.join(script_dir, 'testdata', 'UTC--2021-01-08T18-37-01.187235289Z--00a329c0648769a73afac7f9381e08fb43dbea72')

tests/test_socket.py

@@ -0,0 +1,15 @@
+# standard imports
+import unittest
+import logging
+
+logg = logging.getLogger(__name__)
+
+
+class SocketTest(unittest.TestCase):
+
+    def test_placeholder_warning(self):
+        logg.warning('socket tests are missing! :/')
+
+
+if __name__ == '__main__':
+    unittest.main()