Merge branch 'master' into issues/4673

This commit is contained in:
Joseph Mark
2017-07-12 16:06:23 +07:00
77 changed files with 1670 additions and 309 deletions

View File

@@ -78,6 +78,7 @@ disable_periodic = true
jit = false
[misc]
ntp_server = "pool.ntp.org:123"
logging = "own_tx=trace"
log_file = "/var/log/parity.log"
color = true

View File

@@ -183,8 +183,10 @@ usage! {
or |c: &Config| otry!(c.rpc).apis.as_ref().map(|vec| vec.join(",")),
flag_jsonrpc_hosts: String = "none",
or |c: &Config| otry!(c.rpc).hosts.as_ref().map(|vec| vec.join(",")),
flag_jsonrpc_threads: Option<usize> = None,
or |c: &Config| otry!(c.rpc).threads.map(Some),
flag_jsonrpc_server_threads: Option<usize> = None,
or |c: &Config| otry!(c.rpc).server_threads.map(Some),
flag_jsonrpc_threads: usize = 0usize,
or |c: &Config| otry!(c.rpc).processing_threads,
// WS
flag_no_ws: bool = false,
@@ -355,6 +357,8 @@ usage! {
or |c: &Config| otry!(c.vm).jit.clone(),
// -- Miscellaneous Options
flag_ntp_server: String = "pool.ntp.org:123",
or |c: &Config| otry!(c.misc).ntp_server.clone(),
flag_logging: Option<String> = None,
or |c: &Config| otry!(c.misc).logging.clone().map(Some),
flag_log_file: Option<String> = None,
@@ -471,7 +475,8 @@ struct Rpc {
cors: Option<String>,
apis: Option<Vec<String>>,
hosts: Option<Vec<String>>,
threads: Option<usize>,
server_threads: Option<usize>,
processing_threads: Option<usize>,
}
#[derive(Default, Debug, PartialEq, Deserialize)]
@@ -590,6 +595,7 @@ struct VM {
#[derive(Default, Debug, PartialEq, Deserialize)]
struct Misc {
ntp_server: Option<String>,
logging: Option<String>,
log_file: Option<String>,
color: Option<bool>,
@@ -752,7 +758,8 @@ mod tests {
flag_jsonrpc_cors: Some("null".into()),
flag_jsonrpc_apis: "web3,eth,net,parity,traces,rpc,secretstore".into(),
flag_jsonrpc_hosts: "none".into(),
flag_jsonrpc_threads: None,
flag_jsonrpc_server_threads: None,
flag_jsonrpc_threads: 0,
// WS
flag_no_ws: false,
@@ -889,6 +896,7 @@ mod tests {
flag_dapps_apis_all: None,
// -- Miscellaneous Options
flag_ntp_server: "pool.ntp.org:123".into(),
flag_version: false,
flag_logging: Some("own_tx=trace".into()),
flag_log_file: Some("/var/log/parity.log".into()),
@@ -980,7 +988,8 @@ mod tests {
cors: None,
apis: None,
hosts: None,
threads: None,
server_threads: None,
processing_threads: None,
}),
ipc: Some(Ipc {
disable: None,
@@ -1064,6 +1073,7 @@ mod tests {
jit: Some(false),
}),
misc: Some(Misc {
ntp_server: Some("pool.ntp.org:123".into()),
logging: Some("own_tx=trace".into()),
log_file: Some("/var/log/parity.log".into()),
color: Some(true),

View File

@@ -177,9 +177,12 @@ API and Console Options:
is additional security against some attack
vectors. Special options: "all", "none",
(default: {flag_jsonrpc_hosts}).
--jsonrpc-threads THREADS Enables experimental faster implementation of JSON-RPC server.
--jsonrpc-server-threads NUM Enables experimental faster implementation of JSON-RPC server.
Requires Dapps server to be disabled
using --no-dapps. (default: {flag_jsonrpc_threads:?})
using --no-dapps. (default: {flag_jsonrpc_server_threads:?})
--jsonrpc-threads THREADS Turn on additional processing threads in all RPC servers.
Setting this to non-zero value allows parallel cpu-heavy queries
execution. (default: {flag_jsonrpc_threads})
--no-ws Disable the WebSockets server. (default: {flag_no_ws})
--ws-port PORT Specify the port portion of the WebSockets server
@@ -465,6 +468,8 @@ Internal Options:
--can-restart Executable will auto-restart if exiting with 69.
Miscellaneous Options:
--ntp-server HOST NTP server to provide current time (host:port). Used to verify node health.
(default: {flag_ntp_server})
-l --logging LOGGING Specify the logging level. Must conform to the same
format as RUST_LOG. (default: {flag_logging:?})
--log-file FILENAME Specify a filename into which logging should be

View File

@@ -137,7 +137,7 @@ impl Configuration {
let secretstore_conf = self.secretstore_config()?;
let format = self.format()?;
if self.args.flag_jsonrpc_threads.is_some() && dapps_conf.enabled {
if self.args.flag_jsonrpc_server_threads.is_some() && dapps_conf.enabled {
dapps_conf.enabled = false;
writeln!(&mut stderr(), "Warning: Disabling Dapps server because fast RPC server was enabled.").expect("Error writing to stderr.")
}
@@ -556,6 +556,7 @@ impl Configuration {
fn ui_config(&self) -> UiConfiguration {
UiConfiguration {
enabled: self.ui_enabled(),
ntp_server: self.args.flag_ntp_server.clone(),
interface: self.ui_interface(),
port: self.args.flag_ports_shift + self.args.flag_ui_port,
hosts: self.ui_hosts(),
@@ -565,12 +566,18 @@ impl Configuration {
fn dapps_config(&self) -> DappsConfiguration {
DappsConfiguration {
enabled: self.dapps_enabled(),
ntp_server: self.args.flag_ntp_server.clone(),
dapps_path: PathBuf::from(self.directories().dapps),
extra_dapps: if self.args.cmd_dapp {
self.args.arg_path.iter().map(|path| PathBuf::from(path)).collect()
} else {
vec![]
},
extra_embed_on: if self.args.flag_ui_no_validation {
vec![("localhost".to_owned(), 3000)]
} else {
vec![]
},
}
}
@@ -776,15 +783,11 @@ impl Configuration {
}
fn ws_hosts(&self) -> Option<Vec<String>> {
if self.args.flag_ui_no_validation {
return None;
}
self.hosts(&self.args.flag_ws_hosts, &self.ws_interface())
}
fn ws_origins(&self) -> Option<Vec<String>> {
if self.args.flag_unsafe_expose {
if self.args.flag_unsafe_expose || self.args.flag_ui_no_validation {
return None;
}
@@ -825,11 +828,12 @@ impl Configuration {
},
hosts: self.rpc_hosts(),
cors: self.rpc_cors(),
threads: match self.args.flag_jsonrpc_threads {
server_threads: match self.args.flag_jsonrpc_server_threads {
Some(threads) if threads > 0 => Some(threads),
None => None,
_ => return Err("--jsonrpc-threads number needs to be positive.".into()),
}
_ => return Err("--jsonrpc-server-threads number needs to be positive.".into()),
},
processing_threads: self.args.flag_jsonrpc_threads,
};
Ok(conf)
@@ -1264,6 +1268,7 @@ mod tests {
support_token_api: true
}, UiConfiguration {
enabled: true,
ntp_server: "pool.ntp.org:123".into(),
interface: "127.0.0.1".into(),
port: 8180,
hosts: Some(vec![]),
@@ -1504,6 +1509,7 @@ mod tests {
assert_eq!(conf0.directories().signer, "signer".to_owned());
assert_eq!(conf0.ui_config(), UiConfiguration {
enabled: true,
ntp_server: "pool.ntp.org:123".into(),
interface: "127.0.0.1".into(),
port: 8180,
hosts: Some(vec![]),
@@ -1512,14 +1518,17 @@ mod tests {
assert_eq!(conf1.directories().signer, "signer".to_owned());
assert_eq!(conf1.ui_config(), UiConfiguration {
enabled: true,
ntp_server: "pool.ntp.org:123".into(),
interface: "127.0.0.1".into(),
port: 8180,
hosts: Some(vec![]),
});
assert_eq!(conf1.ws_config().unwrap().hosts, None);
assert_eq!(conf1.dapps_config().extra_embed_on, vec![("localhost".to_owned(), 3000)]);
assert_eq!(conf1.ws_config().unwrap().origins, None);
assert_eq!(conf2.directories().signer, "signer".to_owned());
assert_eq!(conf2.ui_config(), UiConfiguration {
enabled: true,
ntp_server: "pool.ntp.org:123".into(),
interface: "127.0.0.1".into(),
port: 3123,
hosts: Some(vec![]),
@@ -1528,6 +1537,7 @@ mod tests {
assert_eq!(conf3.directories().signer, "signer".to_owned());
assert_eq!(conf3.ui_config(), UiConfiguration {
enabled: true,
ntp_server: "pool.ntp.org:123".into(),
interface: "test".into(),
port: 8180,
hosts: Some(vec![]),

View File

@@ -22,6 +22,7 @@ use ethcore::client::{Client, BlockChainClient, BlockId};
use ethcore::transaction::{Transaction, Action};
use ethsync::LightSync;
use futures::{future, IntoFuture, Future, BoxFuture};
use futures_cpupool::CpuPool;
use hash_fetch::fetch::Client as FetchClient;
use hash_fetch::urlhint::ContractClient;
use helpers::replace_home;
@@ -35,8 +36,10 @@ use util::{Bytes, Address};
#[derive(Debug, PartialEq, Clone)]
pub struct Configuration {
pub enabled: bool,
pub ntp_server: String,
pub dapps_path: PathBuf,
pub extra_dapps: Vec<PathBuf>,
pub extra_embed_on: Vec<(String, u16)>,
}
impl Default for Configuration {
@@ -44,8 +47,10 @@ impl Default for Configuration {
let data_dir = default_data_path();
Configuration {
enabled: true,
ntp_server: "pool.ntp.org:123".into(),
dapps_path: replace_home(&data_dir, "$BASE/dapps").into(),
extra_dapps: vec![],
extra_embed_on: vec![],
}
}
}
@@ -140,6 +145,7 @@ pub struct Dependencies {
pub sync_status: Arc<SyncStatus>,
pub contract_client: Arc<ContractClient>,
pub remote: parity_reactor::TokioRemote,
pub pool: CpuPool,
pub fetch: FetchClient,
pub signer: Arc<SignerService>,
pub ui_address: Option<(String, u16)>,
@@ -152,20 +158,23 @@ pub fn new(configuration: Configuration, deps: Dependencies) -> Result<Option<Mi
server::dapps_middleware(
deps,
&configuration.ntp_server,
configuration.dapps_path,
configuration.extra_dapps,
rpc::DAPPS_DOMAIN.into(),
rpc::DAPPS_DOMAIN,
configuration.extra_embed_on,
).map(Some)
}
pub fn new_ui(enabled: bool, deps: Dependencies) -> Result<Option<Middleware>, String> {
pub fn new_ui(enabled: bool, ntp_server: &str, deps: Dependencies) -> Result<Option<Middleware>, String> {
if !enabled {
return Ok(None);
}
server::ui_middleware(
deps,
rpc::DAPPS_DOMAIN.into(),
ntp_server,
rpc::DAPPS_DOMAIN,
).map(Some)
}
@@ -192,16 +201,19 @@ mod server {
pub fn dapps_middleware(
_deps: Dependencies,
_ntp_server: &str,
_dapps_path: PathBuf,
_extra_dapps: Vec<PathBuf>,
_dapps_domain: String,
_dapps_domain: &str,
_extra_embed_on: Vec<(String, u16)>,
) -> Result<Middleware, String> {
Err("Your Parity version has been compiled without WebApps support.".into())
}
pub fn ui_middleware(
_deps: Dependencies,
_dapps_domain: String,
_ntp_server: &str,
_dapps_domain: &str,
) -> Result<Middleware, String> {
Err("Your Parity version has been compiled without UI support.".into())
}
@@ -226,17 +238,22 @@ mod server {
pub fn dapps_middleware(
deps: Dependencies,
ntp_server: &str,
dapps_path: PathBuf,
extra_dapps: Vec<PathBuf>,
dapps_domain: String,
dapps_domain: &str,
extra_embed_on: Vec<(String, u16)>,
) -> Result<Middleware, String> {
let signer = deps.signer;
let parity_remote = parity_reactor::Remote::new(deps.remote.clone());
let web_proxy_tokens = Arc::new(move |token| signer.web_proxy_access_token_domain(&token));
Ok(parity_dapps::Middleware::dapps(
ntp_server,
deps.pool,
parity_remote,
deps.ui_address,
extra_embed_on,
dapps_path,
extra_dapps,
dapps_domain,
@@ -249,15 +266,18 @@ mod server {
pub fn ui_middleware(
deps: Dependencies,
dapps_domain: String,
ntp_server: &str,
dapps_domain: &str,
) -> Result<Middleware, String> {
let parity_remote = parity_reactor::Remote::new(deps.remote.clone());
Ok(parity_dapps::Middleware::ui(
ntp_server,
deps.pool,
parity_remote,
dapps_domain,
deps.contract_client,
deps.sync_status,
deps.fetch,
dapps_domain,
))
}

View File

@@ -29,6 +29,7 @@ extern crate docopt;
extern crate env_logger;
extern crate fdlimit;
extern crate futures;
extern crate futures_cpupool;
extern crate isatty;
extern crate jsonrpc_core;
extern crate num_cpus;

View File

@@ -30,6 +30,7 @@ use rpc_apis::{self, ApiSet};
pub use parity_rpc::{IpcServer, HttpServer, RequestMiddleware};
pub use parity_rpc::ws::Server as WsServer;
pub use parity_rpc::informant::CpuPool;
pub const DAPPS_DOMAIN: &'static str = "web3.site";
@@ -42,7 +43,8 @@ pub struct HttpConfiguration {
pub apis: ApiSet,
pub cors: Option<Vec<String>>,
pub hosts: Option<Vec<String>>,
pub threads: Option<usize>,
pub server_threads: Option<usize>,
pub processing_threads: usize,
}
impl HttpConfiguration {
@@ -63,7 +65,8 @@ impl Default for HttpConfiguration {
apis: ApiSet::UnsafeContext,
cors: None,
hosts: Some(Vec::new()),
threads: None,
server_threads: None,
processing_threads: 0,
}
}
}
@@ -71,6 +74,7 @@ impl Default for HttpConfiguration {
#[derive(Debug, PartialEq, Clone)]
pub struct UiConfiguration {
pub enabled: bool,
pub ntp_server: String,
pub interface: String,
pub port: u16,
pub hosts: Option<Vec<String>>,
@@ -94,7 +98,8 @@ impl From<UiConfiguration> for HttpConfiguration {
apis: rpc_apis::ApiSet::SafeContext,
cors: None,
hosts: conf.hosts,
threads: None,
server_threads: None,
processing_threads: 0,
}
}
}
@@ -103,6 +108,7 @@ impl Default for UiConfiguration {
fn default() -> Self {
UiConfiguration {
enabled: true && cfg!(feature = "ui-enabled"),
ntp_server: "pool.ntp.org:123".into(),
port: 8180,
interface: "127.0.0.1".into(),
hosts: Some(vec![]),
@@ -176,6 +182,7 @@ pub struct Dependencies<D: rpc_apis::Dependencies> {
pub apis: Arc<D>,
pub remote: TokioRemote,
pub stats: Arc<RpcStats>,
pub pool: Option<CpuPool>,
}
pub fn new_ws<D: rpc_apis::Dependencies>(
@@ -192,11 +199,12 @@ pub fn new_ws<D: rpc_apis::Dependencies>(
let addr = url.parse().map_err(|_| format!("Invalid WebSockets listen host/port given: {}", url))?;
let full_handler = setup_apis(rpc_apis::ApiSet::SafeContext, deps);
let pool = deps.pool.clone();
let full_handler = setup_apis(rpc_apis::ApiSet::SafeContext, deps, pool.clone());
let handler = {
let mut handler = MetaIoHandler::with_middleware((
rpc::WsDispatcher::new(full_handler),
Middleware::new(deps.stats.clone(), deps.apis.activity_notifier())
Middleware::new(deps.stats.clone(), deps.apis.activity_notifier(), pool)
));
let apis = conf.apis.list_apis();
deps.apis.extend_with_set(&mut handler, &apis);
@@ -252,7 +260,8 @@ pub fn new_http<D: rpc_apis::Dependencies>(
let http_address = (conf.interface, conf.port);
let url = format!("{}:{}", http_address.0, http_address.1);
let addr = url.parse().map_err(|_| format!("Invalid {} listen host/port given: {}", id, url))?;
let handler = setup_apis(conf.apis, deps);
let pool = deps.pool.clone();
let handler = setup_apis(conf.apis, deps, pool);
let remote = deps.remote.clone();
let cors_domains = into_domains(conf.cors);
@@ -265,7 +274,7 @@ pub fn new_http<D: rpc_apis::Dependencies>(
handler,
remote,
rpc::RpcExtractor,
match (conf.threads, middleware) {
match (conf.server_threads, middleware) {
(Some(threads), None) => rpc::HttpSettings::Threads(threads),
(None, middleware) => rpc::HttpSettings::Dapps(middleware),
(Some(_), Some(_)) => {
@@ -291,7 +300,8 @@ pub fn new_ipc<D: rpc_apis::Dependencies>(
return Ok(None);
}
let handler = setup_apis(conf.apis, dependencies);
let pool = dependencies.pool.clone();
let handler = setup_apis(conf.apis, dependencies, pool);
let remote = dependencies.remote.clone();
match rpc::start_ipc(&conf.socket_addr, handler, remote, rpc::RpcExtractor) {
Ok(server) => Ok(Some(server)),
@@ -318,11 +328,11 @@ fn with_domain(items: Option<Vec<String>>, domain: &str, addresses: &[Option<(St
})
}
fn setup_apis<D>(apis: ApiSet, deps: &Dependencies<D>) -> MetaIoHandler<Metadata, Middleware<D::Notifier>>
fn setup_apis<D>(apis: ApiSet, deps: &Dependencies<D>, pool: Option<CpuPool>) -> MetaIoHandler<Metadata, Middleware<D::Notifier>>
where D: rpc_apis::Dependencies
{
let mut handler = MetaIoHandler::with_middleware(
Middleware::new(deps.stats.clone(), deps.apis.activity_notifier())
Middleware::new(deps.stats.clone(), deps.apis.activity_notifier(), pool)
);
let apis = apis.list_apis();
deps.apis.extend_with_set(&mut handler, &apis);

View File

@@ -27,8 +27,7 @@ use ethcore::miner::{StratumOptions, Stratum};
use ethcore::service::ClientService;
use ethcore::snapshot;
use ethcore::verification::queue::VerifierSettings;
use ethsync::NetworkConfiguration;
use ethsync::SyncConfig;
use ethsync::{self, SyncConfig};
use fdlimit::raise_fd_limit;
use hash_fetch::fetch::{Fetch, Client as FetchClient};
use informant::{Informant, LightNodeInformantData, FullNodeInformantData};
@@ -84,7 +83,7 @@ pub struct RunCmd {
pub ws_conf: rpc::WsConfiguration,
pub http_conf: rpc::HttpConfiguration,
pub ipc_conf: rpc::IpcConfiguration,
pub net_conf: NetworkConfiguration,
pub net_conf: ethsync::NetworkConfiguration,
pub network_id: Option<u64>,
pub warp_sync: bool,
pub public_node: bool,
@@ -237,7 +236,7 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) ->
network_config: net_conf.into_basic().map_err(|e| format!("Failed to produce network config: {}", e))?,
client: Arc::new(provider),
network_id: cmd.network_id.unwrap_or(spec.network_id()),
subprotocol_name: ::ethsync::LIGHT_PROTOCOL,
subprotocol_name: ethsync::LIGHT_PROTOCOL,
handlers: vec![on_demand.clone()],
};
let light_sync = LightSync::new(sync_params).map_err(|e| format!("Error starting network: {}", e))?;
@@ -277,9 +276,18 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) ->
on_demand: on_demand.clone(),
});
let sync = light_sync.clone();
struct LightSyncStatus(Arc<LightSync>);
impl dapps::SyncStatus for LightSyncStatus {
fn is_major_importing(&self) -> bool { self.0.is_major_importing() }
fn peers(&self) -> (usize, usize) {
let peers = ethsync::LightSyncProvider::peer_numbers(&*self.0);
(peers.connected, peers.max)
}
}
dapps::Dependencies {
sync_status: Arc::new(move || sync.is_major_importing()),
sync_status: Arc::new(LightSyncStatus(light_sync.clone())),
pool: fetch.pool(),
contract_client: contract_client,
remote: event_loop.raw_remote(),
fetch: fetch.clone(),
@@ -289,7 +297,7 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) ->
};
let dapps_middleware = dapps::new(cmd.dapps_conf.clone(), dapps_deps.clone())?;
let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, dapps_deps)?;
let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, &cmd.ui_conf.ntp_server, dapps_deps)?;
// start RPCs
let dapps_service = dapps::service(&dapps_middleware);
@@ -316,6 +324,11 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) ->
apis: deps_for_rpc_apis.clone(),
remote: event_loop.raw_remote(),
stats: rpc_stats.clone(),
pool: if cmd.http_conf.processing_threads > 0 {
Some(rpc::CpuPool::new(cmd.http_conf.processing_threads))
} else {
None
},
};
// start rpc servers
@@ -581,7 +594,7 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
let (sync_provider, manage_network, chain_notify) = modules::sync(
&mut hypervisor,
sync_config,
net_conf.into(),
net_conf.clone().into(),
client.clone(),
snapshot_service.clone(),
client.clone(),
@@ -625,8 +638,20 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
let (sync, client) = (sync_provider.clone(), client.clone());
let contract_client = Arc::new(::dapps::FullRegistrar { client: client.clone() });
struct SyncStatus(Arc<ethsync::SyncProvider>, Arc<Client>, ethsync::NetworkConfiguration);
impl dapps::SyncStatus for SyncStatus {
fn is_major_importing(&self) -> bool {
is_major_importing(Some(self.0.status().state), self.1.queue_info())
}
fn peers(&self) -> (usize, usize) {
let status = self.0.status();
(status.num_peers, status.current_max_peers(self.2.min_peers, self.2.max_peers) as usize)
}
}
dapps::Dependencies {
sync_status: Arc::new(move || is_major_importing(Some(sync.status().state), client.queue_info())),
sync_status: Arc::new(SyncStatus(sync, client, net_conf)),
pool: fetch.pool(),
contract_client: contract_client,
remote: event_loop.raw_remote(),
fetch: fetch.clone(),
@@ -635,7 +660,7 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
}
};
let dapps_middleware = dapps::new(cmd.dapps_conf.clone(), dapps_deps.clone())?;
let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, dapps_deps)?;
let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, &cmd.ui_conf.ntp_server, dapps_deps)?;
let dapps_service = dapps::service(&dapps_middleware);
let deps_for_rpc_apis = Arc::new(rpc_apis::FullDependencies {
@@ -663,6 +688,12 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
apis: deps_for_rpc_apis.clone(),
remote: event_loop.raw_remote(),
stats: rpc_stats.clone(),
pool: if cmd.http_conf.processing_threads > 0 {
Some(rpc::CpuPool::new(cmd.http_conf.processing_threads))
} else {
None
},
};
// start rpc servers