diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b0540e8..98d668c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -61,6 +61,9 @@ jobs:
# musl for static binaries
rustup target add x86_64-unknown-linux-musl
+ - name: Run tests
+ run: make test
+
- name: Build
run: |
set -x
@@ -101,6 +104,8 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
+ - name: Run tests
+ run: make test
- name: Build
run: |
cargo build --no-default-features
diff --git a/Cargo.lock b/Cargo.lock
index 97d18a3..75ea935 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -279,6 +279,7 @@ dependencies = [
"futures-util",
"humantime",
"log",
+ "pretty_assertions",
"quick-xml",
"ratatui",
"semver",
@@ -385,7 +386,7 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "clickhouse-rs"
version = "1.1.0-alpha.1"
-source = "git+https://github.com/azat-rust/clickhouse-rs?branch=mTLS#f88e93e0d0b16591298bffd623510290b1c0782b"
+source = "git+https://github.com/azat-rust/clickhouse-rs?branch=next#20cc32180564a8c7aafd2843504fa0235229765d"
dependencies = [
"byteorder",
"cfg-if",
@@ -417,7 +418,7 @@ dependencies = [
[[package]]
name = "clickhouse-rs-cityhash-sys"
version = "0.1.2"
-source = "git+https://github.com/azat-rust/clickhouse-rs?branch=mTLS#f88e93e0d0b16591298bffd623510290b1c0782b"
+source = "git+https://github.com/azat-rust/clickhouse-rs?branch=next#20cc32180564a8c7aafd2843504fa0235229765d"
dependencies = [
"cc",
]
@@ -747,6 +748,12 @@ dependencies = [
"syn",
]
+[[package]]
+name = "diff"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
+
[[package]]
name = "digest"
version = "0.10.7"
@@ -1941,6 +1948,16 @@ dependencies = [
"zerocopy 0.8.24",
]
+[[package]]
+name = "pretty_assertions"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d"
+dependencies = [
+ "diff",
+ "yansi",
+]
+
[[package]]
name = "proc-macro2"
version = "1.0.94"
@@ -3277,6 +3294,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a67300977d3dc3f8034dae89778f502b6ba20b269527b3223ba59c0cf393bb8a"
+[[package]]
+name = "yansi"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
+
[[package]]
name = "yoke"
version = "0.7.5"
diff --git a/Cargo.toml b/Cargo.toml
index 2954b9e..df58702 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -65,15 +65,18 @@ syntect = { version = "*", default-features = false, features = ["default-syntax
# Drivers
#
# Patches:
-# - rustls
-# - no panic on broken protocol
-clickhouse-rs = { git = "https://github.com/azat-rust/clickhouse-rs", branch = "mTLS", default-features = false, features = ["tokio_io"] }
+# - https://github.com/suharev7/clickhouse-rs/pull/226 - Properly handle terminated connection by the server and fix CI
+# - https://github.com/suharev7/clickhouse-rs/pull/227 - mTLS
+clickhouse-rs = { git = "https://github.com/azat-rust/clickhouse-rs", branch = "next", default-features = false, features = ["tokio_io"] }
tokio = { version = "*", default-features = false, features = ["macros"] }
# Flamegraphs
flamelens = { version = "0.3.1", default-features = false }
ratatui = { version = "0.26.3", features = ["unstable-rendered-line-info"] }
crossterm = { version = "0.27.0", features = ["use-dev-tty"] }
+[dev-dependencies]
+pretty_assertions = { version= "*", default-features = false, features = ["alloc"] }
+
[profile.release]
# Too slow and does not worth it
lto = false
diff --git a/Documentation/FAQ.md b/Documentation/FAQ.md
index 8307927..ace08df 100644
--- a/Documentation/FAQ.md
+++ b/Documentation/FAQ.md
@@ -1,3 +1,31 @@
+### What is format of the URL accepted by `chdig`?
+
+The simplest form is just - **`localhost`**
+
+For a secure connections with user and password _(note: passing the password on
+the command line is not safe)_, use:
+
+```sh
+chdig -u 'user:password@clickhouse-host.com/?secure=true'
+```
+
+A full list of supported connection options is available [here](https://github.com/azat-rust/clickhouse-rs/?tab=readme-ov-file#dns).
+
+_Note: This link currently points to my fork, as some changes have not yet been accepted upstream._
+
+### Environment variables
+
+A safer way to pass the password is via environment variables:
+
+
+```sh
+export CLICKHOUSE_USER='user'
+export CLICKHOUSE_PASSWORD='password'
+chdig -u 'clickhouse-host.com/?secure=true'
+# or specify the port explicitly
+chdig -u 'clickhouse-host.com:9440/?secure=true'
+```
+
### What is --connection?
`--connection` allows you to use predefined connections, that is supported by
@@ -15,6 +43,9 @@ Here is an example in `XML` format:
secret
+
+
+
@@ -30,6 +61,11 @@ connections_credentials:
hostname: prod
user: default
password: secret
+ # secure: false
+ # skip_verify: false
+ # ca_certificate:
+ # client_certificate:
+ # client_private_key:
```
And later, instead of specifying `--url` (with password in plain-text, which is
@@ -92,27 +128,6 @@ highly not recommended), you can use `chdig --connection prod`.
| | **n**/**N** | Move to next/previous match |
| Extended Navigation | **Home** | reset selection/follow item in table |
-### What is format of the URL accepted by `chdig`?
-
-Example for secure connection with all default connection settings & user name
-& password (passing the password in the command line is unsafe)
-
-```sh
-chdig -u 'user:password@clickhouse-host.com:9440/?secure=true&skip_verify=false&compression=lz4&query_timeout=600s&connection_timeout=5s'
-```
-
-Safer option is to pass the password via the environment variable:
-
-```sh
-export CLICKHOUSE_USER='user'
-export CLICKHOUSE_PASSWORD='password'
-chdig -u 'clickhouse-host.com/?secure=true'
-# or with port
-chdig -u 'clickhouse-host.com:9440/?secure=true'
-```
-
-Or via the configuration file (see above)
-
### Why I see IO wait reported as zero?
- You should ensure that ClickHouse uses one of taskstat gathering methods:
diff --git a/Makefile b/Makefile
index af98f69..4638068 100644
--- a/Makefile
+++ b/Makefile
@@ -101,6 +101,9 @@ run: chdig
build: chdig deploy-binary
+test:
+ cargo test $(cargo_build_opts)
+
build_completion: chdig
cargo run $(cargo_build_opts) -- --completion bash > target/chdig.bash-completion
diff --git a/src/interpreter/options.rs b/src/interpreter/options.rs
index 6a71e40..9f8dc46 100644
--- a/src/interpreter/options.rs
+++ b/src/interpreter/options.rs
@@ -12,9 +12,26 @@ use std::io;
use std::net::{SocketAddr, ToSocketAddrs};
use std::path;
use std::process;
+use std::str::FromStr;
use std::time;
-#[derive(Deserialize)]
+#[derive(Deserialize, Debug, PartialEq)]
+struct ClickHouseClientConfigOpenSSLClient {
+ #[serde(rename = "verificationMode")]
+ verification_mode: Option,
+ #[serde(rename = "certificateFile")]
+ certificate_file: Option,
+ #[serde(rename = "privateKeyFile")]
+ private_key_file: Option,
+ #[serde(rename = "caConfig")]
+ ca_config: Option,
+}
+#[derive(Deserialize, Debug, PartialEq)]
+struct ClickHouseClientConfigOpenSSL {
+ client: Option,
+}
+
+#[derive(Deserialize, Debug, PartialEq)]
struct ClickHouseClientConfigConnectionsCredentials {
name: String,
hostname: Option,
@@ -22,15 +39,19 @@ struct ClickHouseClientConfigConnectionsCredentials {
user: Option,
password: Option,
secure: Option,
- // NOTE: this option is not supported in the clickhouse-client config (yet).
+ // NOTE: the following options are not supported in the clickhouse-client config (yet).
skip_verify: Option,
+ ca_certificate: Option,
+ client_certificate: Option,
+ client_private_key: Option,
}
-#[derive(Deserialize, Default)]
+#[derive(Deserialize, Default, Debug, PartialEq)]
struct ClickHouseClientConfig {
user: Option,
password: Option,
secure: Option,
skip_verify: Option,
+ open_ssl: Option,
connections_credentials: Vec,
}
@@ -44,6 +65,8 @@ struct XmlClickHouseClientConfig {
password: Option,
secure: Option,
skip_verify: Option,
+ #[serde(rename = "openSSL")]
+ open_ssl: Option,
connections_credentials: Option,
}
@@ -53,6 +76,8 @@ struct YamlClickHouseClientConfig {
password: Option,
secure: Option,
skip_verify: Option,
+ #[serde(rename = "openSSL")]
+ open_ssl: Option,
connections_credentials: Option>,
}
@@ -98,7 +123,7 @@ pub struct ChDigOptions {
service: ServiceOptions,
}
-#[derive(Args, Clone)]
+#[derive(Args, Clone, Default)]
pub struct ClickHouseOptions {
#[arg(short('u'), long, value_name = "URL", env = "CHDIG_URL")]
pub url: Option,
@@ -195,6 +220,7 @@ fn read_yaml_clickhouse_client_config(path: &str) -> Result Result bool {
return false;
}
-fn clickhouse_url_defaults(options: &mut ChDigOptions) {
- let mut url = parse_url(&options.clickhouse.url.clone().unwrap_or_default());
- let config: Option = read_clickhouse_client_config();
- let connection = &options.clickhouse.connection;
- let mut has_secure: Option = None;
- let mut has_skip_verify: Option = None;
+fn clickhouse_url_defaults(
+ options: &mut ClickHouseOptions,
+ config: Option,
+) {
+ let mut url = parse_url(&options.url.clone().unwrap_or_default());
+ let connection = &options.connection;
+ let mut secure: Option;
+ let mut skip_verify: Option;
+ let mut ca_certificate: Option;
+ let mut client_certificate: Option;
+ let mut client_private_key: Option;
{
let pairs: HashMap<_, _> = url.query_pairs().into_owned().collect();
- if pairs.contains_key("secure") {
- has_secure = Some(true);
- }
- if pairs.contains_key("skip_verify") {
- has_skip_verify = Some(true)
- }
+ secure = pairs.get("secure").and_then(|v| bool::from_str(v).ok());
+ skip_verify = pairs
+ .get("skip_verify")
+ .and_then(|v| bool::from_str(v).ok());
+ ca_certificate = pairs.get("ca_certificate").cloned();
+ client_certificate = pairs.get("client_certificate").cloned();
+ client_private_key = pairs.get("client_private_key").cloned();
}
// host should be set first, since url crate does not allow to set user/password without host.
@@ -321,23 +354,48 @@ fn clickhouse_url_defaults(options: &mut ChDigOptions) {
//
if let Some(config) = config {
if url.username().is_empty() {
- if let Some(user) = &config.user {
+ if let Some(user) = config.user {
url.set_username(user.as_str()).unwrap();
}
}
if url.password().is_none() {
- if let Some(password) = &config.password {
+ if let Some(password) = config.password {
url.set_password(Some(password.as_str())).unwrap();
}
}
- if has_secure.is_none() {
- if let Some(secure) = &config.secure {
- has_secure = Some(*secure);
+ if secure.is_none() {
+ if let Some(conf_secure) = config.secure {
+ secure = Some(conf_secure);
+ }
+ }
+
+ let ssl_client = config.open_ssl.and_then(|ssl| ssl.client);
+ if skip_verify.is_none() {
+ if let Some(conf_skip_verify) = config.skip_verify.or_else(|| {
+ ssl_client
+ .as_ref()
+ .map(|client| client.verification_mode == Some("none".to_string()))
+ }) {
+ skip_verify = Some(conf_skip_verify);
+ }
+ }
+ if ca_certificate.is_none() {
+ if let Some(conf_ca_certificate) = ssl_client.as_ref().map(|v| v.ca_config.clone()) {
+ ca_certificate = conf_ca_certificate.clone();
}
}
- if has_skip_verify.is_none() {
- if let Some(skip_verify) = &config.skip_verify {
- has_skip_verify = Some(*skip_verify);
+ if client_certificate.is_none() {
+ if let Some(conf_client_certificate) =
+ ssl_client.as_ref().map(|v| v.certificate_file.clone())
+ {
+ client_certificate = conf_client_certificate.clone();
+ }
+ }
+ if client_private_key.is_none() {
+ if let Some(conf_client_private_key) =
+ ssl_client.as_ref().map(|v| v.private_key_file.clone())
+ {
+ client_private_key = conf_client_private_key.clone();
}
}
@@ -375,14 +433,29 @@ fn clickhouse_url_defaults(options: &mut ChDigOptions) {
url.set_password(Some(password.as_str())).unwrap();
}
}
- if has_secure.is_none() {
- if let Some(secure) = &c.secure {
- has_secure = Some(*secure);
+ if secure.is_none() {
+ if let Some(con_secure) = c.secure {
+ secure = Some(con_secure);
+ }
+ }
+ if skip_verify.is_none() {
+ if let Some(con_skip_verify) = c.skip_verify {
+ skip_verify = Some(con_skip_verify);
+ }
+ }
+ if ca_certificate.is_none() {
+ if let Some(con_ca_certificate) = &c.ca_certificate {
+ ca_certificate = Some(con_ca_certificate.clone());
+ }
+ }
+ if client_certificate.is_none() {
+ if let Some(con_client_certificate) = &c.client_certificate {
+ client_certificate = Some(con_client_certificate.clone());
}
}
- if has_skip_verify.is_none() {
- if let Some(skip_verify) = &c.skip_verify {
- has_skip_verify = Some(*skip_verify);
+ if client_private_key.is_none() {
+ if let Some(con_client_private_key) = &c.client_private_key {
+ client_private_key = Some(con_client_private_key.clone());
}
}
}
@@ -398,7 +471,7 @@ fn clickhouse_url_defaults(options: &mut ChDigOptions) {
// - 9000 for non secure
// - 9440 for secure
if url.port().is_none() {
- url.set_port(Some(if has_secure.unwrap_or_default() {
+ url.set_port(Some(if secure.unwrap_or_default() {
9440
} else {
9000
@@ -412,7 +485,7 @@ fn clickhouse_url_defaults(options: &mut ChDigOptions) {
if url_safe.password().is_some() {
url_safe.set_password(None).unwrap();
}
- options.clickhouse.url_safe = url_safe.to_string();
+ options.url_safe = url_safe.to_string();
// some default settings in URL
{
@@ -433,19 +506,29 @@ fn clickhouse_url_defaults(options: &mut ChDigOptions) {
if !pairs.contains_key("query_timeout") {
mut_pairs.append_pair("query_timeout", "600s");
}
- if let Some(secure) = has_secure {
+ if let Some(secure) = secure {
mut_pairs.append_pair("secure", secure.to_string().as_str());
}
- if let Some(skip_verify) = has_skip_verify {
+ if let Some(skip_verify) = skip_verify {
mut_pairs.append_pair("skip_verify", skip_verify.to_string().as_str());
}
+ if let Some(ca_certificate) = ca_certificate {
+ mut_pairs.append_pair("ca_certificate", &ca_certificate);
+ }
+ if let Some(client_certificate) = client_certificate {
+ mut_pairs.append_pair("client_certificate", &client_certificate);
+ }
+ if let Some(client_private_key) = client_private_key {
+ mut_pairs.append_pair("client_private_key", &client_private_key);
+ }
}
- options.clickhouse.url = Some(url.to_string());
+ options.url = Some(url.to_string());
}
fn adjust_defaults(options: &mut ChDigOptions) {
- clickhouse_url_defaults(options);
+ let config: Option = read_clickhouse_client_config();
+ clickhouse_url_defaults(&mut options.clickhouse, config);
// FIXME: overrides_with works before default_value_if, hence --no-group-by never works
if options.view.no_group_by {
@@ -475,3 +558,126 @@ pub fn parse() -> ChDigOptions {
return options;
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use pretty_assertions::assert_eq;
+
+ #[test]
+ fn test_url_parse_no_proto() {
+ assert_eq!(
+ parse_url("localhost"),
+ url::Url::parse("tcp://localhost").unwrap()
+ );
+ }
+
+ #[test]
+ fn test_config_empty() {
+ assert_eq!(
+ read_xml_clickhouse_client_config("tests/configs/empty.xml").is_ok(),
+ true
+ );
+ assert_eq!(
+ read_yaml_clickhouse_client_config("tests/configs/empty.yaml").is_ok(),
+ true
+ );
+ }
+
+ #[test]
+ fn test_config_unknown_directives() {
+ assert_eq!(
+ read_xml_clickhouse_client_config("tests/configs/unknown_directives.xml").is_ok(),
+ true
+ );
+ assert_eq!(
+ read_yaml_clickhouse_client_config("tests/configs/unknown_directives.yaml").is_ok(),
+ true
+ );
+ }
+
+ #[test]
+ fn test_config_basic() {
+ let xml_config = read_xml_clickhouse_client_config("tests/configs/basic.xml").unwrap();
+ let yaml_config = read_yaml_clickhouse_client_config("tests/configs/basic.yaml").unwrap();
+ let config = ClickHouseClientConfig {
+ user: Some("foo".into()),
+ password: Some("bar".into()),
+ ..Default::default()
+ };
+ assert_eq!(config, xml_config);
+ assert_eq!(config, yaml_config);
+ }
+
+ #[test]
+ fn test_config_tls() {
+ let xml_config = read_xml_clickhouse_client_config("tests/configs/tls.xml").unwrap();
+ let yaml_config = read_yaml_clickhouse_client_config("tests/configs/tls.yaml").unwrap();
+ let config = ClickHouseClientConfig {
+ secure: Some(true),
+ open_ssl: Some(ClickHouseClientConfigOpenSSL {
+ client: Some(ClickHouseClientConfigOpenSSLClient {
+ verification_mode: Some("strict".into()),
+ certificate_file: Some("cert".into()),
+ private_key_file: Some("key".into()),
+ ca_config: Some("ca".into()),
+ }),
+ }),
+ ..Default::default()
+ };
+ assert_eq!(config, xml_config);
+ assert_eq!(config, yaml_config);
+ }
+
+ #[test]
+ fn test_config_tls_applying_config_to_connection_url() {
+ let config = read_yaml_clickhouse_client_config("tests/configs/tls.yaml").ok();
+ let mut options = ClickHouseOptions {
+ ..Default::default()
+ };
+ clickhouse_url_defaults(&mut options, config);
+ let url = parse_url(&options.url.clone().unwrap_or_default());
+ let args: HashMap<_, _> = url.query_pairs().into_owned().collect();
+
+ assert_eq!(args.get("secure"), Some(&"true".into()));
+ assert_eq!(args.get("ca_certificate"), Some(&"ca".into()));
+ assert_eq!(args.get("client_certificate"), Some(&"cert".into()));
+ assert_eq!(args.get("client_private_key"), Some(&"key".into()));
+ assert_eq!(args.get("skip_verify"), Some(&"false".into()));
+ }
+
+ #[test]
+ fn test_config_connections_applying_config_to_connection_url_play() {
+ let config = read_yaml_clickhouse_client_config("tests/configs/connections.yaml").ok();
+ let mut options = ClickHouseOptions {
+ connection: Some("play".into()),
+ ..Default::default()
+ };
+ clickhouse_url_defaults(&mut options, config);
+ let url = parse_url(&options.url.clone().unwrap_or_default());
+ let args: HashMap<_, _> = url.query_pairs().into_owned().collect();
+
+ assert_eq!(url.host().unwrap().to_string(), "play.clickhouse.com");
+ assert_eq!(args.get("secure"), Some(&"true".into()));
+ assert_eq!(args.contains_key("skip_verify"), false);
+ }
+
+ #[test]
+ fn test_config_connections_applying_config_to_connection_url_play_tls() {
+ let config = read_yaml_clickhouse_client_config("tests/configs/connections.yaml").ok();
+ let mut options = ClickHouseOptions {
+ connection: Some("play-tls".into()),
+ ..Default::default()
+ };
+ clickhouse_url_defaults(&mut options, config);
+ let url = parse_url(&options.url.clone().unwrap_or_default());
+ let args: HashMap<_, _> = url.query_pairs().into_owned().collect();
+
+ assert_eq!(url.host().unwrap().to_string(), "play.clickhouse.com");
+ assert_eq!(args.get("secure"), Some(&"true".into()));
+ assert_eq!(args.get("ca_certificate"), Some(&"ca".into()));
+ assert_eq!(args.get("client_certificate"), Some(&"cert".into()));
+ assert_eq!(args.get("client_private_key"), Some(&"key".into()));
+ assert_eq!(args.get("skip_verify"), Some(&"true".into()));
+ }
+}
diff --git a/tests/configs/basic.xml b/tests/configs/basic.xml
new file mode 100644
index 0000000..34a5a4c
--- /dev/null
+++ b/tests/configs/basic.xml
@@ -0,0 +1,4 @@
+
+ foo
+ bar
+
diff --git a/tests/configs/basic.yaml b/tests/configs/basic.yaml
new file mode 100644
index 0000000..64bd4bb
--- /dev/null
+++ b/tests/configs/basic.yaml
@@ -0,0 +1,3 @@
+---
+user: foo
+password: bar
diff --git a/tests/configs/connections.yaml b/tests/configs/connections.yaml
new file mode 100644
index 0000000..73fe647
--- /dev/null
+++ b/tests/configs/connections.yaml
@@ -0,0 +1,15 @@
+---
+connections_credentials:
+ play:
+ name: play
+ hostname: play.clickhouse.com
+ secure: true
+
+ play-tls:
+ name: play-tls
+ hostname: play.clickhouse.com
+ secure: true
+ ca_certificate: ca
+ client_certificate: cert
+ client_private_key: key
+ skip_verify: true
diff --git a/tests/configs/empty.xml b/tests/configs/empty.xml
new file mode 100644
index 0000000..3cbf717
--- /dev/null
+++ b/tests/configs/empty.xml
@@ -0,0 +1,2 @@
+
+
diff --git a/tests/configs/empty.yaml b/tests/configs/empty.yaml
new file mode 100644
index 0000000..e69de29
diff --git a/tests/configs/tls.xml b/tests/configs/tls.xml
new file mode 100644
index 0000000..19c0d17
--- /dev/null
+++ b/tests/configs/tls.xml
@@ -0,0 +1,11 @@
+
+ true
+
+
+ strict
+ cert
+ key
+ ca
+
+
+
diff --git a/tests/configs/tls.yaml b/tests/configs/tls.yaml
new file mode 100644
index 0000000..d56b56a
--- /dev/null
+++ b/tests/configs/tls.yaml
@@ -0,0 +1,8 @@
+---
+secure: true
+openSSL:
+ client:
+ verificationMode: strict
+ certificateFile: cert
+ privateKeyFile: key
+ caConfig: ca
diff --git a/tests/configs/unknown_directives.xml b/tests/configs/unknown_directives.xml
new file mode 100644
index 0000000..a16c1f6
--- /dev/null
+++ b/tests/configs/unknown_directives.xml
@@ -0,0 +1,3 @@
+
+ bar
+
diff --git a/tests/configs/unknown_directives.yaml b/tests/configs/unknown_directives.yaml
new file mode 100644
index 0000000..23809fe
--- /dev/null
+++ b/tests/configs/unknown_directives.yaml
@@ -0,0 +1,2 @@
+---
+foo: bar