From 184493240605081ba5d610ff6d839ea9094b8dbb Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 8 Nov 2024 21:09:12 +0100 Subject: [PATCH 01/14] Bumps version numbers --- config/_default/config.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/_default/config.toml b/config/_default/config.toml index 7715a91a..e4c89fe4 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -66,7 +66,7 @@ description = "Documentation for AxoSyslog, the scalable security data processor # The version number for the version of the docs represented in this doc set. # Used in the "version-banner" partial to display a version number for the # current doc set. - version = "4.8.0" + version = "4.9.0" version_menu_canonicallinks = true # A link to latest version of the docs. Used in the "version-banner" partial to @@ -160,9 +160,9 @@ description = "Documentation for AxoSyslog, the scalable security data processor [params.product] name = "AxoSyslog" abbrev = "AxoSyslog" -version = "4.8" -techversion = "4.8.1" -configversion = "4.8" +version = "4.9" +techversion = "4.9.0" +configversion = "4.9" syslog-ng = "syslog-ng" selinux = "SELinux" apparmor = "AppArmor" From 14eab5a38d2064f72d771cf64bd477e01b8884fa Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 8 Nov 2024 21:34:56 +0100 Subject: [PATCH 02/14] [4.9] Adds idle-timeout source option https://github.com/axoflow/axosyslog/pull/355 --- .../reference-source-file/_index.md | 2 ++ .../stdin()-source-options/_index.md | 2 ++ .../reference-source-systemd-syslog/_index.md | 2 ++ .../reference-source-wildcard-file/_index.md | 2 ++ .../source-pipe/reference-source-pipe/_index.md | 2 ++ .../reference-source-program/_index.md | 6 ++++-- .../reference-source-unixstream/_index.md | 1 + .../headless/chunk/option-source-idle-timeout.md | 14 ++++++++++++++ content/headless/chunk/source-syslog-options.md | 1 + 9 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 content/headless/chunk/option-source-idle-timeout.md diff --git a/content/chapter-sources/configuring-sources-file/reference-source-file/_index.md b/content/chapter-sources/configuring-sources-file/reference-source-file/_index.md index 77961777..082d5d2b 100644 --- a/content/chapter-sources/configuring-sources-file/reference-source-file/_index.md +++ b/content/chapter-sources/configuring-sources-file/reference-source-file/_index.md @@ -20,6 +20,8 @@ The `file()` driver has the following options: {{< include-headless "chunk/option-destination-hook.md" >}} +{{< include-headless "chunk/option-source-idle-timeout.md" >}} + {{< include-headless "chunk/option-source-keep-timestamp.md" >}} {{< include-headless "chunk/option-source-log-fetch-limit.md" >}} diff --git a/content/chapter-sources/configuring-sources-stdin/stdin()-source-options/_index.md b/content/chapter-sources/configuring-sources-stdin/stdin()-source-options/_index.md index 5dcc9efe..d0b524ac 100644 --- a/content/chapter-sources/configuring-sources-stdin/stdin()-source-options/_index.md +++ b/content/chapter-sources/configuring-sources-stdin/stdin()-source-options/_index.md @@ -20,6 +20,8 @@ The `stdin()` driver has the following options: {{< include-headless "chunk/option-destination-hook.md" >}} +{{< include-headless "chunk/option-source-idle-timeout.md" >}} + {{< include-headless "chunk/option-source-keep-timestamp.md" >}} {{% include-headless "chunk/option-source-log-fetch-limit.md" %}} diff --git a/content/chapter-sources/configuring-sources-systemd-syslog/reference-source-systemd-syslog/_index.md b/content/chapter-sources/configuring-sources-systemd-syslog/reference-source-systemd-syslog/_index.md index 123bd818..4561e1c3 100644 --- a/content/chapter-sources/configuring-sources-systemd-syslog/reference-source-systemd-syslog/_index.md +++ b/content/chapter-sources/configuring-sources-systemd-syslog/reference-source-systemd-syslog/_index.md @@ -9,3 +9,5 @@ The `systemd-syslog()` driver has the following options: {{< include-headless "chunk/option-source-check-hostname.md" >}} {{< include-headless "chunk/option-destination-hook.md" >}} + +{{< include-headless "chunk/option-source-idle-timeout.md" >}} diff --git a/content/chapter-sources/configuring-sources-wildcard-file/reference-source-wildcard-file/_index.md b/content/chapter-sources/configuring-sources-wildcard-file/reference-source-wildcard-file/_index.md index a920562b..8d157c2f 100644 --- a/content/chapter-sources/configuring-sources-wildcard-file/reference-source-wildcard-file/_index.md +++ b/content/chapter-sources/configuring-sources-wildcard-file/reference-source-wildcard-file/_index.md @@ -63,6 +63,8 @@ For example, `filename-pattern("*.log")` matches the `syslog.log` and `auth.log` {{< include-headless "chunk/option-destination-hook.md" >}} +{{< include-headless "chunk/option-source-idle-timeout.md" >}} + {{< include-headless "chunk/option-source-keep-timestamp.md" >}} {{% include-headless "chunk/option-source-log-fetch-limit.md" %}} diff --git a/content/chapter-sources/source-pipe/reference-source-pipe/_index.md b/content/chapter-sources/source-pipe/reference-source-pipe/_index.md index 9eb4f8a9..b21abdd7 100644 --- a/content/chapter-sources/source-pipe/reference-source-pipe/_index.md +++ b/content/chapter-sources/source-pipe/reference-source-pipe/_index.md @@ -16,6 +16,8 @@ The `pipe` driver has the following options: {{< include-headless "chunk/option-destination-hook.md" >}} +{{< include-headless "chunk/option-source-idle-timeout.md" >}} + {{< include-headless "chunk/option-source-keep-timestamp.md" >}} {{% include-headless "chunk/option-source-log-fetch-limit.md" %}} diff --git a/content/chapter-sources/source-program/reference-source-program/_index.md b/content/chapter-sources/source-program/reference-source-program/_index.md index 62990360..f4041620 100644 --- a/content/chapter-sources/source-program/reference-source-program/_index.md +++ b/content/chapter-sources/source-program/reference-source-program/_index.md @@ -12,12 +12,14 @@ The `program` driver has the following options: {{< include-headless "chunk/option-destination-hook.md" >}} +{{< include-headless "chunk/option-source-idle-timeout.md" >}} + +{{% include-headless "chunk/option-inherit-environment.md" %}} + {{< include-headless "chunk/option-source-keep-timestamp.md" >}} {{% include-headless "chunk/option-source-log-fetch-limit.md" %}} -{{% include-headless "chunk/option-inherit-environment.md" %}} - {{% include-headless "chunk/option-source-log-iw-size.md" %}} {{< include-headless "chunk/option-source-log-msg-size.md" >}} diff --git a/content/chapter-sources/source-unixstream/reference-source-unixstream/_index.md b/content/chapter-sources/source-unixstream/reference-source-unixstream/_index.md index 07a63fc1..6a7c7a50 100644 --- a/content/chapter-sources/source-unixstream/reference-source-unixstream/_index.md +++ b/content/chapter-sources/source-unixstream/reference-source-unixstream/_index.md @@ -29,6 +29,7 @@ These two drivers behave similarly: they open an `AF_UNIX` socket and start list {{% include-headless "chunk/option-source-host-override.md" %}} +{{< include-headless "chunk/option-source-idle-timeout.md" >}} ## keep-alive() diff --git a/content/headless/chunk/option-source-idle-timeout.md b/content/headless/chunk/option-source-idle-timeout.md new file mode 100644 index 00000000..76f20334 --- /dev/null +++ b/content/headless/chunk/option-source-idle-timeout.md @@ -0,0 +1,14 @@ +--- +--- + + +## idle-timeout() + +| | | +| ---------------- | ------------------ | +| Accepted values: | number [seconds] | +| Default: | | + +Available in {{< product >}} 4.9 and later. + +If set, {{< product >}} closes the client connection if no data is received for the specified amount of time (in seconds). diff --git a/content/headless/chunk/source-syslog-options.md b/content/headless/chunk/source-syslog-options.md index af5cde04..2e3e85ee 100644 --- a/content/headless/chunk/source-syslog-options.md +++ b/content/headless/chunk/source-syslog-options.md @@ -24,6 +24,7 @@ The `syslog` source uses multiple threads only if the source uses the `tls` or ` {{% include-headless "chunk/option-source-host-override.md" %}} +{{< include-headless "chunk/option-source-idle-timeout.md" >}} ## interface() From 8e720e7a72db9db533d6d2f4c03aa77d3e14da35 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 8 Nov 2024 21:40:03 +0100 Subject: [PATCH 03/14] [4.9][loki] Adds batch-bytes and compression options https://github.com/axoflow/axosyslog/pull/276 --- content/chapter-destinations/destination-loki/_index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/chapter-destinations/destination-loki/_index.md b/content/chapter-destinations/destination-loki/_index.md index 40cc61ab..86284dbb 100644 --- a/content/chapter-destinations/destination-loki/_index.md +++ b/content/chapter-destinations/destination-loki/_index.md @@ -35,6 +35,8 @@ The `loki()` destination has the following options. {{< include-headless "chunk/grpc-authentication.md" >}} +{{% include-headless "chunk/option-destination-batch-bytes.md" %}} + ## batch-lines() | | | @@ -48,6 +50,8 @@ The `loki()` destination has the following options. {{< include-headless "chunk/option-grpc-channel-args.md" >}} +{{< include-headless "chunk/option-destination-grpc-compression.md" >}} + {{< include-headless "chunk/option-grpc-headers.md" >}} {{< include-headless "chunk/option-destination-grpc-keep-alive.md" >}} From 950df3ad2e25a9e86993aabfbc4cbae70478ef47 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 8 Nov 2024 21:45:12 +0100 Subject: [PATCH 04/14] [4.9][bigquery] Adds grpc authentication https://github.com/axoflow/axosyslog/pull/276 --- content/chapter-destinations/google-bigquery/_index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/content/chapter-destinations/google-bigquery/_index.md b/content/chapter-destinations/google-bigquery/_index.md index e5015971..cd2ac751 100644 --- a/content/chapter-destinations/google-bigquery/_index.md +++ b/content/chapter-destinations/google-bigquery/_index.md @@ -56,6 +56,8 @@ By default, the messages are sent with one worker, one message per batch, and wi The `bigquery()` destination has the following options. +{{< include-headless "chunk/grpc-authentication.md" >}} + {{% include-headless "chunk/option-destination-batch-bytes.md" %}} By default, the `batch-bytes()` option of the `bigquery()` destination is 10 MB. This is an upper limit for the `bigquery()` destination. Note that due to a framework limitation, the batch might be at most 1 message larger than the set limit. From 56273ea99807b4b70d664a7f33c3543be80ee73b Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 8 Nov 2024 21:50:06 +0100 Subject: [PATCH 05/14] Fix order of source flags --- content/headless/chunk/option-source-flags.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/headless/chunk/option-source-flags.md b/content/headless/chunk/option-source-flags.md index f8d47d2a..6c7e56a1 100644 --- a/content/headless/chunk/option-source-flags.md +++ b/content/headless/chunk/option-source-flags.md @@ -12,6 +12,7 @@ *Description:* Specifies the log parsing options of the source. - *assume-utf8*: The `assume-utf8` flag assumes that the incoming messages are UTF-8 encoded, but does not verify the encoding. If you explicitly want to validate the UTF-8 encoding of the incoming message, use the `validate-utf8` flag. +- *dont-store-legacy-msghdr*: By default, AxoSyslog stores the original incoming header of the log message. This is useful if the original format of a non-syslog-compliant message must be retained (AxoSyslog automatically corrects minor header errors, for example, adds a whitespace before `msg` in the following message: `Jan 22 10:06:11 host program:msg`). If you do not want to store the original header of the message, enable the `dont-store-legacy-msghdr` flag. - *empty-lines*: Use the `empty-lines` flag to keep the empty lines of the messages. By default, {{% param "product.abbrev" %}} removes empty lines automatically. - *expect-hostname*: If the `expect-hostname` flag is enabled, {{% param "product.abbrev" %}} will assume that the log message contains a hostname and parse the message accordingly. This is the default behavior for TCP sources. Note that pipe sources use the `no-hostname` flag by default. - *guess-timezone*: Attempt to guess the timezone of the message if this information is not available in the message. Works when the incoming message stream is close to real time, and the timezone information is missing from the timestamp. @@ -52,8 +53,6 @@ Essentially, the `no-header` flag signals {{% param "product.abbrev" %}} that th {{% include-headless "chunk/para-flags-no-parse.md" %}} -- *dont-store-legacy-msghdr*: By default, AxoSyslog stores the original incoming header of the log message. This is useful if the original format of a non-syslog-compliant message must be retained (AxoSyslog automatically corrects minor header errors, for example, adds a whitespace before `msg` in the following message: `Jan 22 10:06:11 host program:msg`). If you do not want to store the original header of the message, enable the `dont-store-legacy-msghdr` flag. - - *sanitize-utf8*: When using the `sanitize-utf8` flag, {{% param "product.abbrev" %}} converts non-UTF-8 input to an escaped form, which is valid UTF-8. Prior to version 4.6, this flag worked only when parsing RFC3164 messages. Starting with version 4.6, it works also for RFC5424 and raw messages. From e6c4abecec5d05f8e447d153765e6e101b239584 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 8 Nov 2024 21:52:25 +0100 Subject: [PATCH 06/14] [4.9] Adds exit-on-eof https://github.com/axoflow/axosyslog/pull/351 --- content/headless/chunk/option-source-flags.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/headless/chunk/option-source-flags.md b/content/headless/chunk/option-source-flags.md index 6c7e56a1..0911fb23 100644 --- a/content/headless/chunk/option-source-flags.md +++ b/content/headless/chunk/option-source-flags.md @@ -14,6 +14,7 @@ - *assume-utf8*: The `assume-utf8` flag assumes that the incoming messages are UTF-8 encoded, but does not verify the encoding. If you explicitly want to validate the UTF-8 encoding of the incoming message, use the `validate-utf8` flag. - *dont-store-legacy-msghdr*: By default, AxoSyslog stores the original incoming header of the log message. This is useful if the original format of a non-syslog-compliant message must be retained (AxoSyslog automatically corrects minor header errors, for example, adds a whitespace before `msg` in the following message: `Jan 22 10:06:11 host program:msg`). If you do not want to store the original header of the message, enable the `dont-store-legacy-msghdr` flag. - *empty-lines*: Use the `empty-lines` flag to keep the empty lines of the messages. By default, {{% param "product.abbrev" %}} removes empty lines automatically. +- *exit-on-eof*: If this flag is set on a source, {{< product >}} stops when an EOF (end of file) is received. Available in version 4.9 and later. - *expect-hostname*: If the `expect-hostname` flag is enabled, {{% param "product.abbrev" %}} will assume that the log message contains a hostname and parse the message accordingly. This is the default behavior for TCP sources. Note that pipe sources use the `no-hostname` flag by default. - *guess-timezone*: Attempt to guess the timezone of the message if this information is not available in the message. Works when the incoming message stream is close to real time, and the timezone information is missing from the timestamp. - *kernel*: The `kernel` flag makes the source default to the `LOG_KERN | LOG_NOTICE` priority if not specified otherwise. From 277ec5267e566080480925d74b25859f45e88ee9 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 9 Nov 2024 10:02:02 +0100 Subject: [PATCH 07/14] [4.9][grpc] Adds templated values to grpc headers https://github.com/axoflow/axosyslog/pull/334 --- content/headless/chunk/option-grpc-headers.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/content/headless/chunk/option-grpc-headers.md b/content/headless/chunk/option-grpc-headers.md index 2426b25a..141525d5 100644 --- a/content/headless/chunk/option-grpc-headers.md +++ b/content/headless/chunk/option-grpc-headers.md @@ -11,7 +11,7 @@ Available in {{< product >}} 4.8 and later. -*Description:* Adds custom gRPC headers to each RPC call. Currently only static header names and values are supported. +*Description:* Adds custom gRPC headers to each RPC call. Version 4.8 supported only static header names and values. For example: ```shell headers( @@ -19,3 +19,12 @@ headers( "stream-name" => "axo-stream" ) ``` + +Starting with version 4.9, you can use templates and macros in the header values. + +```shell +headers( + "organization" => "Axoflow" + "stream-name" => "${HOST}" + ) +``` From 4f555b0f069b5266bb2885a12c744b7baff32ecc Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 9 Nov 2024 10:10:51 +0100 Subject: [PATCH 08/14] [4.9][opentelemetry] Adds new options to opentelemetry and syslog-ng-opentelemetry destinations General options from https://github.com/axoflow/axosyslog/pull/334 --- .../destination-syslog-ng-otlp/_index.md | 14 ++++++++++++++ .../chapter-destinations/opentelemetry/_index.md | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/content/chapter-destinations/destination-syslog-ng-otlp/_index.md b/content/chapter-destinations/destination-syslog-ng-otlp/_index.md index 80d4cee1..de38efeb 100644 --- a/content/chapter-destinations/destination-syslog-ng-otlp/_index.md +++ b/content/chapter-destinations/destination-syslog-ng-otlp/_index.md @@ -37,18 +37,32 @@ The `syslog-ng-otlp()` destination has the following options. {{< include-headless "chunk/option-destination-diskbuffer.md" >}} +{{< include-headless "chunk/option-destination-frac-digits.md" >}} + {{< include-headless "chunk/option-destination-hook.md" >}} +{{% include-headless "chunk/option-destination-local-timezone.md" %}} + {{% include-headless "chunk/option-destination-log-fifo-size.md" %}} +{{< include-headless "chunk/option-destination-on-error.md" >}} + {{% include-headless "chunk/option-persist-name.md" %}} {{% include-headless "chunk/option-destination-retries.md" %}} +{{% include-headless "chunk/option-destination-send-timezone.md" %}} + +{{% include-headless "chunk/option-destination-template-escape.md" %}} + {{% include-headless "chunk/option-destination-throttle.md" %}} {{% include-headless "chunk/option-source-time-reopen.md" %}} +{{% include-headless "chunk/option-destination-timezone.md" %}} + +{{< include-headless "chunk/option-destination-ts-format.md" >}} + ## url() | | | diff --git a/content/chapter-destinations/opentelemetry/_index.md b/content/chapter-destinations/opentelemetry/_index.md index 4402bd94..d9585273 100644 --- a/content/chapter-destinations/opentelemetry/_index.md +++ b/content/chapter-destinations/opentelemetry/_index.md @@ -70,6 +70,20 @@ log non_otel_to_otel_tls { {{< include-headless "chunk/option-grpc-headers.md" >}} +{{< include-headless "chunk/option-destination-frac-digits.md" >}} + +{{% include-headless "chunk/option-destination-local-timezone.md" %}} + +{{< include-headless "chunk/option-destination-on-error.md" >}} + +{{% include-headless "chunk/option-destination-send-timezone.md" %}} + +{{% include-headless "chunk/option-destination-template-escape.md" %}} + +{{% include-headless "chunk/option-destination-timezone.md" %}} + +{{< include-headless "chunk/option-destination-ts-format.md" >}} + {{< include-headless "chunk/option-destination-threaded-workers.md" >}} \ No newline at end of file +*Description:* The URL of the OpenTelemetry receiver. + + +{{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} + +{{< include-headless "chunk/option-destination-threaded-workers.md" >}} From 7195c3e11a2070e4489f30df90f381538c2620f2 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 9 Nov 2024 10:42:59 +0100 Subject: [PATCH 10/14] [4.9][man] Adds the syslog-ng-ctl attach command https://github.com/axoflow/axosyslog/pull/326/ --- content/app-man-syslog-ng/syslog-ng-ctl.1.md | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/content/app-man-syslog-ng/syslog-ng-ctl.1.md b/content/app-man-syslog-ng/syslog-ng-ctl.1.md index 51582ce4..45d00dc2 100644 --- a/content/app-man-syslog-ng/syslog-ng-ctl.1.md +++ b/content/app-man-syslog-ng/syslog-ng-ctl.1.md @@ -63,6 +63,7 @@ If you need to use a non-standard control socket to access `syslog-ng`, use the syslog-ng-ctl log-level verbose ``` +To temporarily change the log levels and access the logs of `syslog-ng`, see also the [`attach` command]({{< relref "#attach" >}}). @@ -371,6 +372,40 @@ You can run `syslog-ng-ctl healthcheck --timeout ` to use as a boolean Health checks are also published as periodically updated metrics. You can configure the frequency of these checks with the `stats(healthcheck-freq())` option. The default is 5 minutes. +## The attach command {#attach} + +Available in {{% param "product.abbrev" %}} 4.9 and later. + +Connect to the standard IO (stdin, stdout, stderr) and display the results. Note that there can only be one attached process at a time. + +`syslog-ng-ctl attach [attach-mode] [options]` + +The `syslog-ng-ctl attach` command has the following parameters: + +- Attach mode: `logs` or `stdio`. + + - Use `logs` to access the internal log messages of `syslog-ng`. For example, the following command changes the log level to `trace` and accesses the internal logs of `syslog-ng`: + + ```shell + syslog-ng-ctl attach logs --seconds 10 --log-level trace + ``` + + - Use `stdio` to display the output of the `syslog-ng` process. For example: + + ```shell + syslog-ng-ctl attach stdio --seconds 10 + ``` + +- Change `log-level` to the specified value: + + {{< include-headless "chunk/internal-log-levels.md" >}} + +- How long to attach to the process: `--seconds`. For example: + + ```shell + syslog-ng-ctl attach stdio --seconds 10 + ``` + ## Files `/opt/syslog-ng/sbin/syslog-ng-ctl` From f1003c1a5517a37cd3a567c57230eba5d84585d0 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 9 Nov 2024 11:25:27 +0100 Subject: [PATCH 11/14] [4.9][clickhouse] Draft clickhouse docs --- .../chapter-destinations/clickhouse/_index.md | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 content/chapter-destinations/clickhouse/_index.md diff --git a/content/chapter-destinations/clickhouse/_index.md b/content/chapter-destinations/clickhouse/_index.md new file mode 100644 index 00000000..51c5160c --- /dev/null +++ b/content/chapter-destinations/clickhouse/_index.md @@ -0,0 +1,198 @@ +--- +title: ClickHouse database +linktitle: "ClickHouse" +weight: 200 +driver: "clickhouse()" +short_description: "Send messages to a ClickHouse database" +--- + + +Starting with version 4.9.0, {{% param "product_name" %}} can send data to [ClickHouse databases](https://clickhouse.com/) using its [gRPC interface](https://clickhouse.com/docs/en/interfaces/grpc). + +## Prerequisites + +- A [self-hosted ClickHouse installation](https://clickhouse.com/docs/en/install). + + {{% alert title="Warning" color="warning" %}} +ClickHouse Cloud doesn't support the gRPC interface currently. + {{% /alert %}} + +- The [gRPC interface](https://clickhouse.com/docs/en/interfaces/grpc) must be enabled in your ClickHouse configuration. +- To configure {{% param "product_name" %}}, you'll need: + + - the name of an existing database and a table where you want to send your data, and + - the credentials (username and password) to access the database. + +Example configuration: + +```sh +destination { + clickhouse( + database("default") + table("demo_table") + user("your-username") + password("your-password") + schema( + "user_id" UInt32 => $R_MSEC, + "message" String => "$MSG", + "timestamp" DateTime => "$R_UNIXTIME", + "metric" Float32 => 3.14 + ) + ); +}; +``` + +## Options + +This destination has the following options: + +{{< include-headless "chunk/grpc-authentication.md" >}} + +{{% include-headless "chunk/option-destination-otlp-batch-bytes.md" %}} + +## batch-lines() + +| | | +| -------- | ------ | +| Type: | number | +| Default: | 0 | + +{{% include-headless "chunk/option-description-destination-batch-lines.md" %}} + +{{% include-headless "chunk/option-destination-batch-timeout.md" %}} + +{{< include-headless "chunk/option-grpc-channel-args.md" >}} + +{{< include-headless "chunk/option-destination-grpc-compression.md" >}} + +## database() + +| | | +| -------- | -------------------------- | +| Type: | string | +| Default: | `default` | + +*Description:* The database where {{% param "product_name" %}} sends the data. + +{{< include-headless "chunk/option-destination-diskbuffer.md" >}} + +{{< include-headless "chunk/option-destination-frac-digits.md" >}} + +{{< include-headless "chunk/option-grpc-headers.md" >}} + +{{< include-headless "chunk/option-destination-hook.md" >}} + +{{< include-headless "chunk/option-destination-grpc-keep-alive.md" >}} + +{{% include-headless "chunk/option-destination-local-timezone.md" %}} + +{{% include-headless "chunk/option-destination-log-fifo-size.md" %}} + +{{< include-headless "chunk/option-destination-on-error.md" >}} + +## password() + +| | | +| -------- | ------------ | +| Type: | string | +| Default: | empty string | + +*Description:* The password used for authentication. + +{{% include-headless "chunk/option-persist-name.md" %}} + +## protobuf-schema() + +| | | +| -------- | -------------------------- | +| Type: | See the description | +| Default: | - | + +*Description:* Sets the schema of the database table from a protobuf schema file. + +```shell +protobuf-schema("/tmp/test.proto" => "$MESSAGE", "$PROGRAM", "$HOST", "$PID") +``` + +An example proto file when using the `protobuf-schema()` option: + +```shell +syntax = "proto2"; +​ +message CustomRecord { + optional string message = 1; + optional string app = 2; + optional string host = 3; + optional int64 pid = 4; +} +``` + +Alternatively, you can set the schema with the [`schema()`](#schema) option. + +{{% include-headless "chunk/option-destination-retries.md" %}} + +## schema() + +| | | +| -------- | -------------------------- | +| Type: | arrow list | +| Default: | - | + +*Description:* Sets the schema of the database table. On the left side of the arrow, set the name of the column and its type. On the right side, set any {{% param "product_name" %}} template or macro, which gets evaluated on each log that is routed to the destination. For example: + +```shell +schema( + "user_id" UInt32 => $R_MSEC, + "message" String => "$MSG", + "timestamp" DateTime => "$R_UNIXTIME", + "metric" Float32 => 3.14 +) +``` + + + +Alternatively, you can set the schema with the [`protobuf-schema()`](#protobuf-schema) option. + +{{% include-headless "chunk/option-destination-send-timezone.md" %}} + +## table() + +| | | +| -------- | -------------------------- | +| Type: | string | +| Default: | - | + +*Description:* The name of the table where {{% param "product_name" %}} sends the data. + +{{% include-headless "chunk/option-destination-template-escape.md" %}} + +{{% include-headless "chunk/option-destination-throttle.md" %}} + +{{% include-headless "chunk/option-source-time-reopen.md" %}} + +{{% include-headless "chunk/option-destination-timezone.md" %}} + +{{< include-headless "chunk/option-destination-ts-format.md" >}} + +## url() + +| | | +| -------- | -------------------------- | +| Type: | string | +| Default: | `localhost:9100` | + +*Description:* The URL of the gRPC receiver. + +## user() + +| | | +| -------- | ------------ | +| Type: | string | +| Default: | empty string | + +*Description:* The username used for authentication. + + +{{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} + +{{< include-headless "chunk/option-destination-threaded-workers.md" >}} \ No newline at end of file From 59b6038ed497691d7ed24ad8c464c1e3a089fedc Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 9 Nov 2024 11:29:26 +0100 Subject: [PATCH 12/14] Formatting fixes --- .../chunk/option-destination-diskbuffer.md | 48 +++++++++---------- .../headless/chunk/option-destination-hook.md | 18 +++---- .../chunk/option-destination-on-error.md | 2 +- content/headless/chunk/option-persist-name.md | 3 +- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/content/headless/chunk/option-destination-diskbuffer.md b/content/headless/chunk/option-destination-diskbuffer.md index 5923c5cc..4227484d 100644 --- a/content/headless/chunk/option-destination-diskbuffer.md +++ b/content/headless/chunk/option-destination-diskbuffer.md @@ -127,33 +127,33 @@ If you want to avoid performance fluctuations: In the following case reliable disk-buffer() is used. ```shell - destination d_demo { - network( - "127.0.0.1" - port(3333) - disk-buffer( - flow-control-window-bytes(10000) - capacity-bytes(2000000) - reliable(yes) - dir("/tmp/disk-buffer") - ) - ); - }; +destination d_demo { + network( + "127.0.0.1" + port(3333) + disk-buffer( + flow-control-window-bytes(10000) + capacity-bytes(2000000) + reliable(yes) + dir("/tmp/disk-buffer") + ) + ); +}; ``` In the following case normal disk-buffer() is used. ```shell - destination d_demo { - network( - "127.0.0.1" - port(3333) - disk-buffer( - flow-control-window-size(10000) - capacity-bytes(2000000) - reliable(no) - dir("/tmp/disk-buffer") - ) - ); - }; +destination d_demo { + network( + "127.0.0.1" + port(3333) + disk-buffer( + flow-control-window-size(10000) + capacity-bytes(2000000) + reliable(no) + dir("/tmp/disk-buffer") + ) + ); +}; ``` diff --git a/content/headless/chunk/option-destination-hook.md b/content/headless/chunk/option-destination-hook.md index b5b39abd..bfbb6cdc 100644 --- a/content/headless/chunk/option-destination-hook.md +++ b/content/headless/chunk/option-destination-hook.md @@ -57,19 +57,19 @@ To execute an external program when the {{% param "product.abbrev" %}} configura *Description:* Defines an external program that is executed when the {{% param "product.abbrev" %}} configuration is stopped or torn down, for example, on shutdown or during a {{% param "product.abbrev" %}} reload. -### Example: Using the hook-commands() with a network source +### Example: Using hook-commands() with a network source In the following example, the `hook-commands()` is used with the `network()` driver and it opens an [iptables](https://en.wikipedia.org/wiki/Iptables "https://en.wikipedia.org/wiki/Iptables") port automatically as {{% param "product.abbrev" %}} is started/stopped. The assumption in this example is that the `LOGCHAIN` chain is part of a larger ruleset that routes traffic to it. Whenever the {{% param "product.abbrev" %}} created rule is there, packets can flow, otherwise the port is closed. ```shell - source { - network(transport(udp) - hook-commands( -           startup("iptables -I LOGCHAIN 1 -p udp --dport 514 -j ACCEPT") -           shutdown("iptables -D LOGCHAIN 1") -         ) -      ); - }; +source { + network(transport(udp) + hook-commands( +          startup("iptables -I LOGCHAIN 1 -p udp --dport 514 -j ACCEPT") +          shutdown("iptables -D LOGCHAIN 1") +        ) +     ); +}; ``` diff --git a/content/headless/chunk/option-destination-on-error.md b/content/headless/chunk/option-destination-on-error.md index 80949b8a..daf03466 100644 --- a/content/headless/chunk/option-destination-on-error.md +++ b/content/headless/chunk/option-destination-on-error.md @@ -4,7 +4,7 @@ ## on-error() -| Type: | One of: `drop-message, drop-property, fallback-to-string, silently-drop-message, silently-drop-property, silently-fallback-to-string` | +| Type: | One of: `drop-message`, `drop-property`, `fallback-to-string`, `silently-drop-message`, `silently-drop-property`, `silently-fallback-to-string` | |--------------|-----------| | Default: | Use the global setting (which defaults to `drop-message`) | diff --git a/content/headless/chunk/option-persist-name.md b/content/headless/chunk/option-persist-name.md index 5a8d1055..b0540d17 100644 --- a/content/headless/chunk/option-persist-name.md +++ b/content/headless/chunk/option-persist-name.md @@ -12,8 +12,7 @@ *Description:* If you receive the following error message during {{% param "product.abbrev" %}} startup, set the `persist-name()` option of the duplicate drivers: ```shell - Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down. +Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down. ``` This error happens if you use identical drivers in multiple sources, for example, if you configure two file sources to read from the same file. In this case, set the `persist-name()` of the drivers to a custom string, for example, `persist-name("example-persist-name1")`. - From c7fd74fa08f6e0687efaf496cf256e2443b9a1bc Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Mon, 11 Nov 2024 13:44:35 +0100 Subject: [PATCH 13/14] [4.9][clickhouse] Minor corrections --- content/chapter-destinations/clickhouse/_index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/chapter-destinations/clickhouse/_index.md b/content/chapter-destinations/clickhouse/_index.md index 51c5160c..0c49c409 100644 --- a/content/chapter-destinations/clickhouse/_index.md +++ b/content/chapter-destinations/clickhouse/_index.md @@ -23,7 +23,7 @@ ClickHouse Cloud doesn't support the gRPC interface currently. - the name of an existing database and a table where you want to send your data, and - the credentials (username and password) to access the database. -Example configuration: +Example configuration (sends data to the default `localhost:9100` URL): ```sh destination { @@ -136,7 +136,7 @@ Alternatively, you can set the schema with the [`schema()`](#schema) option. | | | | -------- | -------------------------- | | Type: | arrow list | -| Default: | - | +| Default: | | *Description:* Sets the schema of the database table. On the left side of the arrow, set the name of the column and its type. On the right side, set any {{% param "product_name" %}} template or macro, which gets evaluated on each log that is routed to the destination. For example: @@ -149,10 +149,10 @@ schema( ) ``` - - Alternatively, you can set the schema with the [`protobuf-schema()`](#protobuf-schema) option. +You can find the available column types in the [official ClickHouse documentation](https://clickhouse.com/docs/en/sql-reference/data-types). + {{% include-headless "chunk/option-destination-send-timezone.md" %}} ## table() From 98fcc8206b05f136dbd6e0a38d4db00c71172a68 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Mon, 11 Nov 2024 14:24:53 +0100 Subject: [PATCH 14/14] [4.9] idle-timeout clarification --- content/headless/chunk/option-source-idle-timeout.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/headless/chunk/option-source-idle-timeout.md b/content/headless/chunk/option-source-idle-timeout.md index 76f20334..235bab9f 100644 --- a/content/headless/chunk/option-source-idle-timeout.md +++ b/content/headless/chunk/option-source-idle-timeout.md @@ -7,7 +7,7 @@ | | | | ---------------- | ------------------ | | Accepted values: | number [seconds] | -| Default: | | +| Default: | `0` (disabled) | Available in {{< product >}} 4.9 and later.