From b8d8b9b3ef572ee88615d67bac0426ca94fa0b05 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Wed, 18 Dec 2024 15:02:59 -0600 Subject: [PATCH 1/2] WIP: Test generating Telegraf docs in a local environment (Dockerfile, README) --- telegraf-build/README.md | 81 +++++++++++++++++++ telegraf-build/scripts/Dockerfile | 46 +++++++++++ .../templates/aggregator_index.tmpl | 15 ++++ .../templates/aggregator_plugin.tmpl | 13 +++ telegraf-build/templates/input_index.tmpl | 14 ++++ telegraf-build/templates/input_plugin.tmpl | 13 +++ telegraf-build/templates/output_index.tmpl | 14 ++++ telegraf-build/templates/output_plugin.tmpl | 13 +++ .../templates/plugin_aggregator.tmpl | 0 telegraf-build/templates/plugin_inputs.tmpl | 13 +++ .../templates/plugin_list_entry.tmpl | 12 +++ telegraf-build/templates/plugin_outputs.tmpl | 13 +++ .../templates/plugin_processor.tmpl | 0 telegraf-build/templates/processor_index.tmpl | 15 ++++ .../templates/processor_plugin.tmpl | 13 +++ 15 files changed, 275 insertions(+) create mode 100644 telegraf-build/README.md create mode 100644 telegraf-build/scripts/Dockerfile create mode 100644 telegraf-build/templates/aggregator_index.tmpl create mode 100644 telegraf-build/templates/aggregator_plugin.tmpl create mode 100644 telegraf-build/templates/input_index.tmpl create mode 100644 telegraf-build/templates/input_plugin.tmpl create mode 100644 telegraf-build/templates/output_index.tmpl create mode 100644 telegraf-build/templates/output_plugin.tmpl create mode 100644 telegraf-build/templates/plugin_aggregator.tmpl create mode 100644 telegraf-build/templates/plugin_inputs.tmpl create mode 100644 telegraf-build/templates/plugin_list_entry.tmpl create mode 100644 telegraf-build/templates/plugin_outputs.tmpl create mode 100644 telegraf-build/templates/plugin_processor.tmpl create mode 100644 telegraf-build/templates/processor_index.tmpl create mode 100644 telegraf-build/templates/processor_plugin.tmpl diff --git a/telegraf-build/README.md b/telegraf-build/README.md new file mode 100644 index 0000000000..1dc49d3105 --- /dev/null +++ b/telegraf-build/README.md @@ -0,0 +1,81 @@ + +# Telegraf release and docs build + +The Telegraf release build process (using code in `influxdata/telegraf-internal`) includes automation for building `influxdata/docs-v2` documentation from `influxdata/telegraf` plugin README files. + +## Release build process + +1. Telegraf team triggers the release script, which generates a `docs` binary. + The binary takes a Telegraf release version tag as an argument. +2. When executed (by the Telegraf team for an official release or on your local + system for testing), the binary: + + a. Clones the `docs-v2` repo and the `telegraf` repo and checks out the specified release tag. + b. Applies `docs-v2` frontmatter templates (`docs-v2/telegraf-build/templates`) to the `telegraf` plugin README files. + c. Commits the changes to the local `docs-v2` repo and creates a PR for the changes. + +## Build Telegraf docs for local testing and preview + +> [!Warn] +> +> Use the following steps for local testing and preview only. +> +> _Don't commit the generated files to version control._ +> +> Submit edits and fixes to the `/influxdata/telegraf` repo. +> The Telegraf release process triggers documentation builds and +> submits them to `influxdata/docs-v2` for review. +> + +Follow steps to test the Telegraf docs build process and preview generated docs on your local system (not for an official Telegraf release): + +- [Build using Docker](#build-using-docker) +- [Build manually](#build-manually) + +### Run Docker to build Telegraf docs for testing and preview + +1. If you don't already have an SSH key pair, generate one for your GitHub-associated email address, add your private key to your SSH agent, and add then add the public key to your GitHub account. + +The Dockerfile leverages Docker's BuildKit and the `--ssh` flag to use your SSH keys for GitHub authentication. + +1. Open a terminal and navigate to the directory containing the Dockerfile (`./scripts`), then enter the following command to build the Docker image: + + ```bash + docker build --ssh default -t telegraf-build . + ``` + +2. Run the Docker container using the built image and mount a volume to `/app/repos/docs-v2/telegraf`: + + ```bash + docker run --rm \ + -v /Users/me/Documents/github/docs-v2/content/telegraf:/app/repos/docs-v2/content/telegraf \ + telegraf-build v1.33.0 + ``` + +Replace `/Users/me/Documents/github/docs-v2/content/telegraf` with the actual path on your host machine where you want to access the generated documentation. + +### Manually build Telegraf docs for testing and preview + +To test manually run the build process on your local system +(without a release triggered by `influxdata/telegraf`): + +1. Install a recent version of Go for your system. + +2. Clone the `influxdata/telegraf-internal` repo to your local system (for example, to `~/Documents/github/telegraf-internal`) + +3. To generate the release binaries (`telegraf-internal/telegraf_release/bin/`),change into the `~/Documents/github/telegraf-internal` directory and run `make`. + +4. To generate the documentation, run the `telegraf-internal/telegraf_release/bin/docs` binary and include the Telegraf release tag to build--for example: + + ```bash + # From ~/Documents/github + telegraf-internal/telegraf_release/bin/docs v1.33.0 + ``` + + Currently, the binary uses the templates in `telegraf-internal/telegraf_release/docs/templates/`, however we expect to permanently move them to `docs-v2/telegraf-build/templates` soon. + + The `docs` binary: + a. Clones the `docs-v2` repo and the `telegraf` repo and checks out the specified Telegraf release tag. + b. Commits the changes to the local `docs-v2` repo and creates a PR for the changes. + +5. To test the templates and preview the changes on your local machine, review the generated files in the newly cloned `docs-v2/content/telegraf` or copy them into your existing local `docs-v2` repo (but, _don't_ commit them to version control). diff --git a/telegraf-build/scripts/Dockerfile b/telegraf-build/scripts/Dockerfile new file mode 100644 index 0000000000..33c38bce7a --- /dev/null +++ b/telegraf-build/scripts/Dockerfile @@ -0,0 +1,46 @@ +# Use the official Golang 1.21 image as the base image +FROM golang:1.21 + +# Set the working directory inside the container +WORKDIR /app + +# Install necessary dependencies +RUN apt-get update && apt-get install -y \ + git \ + make \ + wget \ + tar \ + bash \ + openssh-client + +# Configure SSH for GitHub access +RUN mkdir -p /root/.ssh && chmod 0700 /root/.ssh +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts + +# Clone the telegraf-internal repository using SSH +# Use the --mount option to access the SSH agent +RUN --mount=type=ssh git clone git@github.com:influxdata/telegraf-internal.git + +# Set the working directory to the cloned repository +WORKDIR /app/telegraf-internal + +# Checkout the docs branch +RUN git checkout docs + +# (Optional) Ensure Go modules are enabled +ENV GO111MODULE=on + +# Fetch dependencies (if using Go modules) +RUN go mod tidy + +# Remove binaries from the previous build +RUN make clean + +# Build the release binaries +RUN make release + +# Set the working directory to the docs directory +WORKDIR /app/telegraf-internal/telegraf_release/docs + +# Run the docs binary. In your docker run command, specify the Telegraf release tag +CMD ["/app/telegraf-internal/telegraf_release/bin/docs"] \ No newline at end of file diff --git a/telegraf-build/templates/aggregator_index.tmpl b/telegraf-build/templates/aggregator_index.tmpl new file mode 100644 index 0000000000..180c974c60 --- /dev/null +++ b/telegraf-build/templates/aggregator_index.tmpl @@ -0,0 +1,15 @@ +--- +title: "Telegraf Aggregator Plugins" +description: "Telegraf aggregator plugins aggregator data across multiple metrics." +menu: + telegraf_v1_ref: + name: Aggregator plugins + identifier: aggregator_plugins_reference + weight: 10 +tags: [aggregator-plugins] +--- + +Telegraf aggregator plugins aggregator data across multiple metrics using e.g. +statistical functions like min, max or mean. + +{{`{{}}`}} diff --git a/telegraf-build/templates/aggregator_plugin.tmpl b/telegraf-build/templates/aggregator_plugin.tmpl new file mode 100644 index 0000000000..28271da2cd --- /dev/null +++ b/telegraf-build/templates/aggregator_plugin.tmpl @@ -0,0 +1,13 @@ +--- +description: "Telegraf plugin for aggregating metrics using {{.Name}}" +menu: + telegraf_v1_ref: + parent: aggregator_plugins_reference + name: {{.Name}} + identifier: aggregator-{{.ID}} +tags: [{{.Name}}, "aggregator-plugins", "configuration"] +related: + - /telegraf/v1/configure_plugins/ +--- + +{{.Readme}} diff --git a/telegraf-build/templates/input_index.tmpl b/telegraf-build/templates/input_index.tmpl new file mode 100644 index 0000000000..9a896a8edf --- /dev/null +++ b/telegraf-build/templates/input_index.tmpl @@ -0,0 +1,14 @@ +--- +title: "Telegraf Input Plugins" +description: "Telegraf input plugins collect metrics from the system, services, and third-party APIs." +menu: + telegraf_v1_ref: + name: Input plugins + identifier: input_plugins_reference + weight: 10 +tags: [input-plugins] +--- + +Telegraf input plugins collect metrics from the system, services, and third-party APIs. + +{{`{{< children >}}`}} diff --git a/telegraf-build/templates/input_plugin.tmpl b/telegraf-build/templates/input_plugin.tmpl new file mode 100644 index 0000000000..2c6302a123 --- /dev/null +++ b/telegraf-build/templates/input_plugin.tmpl @@ -0,0 +1,13 @@ +--- +description: "Telegraf plugin for collecting metrics from {{.Name}}" +menu: + telegraf_v1_ref: + parent: input_plugins_reference + name: {{.Name}} + identifier: input-{{.ID}} +tags: [{{.Name}}, "input-plugins", "configuration"] +related: + - /telegraf/v1/configure_plugins/ +--- + +{{.Readme}} diff --git a/telegraf-build/templates/output_index.tmpl b/telegraf-build/templates/output_index.tmpl new file mode 100644 index 0000000000..3f8a3e6c0c --- /dev/null +++ b/telegraf-build/templates/output_index.tmpl @@ -0,0 +1,14 @@ +--- +title: "Telegraf Output Plugins" +description: "Telegraf output plugins send metrics to various destinations." +menu: + telegraf_v1_ref: + name: Output plugins + identifier: output_plugins_reference + weight: 20 +tags: [output-plugins] +--- + +Telegraf output plugins send metrics to various destinations. + +{{`{{}}`}} diff --git a/telegraf-build/templates/output_plugin.tmpl b/telegraf-build/templates/output_plugin.tmpl new file mode 100644 index 0000000000..cbc84538d0 --- /dev/null +++ b/telegraf-build/templates/output_plugin.tmpl @@ -0,0 +1,13 @@ +--- +description: "Telegraf plugin for sending metrics to {{.Name}}" +menu: + telegraf_v1_ref: + parent: output_plugins_reference + name: {{.Name}} + identifier: output-{{.ID}} +tags: [{{.Name}}, "output-plugins", "configuration"] +related: + - /telegraf/v1/configure_plugins/ +--- + +{{.Readme}} diff --git a/telegraf-build/templates/plugin_aggregator.tmpl b/telegraf-build/templates/plugin_aggregator.tmpl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/telegraf-build/templates/plugin_inputs.tmpl b/telegraf-build/templates/plugin_inputs.tmpl new file mode 100644 index 0000000000..fdfa6b3d73 --- /dev/null +++ b/telegraf-build/templates/plugin_inputs.tmpl @@ -0,0 +1,13 @@ +--- +description: "Telegraf plugin for collecting metrics from {{.Name}}" +menu: + telegraf_v1_ref: + parent: input_plugins_reference + name: {{.Name}} + identifier: input-{{.ID}} +tags: [{{.Name}}, "input-plugins", "configuration"] +related: + - /telegraf/v1/configure_plugins/ +--- + +{{.Readme}} \ No newline at end of file diff --git a/telegraf-build/templates/plugin_list_entry.tmpl b/telegraf-build/templates/plugin_list_entry.tmpl new file mode 100644 index 0000000000..b18e886066 --- /dev/null +++ b/telegraf-build/templates/plugin_list_entry.tmpl @@ -0,0 +1,12 @@ + - name: {{ .Name }} + id: {{ .ID }} + description: | +{{ .Description }} + introduced: {{ .Introduced }} +{{- with .Deprecated }} + deprecated: {{ . }} +{{- end }} +{{- with .Removal }} + removal: {{ . }} +{{- end }} + tags: [{{ .Tags | join ", " }}] diff --git a/telegraf-build/templates/plugin_outputs.tmpl b/telegraf-build/templates/plugin_outputs.tmpl new file mode 100644 index 0000000000..840afa74c0 --- /dev/null +++ b/telegraf-build/templates/plugin_outputs.tmpl @@ -0,0 +1,13 @@ +--- +description: "Telegraf plugin for sending metrics to {{.Name}}" +menu: + telegraf_v1_ref: + parent: output_plugins_reference + name: {{.Name}} + identifier: output-{{.ID}} +tags: [{{.Name}}, "output-plugins", "configuration"] +related: + - /telegraf/v1/configure_plugins/ +--- + +{{.Readme}} \ No newline at end of file diff --git a/telegraf-build/templates/plugin_processor.tmpl b/telegraf-build/templates/plugin_processor.tmpl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/telegraf-build/templates/processor_index.tmpl b/telegraf-build/templates/processor_index.tmpl new file mode 100644 index 0000000000..997bd35087 --- /dev/null +++ b/telegraf-build/templates/processor_index.tmpl @@ -0,0 +1,15 @@ +--- +title: "Telegraf Processor Plugins" +description: "Telegraf processor plugins transform individual metrics." +menu: + telegraf_v1_ref: + name: Processor plugins + identifier: processor_plugins_reference + weight: 10 +tags: [processor-plugins] +--- + +Telegraf processor plugins transform individual metrics by e.g. converting +tags and fields or data-types. + +{{`{{}}`}} diff --git a/telegraf-build/templates/processor_plugin.tmpl b/telegraf-build/templates/processor_plugin.tmpl new file mode 100644 index 0000000000..e4aa701d3c --- /dev/null +++ b/telegraf-build/templates/processor_plugin.tmpl @@ -0,0 +1,13 @@ +--- +description: "Telegraf plugin for transforming metrics using {{.Name}}" +menu: + telegraf_v1_ref: + parent: processor_plugins_reference + name: {{.Name}} + identifier: processor-{{.ID}} +tags: [{{.Name}}, "processor-plugins", "configuration"] +related: + - /telegraf/v1/configure_plugins/ +--- + +{{.Readme}} From 070cbbe5e15632f54335f64ae8ee52722f4d8d0b Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Mon, 31 Mar 2025 08:36:14 -0500 Subject: [PATCH 2/2] feat(telegraf): Use metadata from autogenerated Telegraf docs in template frontmatter. - Use plugin type and version to build related links - Insert tag list into frontmatter tags - Rename metadata in content body --- .gitignore | 4 +++ telegraf-build/README.md | 26 ++++++++++++++++--- .../templates/aggregator_plugin.tmpl | 7 ++++- telegraf-build/templates/input_plugin.tmpl | 7 ++++- telegraf-build/templates/output_plugin.tmpl | 7 ++++- .../templates/plugin_list_entry.tmpl | 3 +++ .../templates/processor_plugin.tmpl | 7 ++++- 7 files changed, 53 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 7faf70c946..8f4f5fd283 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,10 @@ node_modules /content/influxdb*/**/api/**/*.html !api-docs/**/.config.yml /api-docs/redoc-static.html* +/telegraf-build +!telegraf-build/templates +!telegraf-build/scripts +!telegraf-build/README.md .vscode/* .idea **/config.toml diff --git a/telegraf-build/README.md b/telegraf-build/README.md index 1dc49d3105..203be6d932 100644 --- a/telegraf-build/README.md +++ b/telegraf-build/README.md @@ -68,14 +68,32 @@ To test manually run the build process on your local system 4. To generate the documentation, run the `telegraf-internal/telegraf_release/bin/docs` binary and include the Telegraf release tag to build--for example: ```bash - # From ~/Documents/github - telegraf-internal/telegraf_release/bin/docs v1.33.0 + # Change to `telegraf-build` in your local docs-v2 repo. + cd ~/Documents/github/docs-v2/telegraf-build + # Run the `docs` binary to generate the documentation. + ~/Documents/github/telegraf-internal/telegraf_release/bin/docs v1.33.0 ``` - Currently, the binary uses the templates in `telegraf-internal/telegraf_release/docs/templates/`, however we expect to permanently move them to `docs-v2/telegraf-build/templates` soon. + You can skip steps for local testing: + + ```bash + ~/Documents/github/telegraf-internal/telegraf_release/bin/docs -skip changelog,pull-request v1.33.0 + ``` + + The binary looks for `.tmpl` template files in `./templates` `telegraf-internal/telegraf_release/docs/templates/`, however we expect to permanently move them to `docs-v2/telegraf-build/templates` soon. The `docs` binary: a. Clones the `docs-v2` repo and the `telegraf` repo and checks out the specified Telegraf release tag. b. Commits the changes to the local `docs-v2` repo and creates a PR for the changes. -5. To test the templates and preview the changes on your local machine, review the generated files in the newly cloned `docs-v2/content/telegraf` or copy them into your existing local `docs-v2` repo (but, _don't_ commit them to version control). +5. To test the templates and preview the changes on your local machine, change to `telegraf-build/repos/docs-v2`, install dependencies, and start Hugo: + + ```bash + cd ~/Documents/github/docs-v2/telegraf-build/repos/docs-v2 + # Install dependencies + yarn install + # Start Hugo server + npx hugo serve + ``` + + Alternatively, copy the generated files your existing local `docs-v2` repo (but, _don't_ commit them to version control). diff --git a/telegraf-build/templates/aggregator_plugin.tmpl b/telegraf-build/templates/aggregator_plugin.tmpl index 28271da2cd..6dbd5eed4d 100644 --- a/telegraf-build/templates/aggregator_plugin.tmpl +++ b/telegraf-build/templates/aggregator_plugin.tmpl @@ -5,9 +5,14 @@ menu: parent: aggregator_plugins_reference name: {{.Name}} identifier: aggregator-{{.ID}} -tags: [{{.Name}}, "aggregator-plugins", "configuration"] +tags: [{{.Name}}, "aggregator-plugins", "configuration"{{range .Tags}}, "{{.}}"{{end}}] +{{if .Introduced}}introduced: "{{.Introduced}}"{{end}} +{{if .Deprecated}}deprecated: "{{.Deprecated}}"{{end}} +{{if .Removal}}removal: "{{.Removal}}"{{end}} +{{if .OS}}os_support: "{{.OS}}"{{end}} related: - /telegraf/v1/configure_plugins/ + - {{.URL}}, {{.Name}} Plugin Source --- {{.Readme}} diff --git a/telegraf-build/templates/input_plugin.tmpl b/telegraf-build/templates/input_plugin.tmpl index 2c6302a123..b93a2f4df3 100644 --- a/telegraf-build/templates/input_plugin.tmpl +++ b/telegraf-build/templates/input_plugin.tmpl @@ -5,9 +5,14 @@ menu: parent: input_plugins_reference name: {{.Name}} identifier: input-{{.ID}} -tags: [{{.Name}}, "input-plugins", "configuration"] +tags: [{{.Name}}, "input-plugins", "configuration"{{range .Tags}}, "{{.}}"{{end}}] +{{if .Introduced}}introduced: "{{.Introduced}}"{{end}} +{{if .Deprecated}}deprecated: "{{.Deprecated}}"{{end}} +{{if .Removal}}removal: "{{.Removal}}"{{end}} +{{if .OS}}os_support: "{{.OS}}"{{end}} related: - /telegraf/v1/configure_plugins/ + - {{.URL}}, {{.Name}} Plugin Source --- {{.Readme}} diff --git a/telegraf-build/templates/output_plugin.tmpl b/telegraf-build/templates/output_plugin.tmpl index cbc84538d0..bc87cbf0f6 100644 --- a/telegraf-build/templates/output_plugin.tmpl +++ b/telegraf-build/templates/output_plugin.tmpl @@ -5,9 +5,14 @@ menu: parent: output_plugins_reference name: {{.Name}} identifier: output-{{.ID}} -tags: [{{.Name}}, "output-plugins", "configuration"] +tags: [{{.Name}}, "output-plugins", "configuration"{{range .Tags}}, "{{.}}"{{end}}] +{{if .Introduced}}introduced: "{{.Introduced}}"{{end}} +{{if .Deprecated}}deprecated: "{{.Deprecated}}"{{end}} +{{if .Removal}}removal: "{{.Removal}}"{{end}} +{{if .OS}}os_support: "{{.OS}}"{{end}} related: - /telegraf/v1/configure_plugins/ + - {{.URL}}, {{.Name}} Plugin Source --- {{.Readme}} diff --git a/telegraf-build/templates/plugin_list_entry.tmpl b/telegraf-build/templates/plugin_list_entry.tmpl index b18e886066..3f342d9681 100644 --- a/telegraf-build/templates/plugin_list_entry.tmpl +++ b/telegraf-build/templates/plugin_list_entry.tmpl @@ -8,5 +8,8 @@ {{- end }} {{- with .Removal }} removal: {{ . }} +{{- end }} +{{ with .OS }} + os_support: {{ . }} {{- end }} tags: [{{ .Tags | join ", " }}] diff --git a/telegraf-build/templates/processor_plugin.tmpl b/telegraf-build/templates/processor_plugin.tmpl index e4aa701d3c..c74d32bb42 100644 --- a/telegraf-build/templates/processor_plugin.tmpl +++ b/telegraf-build/templates/processor_plugin.tmpl @@ -5,9 +5,14 @@ menu: parent: processor_plugins_reference name: {{.Name}} identifier: processor-{{.ID}} -tags: [{{.Name}}, "processor-plugins", "configuration"] +tags: [{{.Name}}, "processor-plugins", "configuration"{{range .Tags}}, "{{.}}"{{end}}] +{{if .Introduced}}introduced: "{{.Introduced}}"{{end}} +{{if .Deprecated}}deprecated: "{{.Deprecated}}"{{end}} +{{if .Removal}}removal: "{{.Removal}}"{{end}} +{{if .OS}}os_support: "{{.OS}}"{{end}} related: - /telegraf/v1/configure_plugins/ + - {{.URL}}, {{.Name}} Plugin Source --- {{.Readme}}