Collect data from the inverter and floor heating/cooling sensors

[zmoog]

Goals

• Publish a new Collector build.
• Collect all logs & metrics in a ECH deployment in Elastic Cloud.
• toggl: parse tags.
• zcs: build new dashboard with datapoint from the *.otel datasets.

[zmoog]

Deployed collector 0.5.0 as latest@sha256:3316393cc5ecf27b80ba5972415131ea037ddcec24b30f1b96d22ee3b9844554

[zmoog]

Updated the collector pod, now the collector is scraping all metrics (zcs, ws) and logs (toggl).

Missing:

• ws/zcs: add dashboard for new data *.otel datasets
• toggl: tags parsing

[zmoog]

Added two missing features to collect Toggl timer entries as logs:

• deduplication by setting _id
• tags parsing as work.[type|activity|stream|project]

GET _ingest/pipeline/toggl-track-pipeline-otel
{
  "toggl-track-pipeline-otel": {
    "processors": [
      {
        "set": {
          "field": "_id",
          "copy_from": "attributes.id"
        }
      },
      {
        "foreach": {
          "field": "attributes.tags",
          "processor": {
            "grok": {
              "field": "_ingest._value",
              "pattern_definitions": {
                "STREAMID": ".+",
                "PROJECTID": ".+",
                "TYPEID": ".+",
                "ACTIVITYID": ".+"
              },
              "patterns": [
                "type:%{TYPEID:work.type}/%{ACTIVITYID:work.activity}",
                "type:%{TYPEID:work.type}",
                "stream:%{STREAMID:work.stream}/%{PROJECTID:work.project}",
                "stream:%{STREAMID:work.stream}"
              ]
            }
          },
          "if": "ctx.attributes?.tags != null"
        }
      },
      {
        "set": {
          "field": "work.activity",
          "copy_from": "work.type",
          "if": "ctx.work?.activity == null && ctx.work?.type != null "
        }
      },
      {
        "set": {
          "field": "work.project",
          "copy_from": "work.stream",
          "if": "ctx.work?.project == null && ctx.work?.stream != null "
        }
      }
    ]
  }
}

Index template:

PUT _index_template/logs-toggltrackreceiver.otel@template
{
  "version": 9,
  "priority": 130,
  "template": {
    "settings": {
      "index": {
        "mode": "logsdb",
        "default_pipeline": "toggl-track-pipeline-otel"
      }
    },
    "mappings": {
      "_source": {
        "mode": "synthetic"
      },
      "dynamic_templates": [],
      "properties": {
        "work": {
          "dynamic": true,
          "type": "object",
          "enabled": true,
          "properties": {
            "activity": {
              "type": "keyword"
            },
            "stream": {
              "type": "keyword"
            },
            "project": {
              "type": "keyword"
            },
            "type": {
              "type": "keyword"
            }
          },
          "subobjects": true
        },
        "data_stream.type": {
          "type": "constant_keyword",
          "value": "logs"
        }
      }
    }
  },
  "index_patterns": [
    "logs-toggltrackreceiver.otel-*"
  ],
  "data_stream": {
    "hidden": false,
    "allow_custom_routing": false
  },
  "composed_of": [
    "logs@mappings",
    "logs@settings",
    "otel@mappings",
    "logs-otel@mappings",
    "semconv-resource-to-ecs@mappings",
    "ecs@mappings",
    "logs@custom",
    "logs-otel@custom"
  ],
  "ignore_missing_component_templates": [
    "logs@custom",
    "logs-otel@custom"
  ],
  "allow_auto_create": true,
  "_meta": {
    "description": "default OpenTelemetry logs template installed by x-pack",
    "managed": true
  }
}