api/vlagent: support logs collection

 This commit adds Kubernetes logs collection with vlagent. It transfers vlagent into daemonset deployment type.
Adds needed RBAC permissions and volume mounts.

Fixes #1501

Author: AndrewChubatiuk

api/operator/v1/vlagent_types.go

@@ -73,6 +73,9 @@ type VLAgentSpec struct {
 	// +optional
 	SyslogSpec *SyslogServerSpec `json:"syslogSpec,omitempty"`
 
+	// K8sCollector configures VLAgent logs collection from K8s pods
+	K8sCollector VLAgentK8sCollector `json:"k8sCollector,omitempty"`
+
 	// ServiceAccountName is the name of the ServiceAccount to use to run the pods
 	// +optional
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
@@ -82,6 +85,40 @@ type VLAgentSpec struct {
 	vmv1beta1.CommonApplicationDeploymentParams `json:",inline,omitempty"`
 }
 
+type VLAgentK8sCollector struct {
+	// Enabled switches VLAgent to log collection mode.
+	// Note, for this purpose operator uses DaemonSet, while by default VLAgent uses StatefulSet.
+	// Switching this option will drop all persisted data.
+	Enabled bool `json:"enabled,omitempty"`
+
+	// LogsPath configures root for logs path
+	// By default VLAgent collects logs from /var/log/containers
+	LogsPath string `json:"logsPath,omitempty"`
+
+	// CheckpointsPath configures path to file where logs checkpoints are stored.
+	// By default it's stored at host's /var/lib/vlagent_checkpoints/checkpoints.json.
+	CheckpointsPath string `json:"checkpointsPath,omitempty"`
+
+	// TenantID defines default tenant ID to use for logs collected from pods in format: <accountID>:<projectID>
+	TenantID string `json:"tenantID,omitempty"`
+
+	// IgnoreFields defines fields to ignore across logs ingested from Kubernetes
+	IgnoreFields []string `json:"ignoreFields,omitempty"`
+
+	// DecolorizeFields defines fields to remove ANSI color codes across logs ingested from Kubernetes
+	DecolorizeFields []string `json:"decolorizeFields,omitempty"`
+
+	// MsgField defines fields that may contain the _msg field
+	MsgFields []string `json:"msgFields,omitempty"`
+
+	// TimeFields defines fields that may contain the _time field
+	TimeFields []string `json:"timeFields,omitempty"`
+
+	// ExtraFields defines extra fields as JSON string which should be added to each collected log line
+	// Example: '{"env":"dev","cluster":"staging"}'
+	ExtraFields string `json:"extraFields,omitempty"`
+}
+
 // SetLastSpec implements objectWithLastAppliedState interface
 func (cr *VLAgent) SetLastSpec(prevSpec VLAgentSpec) {
 	cr.ParsedLastAppliedSpec = &prevSpec
@@ -98,6 +135,14 @@ func (cr *VLAgent) Validate() error {
 	if len(cr.Spec.RemoteWrite) == 0 {
 		return fmt.Errorf("spec.remoteWrite cannot be empty array, provide at least one remoteWrite")
 	}
+	if cr.Spec.K8sCollector.Enabled {
+		if len(cr.Spec.K8sCollector.ExtraFields) > 0 {
+			var raw map[string]any
+			if err := json.Unmarshal([]byte(cr.Spec.K8sCollector.ExtraFields), &raw); err != nil {
+				return fmt.Errorf("spec.k8sCollector.extraFields is not a valid JSON: %w", err)
+			}
+		}
+	}
 	for idx, rw := range cr.Spec.RemoteWrite {
 		if rw.URL == "" {
 			return fmt.Errorf("remoteWrite.url cannot be empty at idx: %d", idx)
@@ -298,6 +343,11 @@ func (cr *VLAgent) FinalAnnotations() map[string]string {
 	return v
 }
 
+// AsCRDOwner implements interface
+func (*VLAgent) AsCRDOwner() *metav1.OwnerReference {
+	return vmv1beta1.GetCRDAsOwner(vmv1beta1.VLAgentCRD)
+}
+
 // SelectorLabels returns selector labels for querying any vlagent related resources
 func (cr *VLAgent) SelectorLabels() map[string]string {
 	return map[string]string{
@@ -402,6 +452,10 @@ func (cr *VLAgent) ProbePort() string {
 	return cr.Spec.Port
 }
 
+func (cr *VLAgent) GetClusterRoleName() string {
+	return fmt.Sprintf("monitoring:%s:vlagent-%s", cr.Namespace, cr.Name)
+}
+
 // ProbeNeedLiveness implements build.probeCRD interface
 func (*VLAgent) ProbeNeedLiveness() bool {
 	return true

api/operator/v1/zz_generated.deepcopy.go

@@ -13,11 +13,15 @@ import (
 type CRDName int
 
 const (
-	Agent CRDName = iota
+	VMAgentCRD CRDName = iota
+	VLAgentCRD
 )
 
 func (c CRDName) String() string {
-	return []string{"vmagents.operator.victoriametrics.com", "vmalerts.operator.victoriametrics.com", "vmsingles.operator.victoriametrics.com", "vmclusters.operator.victoriametrics.com", "vmauths.operator.victoriametrics.com", "vmalertmanagers.operator.victoriametrics.com"}[c]
+	return []string{
+		"vmagents.operator.victoriametrics.com",
+		"vlagents.operator.victoriametrics.com",
+	}[c]
 }
 
 type crdInfo struct {
@@ -39,7 +43,9 @@ func Init(ctx context.Context, rclient client.Client) error {
 		var n CRDName
 		switch item.Name {
 		case "vmagents.operator.victoriametrics.com":
-			n = Agent
+			n = VMAgentCRD
+		case "vlagents.operator.victoriametrics.com":
+			n = VLAgentCRD
 		default:
 			continue
 		}
@@ -54,17 +60,15 @@ func Init(ctx context.Context, rclient client.Client) error {
 
 // GetCRDAsOwner returns owner references with global CustomResourceDefinition object as owner
 // useful for non-namespaced objects, like clusterRole
-func GetCRDAsOwner(name CRDName) []metav1.OwnerReference {
+func GetCRDAsOwner(name CRDName) *metav1.OwnerReference {
 	crdData := crdCache[name]
 	if crdData == nil {
 		return nil
 	}
-	return []metav1.OwnerReference{
-		{
-			Name:       name.String(),
-			UID:        crdData.uuid,
-			Kind:       "CustomResourceDefinition",
-			APIVersion: crdData.apiVersion,
-		},
+	return &metav1.OwnerReference{
+		Name:       name.String(),
+		UID:        crdData.uuid,
+		Kind:       "CustomResourceDefinition",
+		APIVersion: crdData.apiVersion,
 	}
 }

api/operator/v1beta1/owner.go

@@ -787,8 +787,8 @@ func (cr *VMAgent) AsURL() string {
 }
 
 // AsCRDOwner implements interface
-func (*VMAgent) AsCRDOwner() []metav1.OwnerReference {
-	return GetCRDAsOwner(Agent)
+func (*VMAgent) AsCRDOwner() *metav1.OwnerReference {
+	return GetCRDAsOwner(VMAgentCRD)
 }
 
 func (cr *VMAgent) Probe() *EmbeddedProbes {

api/operator/v1beta1/vmagent_types.go

@@ -736,6 +736,60 @@ spec:
                   type: object
                   x-kubernetes-preserve-unknown-fields: true
                 type: array
+              k8sCollector:
+                description: K8sCollector configures VLAgent logs collection from
+                  K8s pods
+                properties:
+                  checkpointsPath:
+                    description: |-
+                      CheckpointsPath configures path to file where logs checkpoints are stored.
+                      By default it's stored at host's /var/lib/vlagent_checkpoints/checkpoints.json.
+                    type: string
+                  decolorizeFields:
+                    description: DecolorizeFields defines fields to remove ANSI color
+                      codes across logs ingested from Kubernetes
+                    items:
+                      type: string
+                    type: array
+                  enabled:
+                    description: |-
+                      Enabled switches VLAgent to log collection mode.
+                      Note, for this purpose operator uses DaemonSet, while by default VLAgent uses StatefulSet.
+                      Switching this option will drop all persisted data.
+                    type: boolean
+                  extraFields:
+                    description: |-
+                      ExtraFields defines extra fields as JSON string which should be added to each collected log line
+                      Example: '{"env":"dev","cluster":"staging"}'
+                    type: string
+                  ignoreFields:
+                    description: IgnoreFields defines fields to ignore across logs
+                      ingested from Kubernetes
+                    items:
+                      type: string
+                    type: array
+                  logsPath:
+                    description: |-
+                      LogsPath configures root for logs path
+                      By default VLAgent collects logs from /var/log/containers
+                    type: string
+                  msgFields:
+                    description: MsgField defines fields that may contain the _msg
+                      field
+                    items:
+                      type: string
+                    type: array
+                  tenantID:
+                    description: 'TenantID defines default tenant ID to use for logs
+                      collected from pods in format: <accountID>:<projectID>'
+                    type: string
+                  timeFields:
+                    description: TimeFields defines fields that may contain the _time
+                      field
+                    items:
+                      type: string
+                    type: array
+                type: object
               livenessProbe:
                 description: LivenessProbe that will be added CRD pod
                 type: object

config/crd/overlay/crd.yaml

@@ -0,0 +1,34 @@
+apiVersion: operator.victoriametrics.com/v1
+kind: VLAgent
+metadata:
+  name: example
+spec:
+  resources:
+    requests:
+      cpu: "50m"
+      memory: "350Mi"
+    limits:
+      cpu: "500m"
+      memory: "850Mi"
+  k8sCollector:
+    enabled: true
+    extraFields: '{"env":"dev","cluster":"staging"}'
+    msgFields:
+      - msg
+      - message
+      - log.msg
+    timeFields:
+      - time
+      - ts
+      - timestamp
+  remoteWrite:
+    - url: "http://vlsingle-example-0.default.svc:9428/internal/insert"
+  remoteWriteSettings:
+    tmpDataPath: /opt/queue
+  volumes:
+    - name: persistent-queue
+      hostPath:
+        path: /opt/mnt/vmagent-queue
+  volumeMounts:
+    - mountPath: /opt/queue
+      name: persistent-queue

config/examples/vlagent-collector.yaml

@@ -14,6 +14,7 @@ aliases:
 ## tip
 
 * FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): support `namespace` parameter in `attach_metadata` section for all scrape configurations. See [#1654](https://github.com/VictoriaMetrics/operator/issues/1654).
+* FEATURE: [vlagent](https://docs.victoriametrics.com/operator/resources/vlagent): support logs collection. See [#1501](https://github.com/VictoriaMetrics/operator/issues/1501).
 
 ## [v0.66.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.66.1)
 

docs/CHANGELOG.md

@@ -170,6 +170,27 @@ VLAgent - is a tiny but brave agent, which helps you collect logs from various s
 | spec<a href="#vlagent-spec" id="vlagent-spec">#</a><br/>_[VLAgentSpec](#vlagentspec)_ | _(Required)_<br/> |
 
 
+#### VLAgentK8sCollector
+
+
+
+
+
+Appears in: [VLAgentSpec](#vlagentspec)
+
+| Field | Description |
+| --- | --- |
+| checkpointsPath<a href="#vlagentk8scollector-checkpointspath" id="vlagentk8scollector-checkpointspath">#</a><br/>_string_ | _(Required)_<br/>CheckpointsPath configures path to file where logs checkpoints are stored.<br />By default it's stored at host's /var/lib/vlagent_checkpoints/checkpoints.json. |
+| decolorizeFields<a href="#vlagentk8scollector-decolorizefields" id="vlagentk8scollector-decolorizefields">#</a><br/>_string array_ | _(Required)_<br/>DecolorizeFields defines fields to remove ANSI color codes across logs ingested from Kubernetes |
+| enabled<a href="#vlagentk8scollector-enabled" id="vlagentk8scollector-enabled">#</a><br/>_boolean_ | _(Required)_<br/>Enabled switches VLAgent to log collection mode.<br />Note, for this purpose operator uses DaemonSet, while by default VLAgent uses StatefulSet.<br />Switching this option will drop all persisted data. |
+| extraFields<a href="#vlagentk8scollector-extrafields" id="vlagentk8scollector-extrafields">#</a><br/>_string_ | _(Required)_<br/>ExtraFields defines extra fields as JSON string which should be added to each collected log line<br />Example: '\{"env":"dev","cluster":"staging"\}' |
+| ignoreFields<a href="#vlagentk8scollector-ignorefields" id="vlagentk8scollector-ignorefields">#</a><br/>_string array_ | _(Required)_<br/>IgnoreFields defines fields to ignore across logs ingested from Kubernetes |
+| logsPath<a href="#vlagentk8scollector-logspath" id="vlagentk8scollector-logspath">#</a><br/>_string_ | _(Required)_<br/>LogsPath configures root for logs path<br />By default VLAgent collects logs from /var/log/containers |
+| msgFields<a href="#vlagentk8scollector-msgfields" id="vlagentk8scollector-msgfields">#</a><br/>_string array_ | _(Required)_<br/>MsgField defines fields that may contain the _msg field |
+| tenantID<a href="#vlagentk8scollector-tenantid" id="vlagentk8scollector-tenantid">#</a><br/>_string_ | _(Required)_<br/>TenantID defines default tenant ID to use for logs collected from pods in format: <accountID>:<projectID> |
+| timeFields<a href="#vlagentk8scollector-timefields" id="vlagentk8scollector-timefields">#</a><br/>_string array_ | _(Required)_<br/>TimeFields defines fields that may contain the _time field |
+
+
 #### VLAgentRemoteWriteSettings
 
 
@@ -236,6 +257,7 @@ Appears in: [VLAgent](#vlagent)
 | image<a href="#vlagentspec-image" id="vlagentspec-image">#</a><br/>_[Image](#image)_ | _(Optional)_<br/>Image - docker image settings<br />if no specified operator uses default version from operator config |
 | imagePullSecrets<a href="#vlagentspec-imagepullsecrets" id="vlagentspec-imagepullsecrets">#</a><br/>_[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core) array_ | _(Optional)_<br/>ImagePullSecrets An optional list of references to secrets in the same namespace<br />to use for pulling images from registries<br />see https://kubernetes.io/docs/concepts/containers/images/#referring-to-an-imagepullsecrets-on-a-pod |
 | initContainers<a href="#vlagentspec-initcontainers" id="vlagentspec-initcontainers">#</a><br/>_[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#container-v1-core) array_ | _(Optional)_<br/>InitContainers allows adding initContainers to the pod definition.<br />Any errors during the execution of an initContainer will lead to a restart of the Pod.<br />More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ |
+| k8sCollector<a href="#vlagentspec-k8scollector" id="vlagentspec-k8scollector">#</a><br/>_[VLAgentK8sCollector](#vlagentk8scollector)_ | _(Required)_<br/>K8sCollector configures VLAgent logs collection from K8s pods |
 | logFormat<a href="#vlagentspec-logformat" id="vlagentspec-logformat">#</a><br/>_string_ | _(Optional)_<br/>LogFormat for VLAgent to be configured with. |
 | logLevel<a href="#vlagentspec-loglevel" id="vlagentspec-loglevel">#</a><br/>_string_ | _(Optional)_<br/>LogLevel for VLAgent to be configured with.<br />INFO, WARN, ERROR, FATAL, PANIC |
 | managedMetadata<a href="#vlagentspec-managedmetadata" id="vlagentspec-managedmetadata">#</a><br/>_[ManagedObjectsMetadata](#managedobjectsmetadata)_ | _(Required)_<br/>ManagedMetadata defines metadata that will be added to the all objects<br />created by operator for the given CustomResource |

docs/api.md

@@ -1,7 +1,7 @@
 | Environment variables |
 | --- |
 | VM_METRICS_VERSION: `v1.131.0` <a href="#variables-vm-metrics-version" id="variables-vm-metrics-version">#</a> |
-| VM_LOGS_VERSION: `v1.36.1` <a href="#variables-vm-logs-version" id="variables-vm-logs-version">#</a> |
+| VM_LOGS_VERSION: `v1.40.0` <a href="#variables-vm-logs-version" id="variables-vm-logs-version">#</a> |
 | VM_ANOMALY_VERSION: `v1.26.1` <a href="#variables-vm-anomaly-version" id="variables-vm-anomaly-version">#</a> |
 | VM_TRACES_VERSION: `v0.5.0` <a href="#variables-vm-traces-version" id="variables-vm-traces-version">#</a> |
 | VM_OPERATOR_VERSION: `v0.65.0` <a href="#variables-vm-operator-version" id="variables-vm-operator-version">#</a> |

docs/env.md

@@ -136,6 +136,14 @@ then `VLAgent` pods will be created without resource requests and limits.
 
 Also, you can specify requests without limits - in this case default values for limits will not be used.
 
+## K8s logs collection
+
+VLAgent supports collecting logs from K8s pods. To enable it it's required to set `spec.k8sCollector.enabled: true`.
+With this setting operator:
+ - switches VLAgent to DaemonSet mode
+ - mounts host paths, where k8s logs are stored
+ - adds RBAC to be able to get pods, namespaces and nodes information using K8s API
+
 ## Storage management
 
 In case of errors sending logs to `remoteWrite`,