replaced dataPath with checkpointsPath

Author: AndrewChubatiuk

api/operator/v1/vlagent_types.go

@@ -95,10 +95,9 @@ type VLAgentK8sCollector struct {
 	// By default VLAgent collects logs from /var/log/containers
 	LogsPath string `json:"logsPath,omitempty"`
 
-	// DataPath configures path where logs checkpoints are stored.
-	// By default it emptyDir is used as a volume for data path.
-	// To guarantee checkpoints persistence during pods recreation consider explicitly setting this option to destination, where hostPath volume is mounted.
-	DataPath string `json:"dataPath,omitempty"`
+	// CheckpointsPath configures path to file where logs checkpoints are stored.
+	// By default it's stored at host's /var/lib/vlagent_checkpoints/checkpoints.json.
+	CheckpointsPath string `json:"checkpointsPath,omitempty"`
 
 	// TenantID defines default tenant ID to use for logs collected from pods in format: <accountID>:<projectID>
 	TenantID string `json:"tenantID,omitempty"`

config/crd/overlay/crd.yaml

@@ -740,11 +740,10 @@ spec:
                 description: K8sCollector configures VLAgent logs collection from
                   K8s pods
                 properties:
-                  dataPath:
+                  checkpointsPath:
                     description: |-
-                      DataPath configures path where logs checkpoints are stored.
-                      By default it emptyDir is used as a volume for data path.
-                      To guarantee checkpoints persistence during pods recreation consider explicitly setting this option to destination, where hostPath volume is mounted.
+                      CheckpointsPath configures path to file where logs checkpoints are stored.
+                      By default it's stored at host's /var/lib/vlagent_checkpoints/checkpoints.json.
                     type: string
                   decolorizeFields:
                     description: DecolorizeFields defines fields to remove ANSI color

config/examples/vlagent-full-collector.yaml

@@ -12,14 +12,14 @@ spec:
       memory: "850Mi"
   k8sCollector:
     enabled: true
-    dataPath: /var/lib/vl-collector
     extraFields: '{"env":"dev","cluster":"staging"}'
-  volumes:
-    - name: data
-      hostPath:
-        path: /var/lib/vl-collector
-  volumeMounts:
-    - name: data
-      mountPath: /var/lib/vl-collector
+    msgFields:
+      - msg
+      - message
+      - log.msg
+    timeFields:
+      - time
+      - ts
+      - timestamp
   remoteWrite:
     - url: "http://vlsingle-example-0.default.svc:9428/internal/insert"

docs/api.md

@@ -180,7 +180,7 @@ Appears in: [VLAgentSpec](#vlagentspec)
 
 | Field | Description |
 | --- | --- |
-| dataPath<a href="#vlagentk8scollector-datapath" id="vlagentk8scollector-datapath">#</a><br/>_string_ | _(Required)_<br/>DataPath configures path where logs checkpoints are stored.<br />By default it emptyDir is used as a volume for data path.<br />To guarantee checkpoints persistence during pods recreation consider explicitly setting this option to destination, where hostPath volume is mounted. |
+| checkpointsPath<a href="#vlagentk8scollector-checkpointspath" id="vlagentk8scollector-checkpointspath">#</a><br/>_string_ | _(Required)_<br/>CheckpointsPath configures path to file where logs checkpoints are stored.<br />By default it's stored at host's /var/lib/vlagent_checkpoints/checkpoints.json. |
 | decolorizeFields<a href="#vlagentk8scollector-decolorizefields" id="vlagentk8scollector-decolorizefields">#</a><br/>_string array_ | _(Required)_<br/>DecolorizeFields defines fields to remove ANSI color codes across logs ingested from Kubernetes |
 | enabled<a href="#vlagentk8scollector-enabled" id="vlagentk8scollector-enabled">#</a><br/>_boolean_ | _(Required)_<br/>Enabled switches VLAgent to log collection mode.<br />Note, for this purpose operator uses DaemonSet, while by default VLAgent uses StatefulSet.<br />Switching this option will drop all persisted data. |
 | extraFields<a href="#vlagentk8scollector-extrafields" id="vlagentk8scollector-extrafields">#</a><br/>_string_ | _(Required)_<br/>ExtraFields defines extra fields as JSON string which should be added to each collected log line<br />Example: '\{"env":"dev","cluster":"staging"\}' |

internal/controller/operator/factory/vlagent/vlagent.go

@@ -31,9 +31,9 @@ const (
 	persistentQueueSTSDir    = "/vlagent_pq/vlagent-remotewrite-data"
 	persistentQueueMountName = "persistent-queue-data"
 
-	defaultLogsPath = "/var/log/containers"
-	defaultDataPath = "/vlagent"
-	dataVolumeName  = "data"
+	defaultLogsPath        = "/var/log/containers"
+	defaultCheckpointsPath = "/var/lib/vlagent_checkpoints"
+	checkpointsVolumeName  = "checkpoints"
 
 	remoteWriteAssetsMounthPath = "/etc/vl/remote-write-assets"
 	tlsServerConfigMountPath    = "/etc/vl/tls-server-secrets"
@@ -321,22 +321,24 @@ func newPodSpec(cr *vmv1.VLAgent) (*corev1.PodSpec, error) {
 		} else {
 			args = append(args, fmt.Sprintf("-kubernetesCollector.logsPath=%s", cr.Spec.K8sCollector.LogsPath))
 		}
-		dataPath := defaultDataPath
-		if len(cr.Spec.K8sCollector.DataPath) == 0 || cr.Spec.K8sCollector.DataPath == defaultDataPath {
+		checkpointsPath := defaultCheckpointsPath
+		if len(cr.Spec.K8sCollector.CheckpointsPath) == 0 || cr.Spec.K8sCollector.CheckpointsPath == defaultCheckpointsPath {
 			volumes = append(volumes, corev1.Volume{
-				Name: dataVolumeName,
+				Name: checkpointsVolumeName,
 				VolumeSource: corev1.VolumeSource{
-					EmptyDir: &corev1.EmptyDirVolumeSource{},
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: checkpointsPath,
+					},
 				},
 			})
 			agentVolumeMounts = append(agentVolumeMounts, corev1.VolumeMount{
-				Name:      dataVolumeName,
-				MountPath: dataPath,
+				Name:      checkpointsVolumeName,
+				MountPath: checkpointsPath,
 			})
+			checkpointsPath = path.Join(checkpointsPath, "checkpoints.json")
 		} else {
-			dataPath = cr.Spec.K8sCollector.DataPath
+			checkpointsPath = cr.Spec.K8sCollector.CheckpointsPath
 		}
-		checkpointsPath := path.Join(dataPath, "checkpoints.json")
 		args = append(args, fmt.Sprintf("-kubernetesCollector.checkpointsPath=%s", checkpointsPath))
 
 		if cr.Spec.RemoteWriteSettings == nil || cr.Spec.RemoteWriteSettings.TmpDataPath == nil {

internal/controller/operator/factory/vlagent/vlagent_test.go

@@ -1081,7 +1081,7 @@ containers:
     args:
       - -httpListenAddr=:9425
       - -kubernetesCollector
-      - -kubernetesCollector.checkpointsPath=/vlagent/checkpoints.json
+      - -kubernetesCollector.checkpointsPath=/var/lib/vlagent_checkpoints/checkpoints.json
       - -kubernetesCollector.msgField="msg,message"
       - -remoteWrite.maxDiskUsagePerURL=10GB,10GB,
       - -remoteWrite.tmpDataPath=/vlagent_pq/vlagent-remotewrite-data
@@ -1101,8 +1101,8 @@ containers:
       - name: varlib
         readonly: true
         mountpath: /var/lib
-      - name: data
-        mountpath: /vlagent
+      - name: checkpoints
+        mountpath: /var/lib/vlagent_checkpoints
       - name: persistent-queue-data
         readonly: false
         mountpath: /vlagent_pq/vlagent-remotewrite-data
@@ -1141,9 +1141,10 @@ volumes:
   volumesource:
     hostpath:
       path: /var/lib
-- name: data
+- name: checkpoints
   volumesource:
-    emptydir: {}
+    hostpath:
+      path: /var/lib/vlagent_checkpoints
 - name: persistent-queue-data
   volumesource:
     emptydir: {}