diff --git a/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml b/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml index d977fb3d41..d4fefccfc9 100644 --- a/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml +++ b/config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml @@ -985,6 +985,12 @@ spec: type: string mongosParams: type: string + querySource: + default: profiler + enum: + - profiler + - mongolog + type: string resources: properties: claims: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 196bdfe501..877cfcb96d 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -1939,6 +1939,12 @@ spec: type: string mongosParams: type: string + querySource: + default: profiler + enum: + - profiler + - mongolog + type: string resources: properties: claims: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index a84379fa7f..65ed5ef938 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -83,6 +83,7 @@ spec: # customClusterName: mongo-cluster # mongodParams: --environment=ENVIRONMENT # mongosParams: --environment=ENVIRONMENT +# querySource: profiler # resources: # requests: # memory: 150M diff --git a/deploy/crd.yaml b/deploy/crd.yaml index f02c1e7087..8b3eb45a30 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -1939,6 +1939,12 @@ spec: type: string mongosParams: type: string + querySource: + default: profiler + enum: + - profiler + - mongolog + type: string resources: properties: claims: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 734652fe32..abfb3227a2 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -1939,6 +1939,12 @@ spec: type: string mongosParams: type: string + querySource: + default: profiler + enum: + - profiler + - mongolog + type: string resources: properties: claims: diff --git a/e2e-tests/functions b/e2e-tests/functions index 85b3f74249..429158af7a 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -17,6 +17,7 @@ IMAGE_PMM_CLIENT=${IMAGE_PMM_CLIENT:-"percona/pmm-client:2.44.1-1"} IMAGE_PMM_SERVER=${IMAGE_PMM_SERVER:-"perconalab/pmm-server:dev-latest"} IMAGE_PMM3_CLIENT=${IMAGE_PMM3_CLIENT:-"perconalab/pmm-client:3-dev-latest"} IMAGE_PMM3_SERVER=${IMAGE_PMM3_SERVER:-"perconalab/pmm-server:3-dev-latest"} +IMAGE_LOGCOLLECTOR=${IMAGE_LOGCOLLECTOR:-"perconalab/fluentbit:main-logcollector"} REGISTRY_NAME=${REGISTRY_NAME:-docker.io} REGISTRY_NAME_FULL="${REGISTRY_NAME%/}/" diff --git a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg-oc.yml b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg-oc.yml index ad23d880a1..96c57a6678 100644 --- a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg-oc.yml +++ b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg-oc.yml @@ -236,7 +236,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' - name: PMM_AGENT_SIDECAR value: "true" diff --git a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml index 52f3cca908..4b8629a2f0 100644 --- a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml +++ b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml @@ -237,7 +237,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' - name: PMM_AGENT_SIDECAR value: "true" diff --git a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos-oc.yml b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos-oc.yml index 5c815cdbd5..0994b683da 100644 --- a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos-oc.yml +++ b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos-oc.yml @@ -229,7 +229,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' - name: PMM_AGENT_SIDECAR value: "true" diff --git a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml index 68a8b67603..78e8e68075 100644 --- a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml +++ b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml @@ -230,7 +230,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' - name: PMM_AGENT_SIDECAR value: "true" diff --git a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-oc.yml b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-oc.yml index fd409c4013..717c2f9f97 100644 --- a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-oc.yml +++ b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-oc.yml @@ -224,7 +224,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' - name: PMM_AGENT_SIDECAR value: "true" diff --git a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml index ede8b571f5..adb323199e 100644 --- a/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml +++ b/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml @@ -225,7 +225,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' - name: PMM_AGENT_SIDECAR value: "true" diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg-oc.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg-oc.yml index 3b1b7ad45b..45cfbe996d 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg-oc.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg-oc.yml @@ -78,6 +78,8 @@ spec: value: "27019" - name: MONGODB_REPLSET value: cfg + - name: LOGCOLLECTOR_ENABLED + value: "true" envFrom: - secretRef: name: internal-monitoring-pmm3-users @@ -229,7 +231,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' imagePullPolicy: Always lifecycle: @@ -275,6 +277,64 @@ spec: - mountPath: /data/db name: mongod-data readOnly: true + - args: + - fluent-bit + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: LOG_DATA_DIR + value: /data/db/logs + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + name: logs + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin + - args: + - logrotate + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: MONGODB_HOST + value: localhost + - name: MONGODB_PORT + value: "27019" + - name: MONGODB_USER + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_USER_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_PASSWORD_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + imagePullPolicy: IfNotPresent + name: logrotate + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin dnsPolicy: ClusterFirst initContainers: - command: diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml index 0a0ec02c9e..0d4370b3ff 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml @@ -78,6 +78,8 @@ spec: value: "27019" - name: MONGODB_REPLSET value: cfg + - name: LOGCOLLECTOR_ENABLED + value: "true" envFrom: - secretRef: name: internal-monitoring-pmm3-users @@ -230,7 +232,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' imagePullPolicy: Always lifecycle: @@ -276,6 +278,64 @@ spec: - mountPath: /data/db name: mongod-data readOnly: true + - args: + - fluent-bit + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: LOG_DATA_DIR + value: /data/db/logs + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + name: logs + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin + - args: + - logrotate + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: MONGODB_HOST + value: localhost + - name: MONGODB_PORT + value: "27019" + - name: MONGODB_USER + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_USER_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_PASSWORD_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + imagePullPolicy: IfNotPresent + name: logrotate + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin dnsPolicy: ClusterFirst initContainers: - command: diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos-oc.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos-oc.yml index fb4e0f19b6..64c44c4f2b 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos-oc.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos-oc.yml @@ -222,7 +222,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' imagePullPolicy: Always lifecycle: diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml index 34beec4a3e..6b3f333781 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml @@ -223,7 +223,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' imagePullPolicy: Always lifecycle: diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm-oc.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm-oc.yml index 650edf4ba9..ef62b5c323 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm-oc.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm-oc.yml @@ -66,6 +66,8 @@ spec: value: "27019" - name: MONGODB_REPLSET value: rs0 + - name: LOGCOLLECTOR_ENABLED + value: "true" envFrom: - secretRef: name: internal-monitoring-pmm3-users @@ -143,6 +145,64 @@ spec: - mountPath: /etc/users-secret name: users-secret-file workingDir: /data/db + - args: + - fluent-bit + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: LOG_DATA_DIR + value: /data/db/logs + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + name: logs + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin + - args: + - logrotate + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: MONGODB_HOST + value: localhost + - name: MONGODB_PORT + value: "27019" + - name: MONGODB_USER + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_USER_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_PASSWORD_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + imagePullPolicy: IfNotPresent + name: logrotate + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin dnsPolicy: ClusterFirst initContainers: - command: diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml index 0acae823d1..de7f3caa82 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml @@ -66,6 +66,8 @@ spec: value: "27019" - name: MONGODB_REPLSET value: rs0 + - name: LOGCOLLECTOR_ENABLED + value: "true" envFrom: - secretRef: name: internal-monitoring-pmm3-users @@ -144,6 +146,64 @@ spec: - mountPath: /etc/users-secret name: users-secret-file workingDir: /data/db + - args: + - fluent-bit + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: LOG_DATA_DIR + value: /data/db/logs + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + name: logs + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin + - args: + - logrotate + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: MONGODB_HOST + value: localhost + - name: MONGODB_PORT + value: "27019" + - name: MONGODB_USER + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_USER_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_PASSWORD_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + imagePullPolicy: IfNotPresent + name: logrotate + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin dnsPolicy: ClusterFirst initContainers: - command: diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-oc.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-oc.yml index 3e5f3bad7e..d99e24a4e6 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-oc.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-oc.yml @@ -66,6 +66,8 @@ spec: value: "27019" - name: MONGODB_REPLSET value: rs0 + - name: LOGCOLLECTOR_ENABLED + value: "true" envFrom: - secretRef: name: internal-monitoring-pmm3-users @@ -217,7 +219,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' imagePullPolicy: Always lifecycle: @@ -263,6 +265,64 @@ spec: - mountPath: /data/db name: mongod-data readOnly: true + - args: + - fluent-bit + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: LOG_DATA_DIR + value: /data/db/logs + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + name: logs + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin + - args: + - logrotate + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: MONGODB_HOST + value: localhost + - name: MONGODB_PORT + value: "27019" + - name: MONGODB_USER + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_USER_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_PASSWORD_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + imagePullPolicy: IfNotPresent + name: logrotate + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin dnsPolicy: ClusterFirst initContainers: - command: diff --git a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml index 320938ebd8..4daf7aae34 100644 --- a/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml +++ b/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml @@ -66,6 +66,8 @@ spec: value: "27019" - name: MONGODB_REPLSET value: rs0 + - name: LOGCOLLECTOR_ENABLED + value: "true" envFrom: - secretRef: name: internal-monitoring-pmm3-users @@ -218,7 +220,7 @@ spec: value: |- cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem; pmm-admin status --wait=10s; - pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; + pmm-admin add $(DB_TYPE) $(PMM_ADMIN_CUSTOM_PARAMS) --skip-connection-check --metrics-mode=push --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) --service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT) --query-source=profiler --tls --tls-skip-verify --tls-certificate-key-file=/tmp/tls.pem --tls-ca-file=/etc/mongodb-ssl/ca.crt --authentication-mechanism=SCRAM-SHA-256 --authentication-database=admin; pmm-admin annotate --service-name=$(PMM_AGENT_SETUP_NODE_NAME) 'Service restarted' imagePullPolicy: Always lifecycle: @@ -264,6 +266,64 @@ spec: - mountPath: /data/db name: mongod-data readOnly: true + - args: + - fluent-bit + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: LOG_DATA_DIR + value: /data/db/logs + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + name: logs + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin + - args: + - logrotate + command: + - /opt/percona/logcollector/entrypoint.sh + env: + - name: MONGODB_HOST + value: localhost + - name: MONGODB_PORT + value: "27019" + - name: MONGODB_USER + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_USER_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + - name: MONGODB_PASSWORD + valueFrom: + secretKeyRef: + key: MONGODB_CLUSTER_ADMIN_PASSWORD_ESCAPED + name: internal-monitoring-pmm3-users + optional: false + imagePullPolicy: IfNotPresent + name: logrotate + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /data/db + name: mongod-data + - mountPath: /opt/percona + name: bin dnsPolicy: ClusterFirst initContainers: - command: diff --git a/e2e-tests/monitoring-pmm3/conf/monitoring-pmm3-rs0.yml b/e2e-tests/monitoring-pmm3/conf/monitoring-pmm3-rs0.yml index 5997707bf9..e2e3780174 100644 --- a/e2e-tests/monitoring-pmm3/conf/monitoring-pmm3-rs0.yml +++ b/e2e-tests/monitoring-pmm3/conf/monitoring-pmm3-rs0.yml @@ -63,6 +63,10 @@ spec: annotations: test: monitoring-pmm3 + logcollector: + enabled: true + image: + pmm: enabled: true image: diff --git a/e2e-tests/monitoring-pmm3/run b/e2e-tests/monitoring-pmm3/run index 5de242fb88..dc25581411 100755 --- a/e2e-tests/monitoring-pmm3/run +++ b/e2e-tests/monitoring-pmm3/run @@ -350,6 +350,7 @@ custom_cluster_name="super-custom" yq eval '(.spec | select(.image == null)).image = "'"$IMAGE_MONGOD"'"' "$test_dir/conf/$cluster-rs0.yml" \ | yq eval '(.spec | select(has("pmm"))).pmm.image = "'"$IMAGE_PMM3_CLIENT"'"' - \ | yq eval '(.spec | select(has("pmm"))).pmm.customClusterName = "'"$custom_cluster_name"'"' - \ + | yq eval '(.spec | select(has("logcollector"))).logcollector.image = "'"$IMAGE_LOGCOLLECTOR"'"' - \ | yq eval '(.spec | select(has("initImage"))).initImage = "'"$IMAGE"'"' - \ | yq eval '(.spec | select(has("backup"))).backup.image = "'"$IMAGE_BACKUP"'"' - \ | yq eval '.spec.upgradeOptions.apply = "Never"' - \ @@ -442,6 +443,19 @@ for node_id in "${nodeList_from_pmm[@]}"; do fi done +desc 'patch cluster to switch QAN source to mongolog' +kubectl_bin patch psmdb $cluster --type=json -p='[ + {"op":"add","path":"/spec/pmm/querySource","value":"mongolog"}, + {"op":"replace","path":"/spec/pmm/mongodParams","value":"--enable-all-collectors --environment=dev-mongod-mongolog"} +]' + +wait_for_running $cluster-rs0 3 +# wait for QAN data to be collected and pushed from mongolog +sleep 90 + +desc 'check mongolog QAN data' +get_qan_values mongodb "dev-mongod-mongolog" $NEW_TOKEN + kubectl_bin patch psmdb ${cluster} --type json -p='[{"op":"add","path":"/spec/pause","value":true}]' wait_for_delete "pod/${cluster}-mongos-0" wait_for_delete "pod/${cluster}-rs0-0" diff --git a/e2e-tests/version-service/conf/crd.yaml b/e2e-tests/version-service/conf/crd.yaml index f02c1e7087..8b3eb45a30 100644 --- a/e2e-tests/version-service/conf/crd.yaml +++ b/e2e-tests/version-service/conf/crd.yaml @@ -1939,6 +1939,12 @@ spec: type: string mongosParams: type: string + querySource: + default: profiler + enum: + - profiler + - mongolog + type: string resources: properties: claims: diff --git a/pkg/apis/psmdb/v1/psmdb_types.go b/pkg/apis/psmdb/v1/psmdb_types.go index bb34d066ef..00bb34ddfc 100644 --- a/pkg/apis/psmdb/v1/psmdb_types.go +++ b/pkg/apis/psmdb/v1/psmdb_types.go @@ -459,6 +459,14 @@ type PMMSpec struct { // authenticate the clusterMonitor user against mongod/mongos. // +kubebuilder:validation:Enum=SCRAM-SHA-256;SCRAM-SHA-1 AuthenticationMechanism string `json:"authenticationMechanism,omitempty"` + + // QuerySource defines the source for Query Analytics (QAN) data collection. + // Use "profiler" to collect queries via the MongoDB profiler (default), + // or "mongolog" to collect queries from mongod log files (requires PMM >= 3.3.0 + // and mongod configured to write logs to /data/db/logs/). + // +kubebuilder:validation:Enum=profiler;mongolog + // +kubebuilder:default=profiler + QuerySource string `json:"querySource,omitempty"` } // HasSecret is used for PMM2. PMM2 is reaching its EOL. diff --git a/pkg/psmdb/pmm/pmm.go b/pkg/psmdb/pmm/pmm.go index 15a6bc4a58..20b85ce80a 100644 --- a/pkg/psmdb/pmm/pmm.go +++ b/pkg/psmdb/pmm/pmm.go @@ -280,6 +280,10 @@ func pmmAgentScript(cr *api.PerconaServerMongoDB) []corev1.EnvVar { pmmServerArgs += " --username=$(DB_USER) --password=$(DB_PASSWORD) --cluster=$(CLUSTER_NAME) " pmmServerArgs += "--service-name=$(PMM_AGENT_SETUP_NODE_NAME) --host=$(DB_HOST) --port=$(DB_PORT)" + if cr.Spec.PMM.QuerySource != "" { + pmmServerArgs += " --query-source=" + cr.Spec.PMM.QuerySource + } + if cr.TLSEnabled() { authMechanism := scramSHA256AuthMechanism switch { diff --git a/pkg/psmdb/pmm/pmm_test.go b/pkg/psmdb/pmm/pmm_test.go index a181108223..12d6018884 100644 --- a/pkg/psmdb/pmm/pmm_test.go +++ b/pkg/psmdb/pmm/pmm_test.go @@ -89,6 +89,24 @@ func TestContainer(t *testing.T) { }, assert: assertNoAuthMechanism(), }, + "pmm enabled - query-source=mongolog is set": { + secret: tokenSecret, + setup: func(cr *api.PerconaServerMongoDB) { + cr.Spec.PMM.QuerySource = "mongolog" + }, + assert: assertQuerySource("mongolog"), + }, + "pmm enabled - query-source=profiler is set": { + secret: tokenSecret, + setup: func(cr *api.PerconaServerMongoDB) { + cr.Spec.PMM.QuerySource = "profiler" + }, + assert: assertQuerySource("profiler"), + }, + "pmm enabled - query-source not set omits flag": { + secret: tokenSecret, + assert: assertNoQuerySource(), + }, } for name, tt := range tests { @@ -152,6 +170,38 @@ func assertAuthMechanism(want string) func(t *testing.T, container *corev1.Conta } } +func assertQuerySource(want string) func(t *testing.T, container *corev1.Container) { + return func(t *testing.T, container *corev1.Container) { + if !assert.NotNil(t, container) { + return + } + var prerun string + for _, ev := range container.Env { + if ev.Name == "PMM_AGENT_PRERUN_SCRIPT" { + prerun = ev.Value + break + } + } + assert.Contains(t, prerun, "--query-source="+want) + } +} + +func assertNoQuerySource() func(t *testing.T, container *corev1.Container) { + return func(t *testing.T, container *corev1.Container) { + if !assert.NotNil(t, container) { + return + } + var prerun string + for _, ev := range container.Env { + if ev.Name == "PMM_AGENT_PRERUN_SCRIPT" { + prerun = ev.Value + break + } + } + assert.NotContains(t, prerun, "--query-source") + } +} + func assertNoAuthMechanism() func(t *testing.T, container *corev1.Container) { return func(t *testing.T, container *corev1.Container) { if !assert.NotNil(t, container) {