Skip to content

[gcp.audit] Ingest pipeline fails to parse plain-text upgrade events #17591

@s-feinstein

Description

@s-feinstein

Integration Name

Google Cloud Provider [gcp]

Dataset Name

gcp.audit

Integration Version

8

Agent Version

9.3.0

Agent Output Type

elasticsearch

Elasticsearch Version

9.3.0

OS Version and Architecture

ECH

Software/API Version

No response

Error Message

Variations of system upgrade logs

Unrecognized token 'Node': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')
 at [Source: (String)"Node pool projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80 is upgrading to version 1.33.5-gke.2326000."; line: 1, column: 5]

or

Unrecognized token 'Master': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')
 at [Source: (String)"Master is upgrading to version 1.33.5-gke.2326000."; line: 1, column: 7]

Event Original

"Node pool projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80 is upgrading to version 1.33.5-gke.2326000."
or
"Master is upgrading to version 1.33.5-gke.2326000."

What did you do?

We have not customized the integration, using it out of the box

What did you see?

{
  "_index": [Redacted],
  "_id": [Redacted],
  "_version": 1,
  "_source": {
    "@timestamp": "2026-02-26T13:42:05.288Z",
    "_conf": {
      "keep_json": false
    },
    "agent": {
      "ephemeral_id": [Redacted],
      "id": [Redacted],
      "name": "elastic-agent-gcp-infosec-auditlogs-agent-946bbb55d-hmrcz",
      "type": "filebeat",
      "version": "9.3.0"
    },
    "cloud": {
      "account": {
        "id": "elastic-infosec"
      },
      "availability_zone": "us-central1-a",
      "instance": {
        "id": [Redacted],
        "name": "gke-infosec-microser-n2-standard-16-i-1de61d56-m45p"
      },
      "project": {
        "id": "elastic-infosec"
      },
      "provider": "gcp",
      "region": "us-central1",
      "service": {
        "name": "GCE"
      }
    },
    "data_stream": {
      "dataset": "gcp.audit",
      "namespace": "infosec",
      "type": "logs"
    },
    "ecs": {
      "version": "8.11.0"
    },
    "elastic_agent": {
      "id": [Redacted],
      "snapshot": false,
      "version": "9.3.0"
    },
    "error": {
      "message": [
        "Unrecognized token 'Node': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)\"Node pool projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80 is upgrading to version 1.33.5-gke.2326000.\"; line: 1, column: 5]"
      ]
    },
    "event": {
      "agent_id_status": "auth_metadata_missing",
      "created": "2026-02-26T13:42:05.289Z",
      "dataset": "gcp.audit",
      "id": [Redacted],
      "ingested": "2026-02-26T13:42:08Z",
      "kind": "pipeline_error",
      "original": "Node pool projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80 is upgrading to version 1.33.5-gke.2326000."
    },
    "input": {
      "type": "gcp-pubsub"
    },
    "labels": {
      "cluster_location": "us-central1-a",
      "cluster_name": "white-sector-qa",
      "payload": "{\"resourceType\":\"NODE_POOL\",\"operation\":\"operation-[Redacted]\",\"operationStartTime\":\"2026-02-26T13:41:59.219008777Z\",\"currentVersion\":\"1.33.5-gke.2228001\",\"targetVersion\":\"1.33.5-gke.2326000\",\"resource\":\"projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80\"}",
      "project_id": [Redacted],
      "type_url": "type.googleapis.com/google.container.v1beta1.UpgradeEvent"
    },
    "orchestrator": {
      "cluster": {
        "name": "infosec-microservices"
      }
    },
    "tags": [
      "preserve_original_event",
      "forwarded",
      "gcp-audit"
    ]
  },
  "fields": {
    "orchestrator.cluster.name": [
      "infosec-microservices"
    ],
    "elastic_agent.version": [
      "9.3.0"
    ],
    "_conf.keep_json": [
      false
    ],
    "cloud.availability_zone": [
      "us-central1-a"
    ],
    "cloud.instance.id": [
      [Redacted]
    ],
    "agent.type": [
      "filebeat"
    ],
    "event.module": [
      "gcp"
    ],
    "agent.name.text": [
      "elastic-agent-gcp-infosec-auditlogs-agent-946bbb55d-hmrcz"
    ],
    "cloud.service.name.text": [
      "GCE"
    ],
    "agent.name": [
      "elastic-agent-gcp-infosec-auditlogs-agent-946bbb55d-hmrcz"
    ],
    "elastic_agent.snapshot": [
      false
    ],
    "event.agent_id_status": [
      "auth_metadata_missing"
    ],
    "event.kind": [
      "pipeline_error"
    ],
    "labels.payload": [
      "{\"resourceType\":\"NODE_POOL\",\"operation\":\"operation-[Redacted]\",\"operationStartTime\":\"2026-02-26T13:41:59.219008777Z\",\"currentVersion\":\"1.33.5-gke.2228001\",\"targetVersion\":\"1.33.5-gke.2326000\",\"resource\":\"projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80\"}"
    ],
    "event.original": [
      "Node pool projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80 is upgrading to version 1.33.5-gke.2326000."
    ],
    "cloud.region": [
      "us-central1"
    ],
    "cloud.instance.name.text": [
      "gke-infosec-microser-n2-standard-16-i-1de61d56-m45p"
    ],
    "elastic_agent.id": [
      [Redacted]
    ],
    "data_stream.namespace": [
      "infosec"
    ],
    "input.type": [
      "gcp-pubsub"
    ],
    "labels.type_url": [
      "type.googleapis.com/google.container.v1beta1.UpgradeEvent"
    ],
    "data_stream.type": [
      "logs"
    ],
    "labels.cluster_location": [
      "us-central1-a"
    ],
    "tags": [
      "preserve_original_event",
      "forwarded",
      "gcp-audit"
    ],
    "cloud.provider": [
      "gcp"
    ],
    "event.ingested": [
      "2026-02-26T13:42:08.000Z"
    ],
    "orchestrator.cluster.name.text": [
      "infosec-microservices"
    ],
    "@timestamp": [
      "2026-02-26T13:42:05.288Z"
    ],
    "agent.id": [
      [Redacted]
    ],
    "cloud.service.name": [
      "GCE"
    ],
    "cloud.account.id": [
      "elastic-infosec"
    ],
    "ecs.version": [
      "8.11.0"
    ],
    "labels.cluster_name": [
      "white-sector-qa"
    ],
    "error.message": [
      "Unrecognized token 'Node': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)\"Node pool projects/elastic-infosec/locations/us-central1-a/clusters/white-sector-qa/nodePools/n2-custom-20-80 is upgrading to version 1.33.5-gke.2326000.\"; line: 1, column: 5]"
    ],
    "data_stream.dataset": [
      "gcp.audit"
    ],
    "event.created": [
      "2026-02-26T13:42:05.289Z"
    ],
    "agent.ephemeral_id": [
      [Redacted]
    ],
    "agent.version": [
      "9.3.0"
    ],
    "event.id": [
      [Redacted]
    ],
    "event.dataset": [
      "gcp.audit"
    ],
    "labels.project_id": [
      [Redacted]
    ],
    "cloud.instance.name": [
      "gke-infosec-microser-n2-standard-16-i-1de61d56-m45p"
    ],
    "cloud.project.id": [
      "elastic-infosec"
    ]
  }
}

What did you expect to see?

Handling for when logs are plain-text / not formatted in json, particularly logs of type type.googleapis.com/google.container.v1beta1.UpgradeEvent

Anything else?

Linking error investigation

Metadata

Metadata

Labels

Integration:gcpGoogle Cloud PlatformTeam:SDE-CrestCrest developers on the Security Integrations team [elastic/sit-crest-contractors]Team:Security-Service IntegrationsSecurity Service Integrations team [elastic/security-service-integrations]bugSomething isn't working, use only for issues

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions