Skip to content

Commit 853777f

Browse files
Linu-EliasLinu-Elias
authored
[AWS] Add cookies field in cloudfront datastream (elastic#16122)
* fix * add log
1 parent a6cca0a commit 853777f

File tree

7 files changed

+109
-2
lines changed

7 files changed

+109
-2
lines changed

‎packages/aws/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "5.3.1"
3+
changes:
4+
- description: Add `cookies` field in cloudfront logs datastream.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/16122
27
- version: "5.3.0"
38
changes:
49
- description: Added rate limiting and retry configuration for AWS Config data stream.

‎packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@
1212
2022-11-15 08:43:04 SEA19-C2 10157 81.2.69.143 GET d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(X11;%20Linux%20x86_64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20HeadlessChrome/100.0.4896.88%20Safari/537.36 - - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.093 81.2.69.142, 216.160.83.56 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Miss HTTP/1.1 - - 33359 0.093 Miss application/javascript - - -
1313
2022-11-15 08:43:04 SEA19-C2 10157 81.2.69.143 GET d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(X11;%20Linux%20x86_64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20HeadlessChrome/100.0.4896.88%20Safari/537.36 - - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.093 localhost:8080 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Miss HTTP/1.1 - - 33359 0.093 Miss application/javascript - - -
1414
2024-07-13 15:29:45 EWR53-C1 198083 127.0.0.1 GET xxxxxxxxxxxxx.cloudfront.net /en(test) 404 https://domain.tld/ User-Agent:%20Mozilla/4.0%20(compatible;%20MSIE%207.0;%20Windows%20NT%205.1;%20360SE) - - Error somevalidbase64== domain.tld https 609 0.318 - TLSv1.3 TLS_AES_128_GCM_SHA256 Error HTTP/1.1 - - 50294 0.318 Error text/html -
15+
2024-07-13 15:29:45 EWR53-C1 198083 127.0.0.1 GET xxxxxxxxxxxxx.cloudfront.net /en(test) 404 https://domain.tld/ User-Agent:%20Mozilla/4.0%20(compatible;%20MSIE%207.0;%20Windows%20NT%205.1;%20360SE) - sessionid=abc123; theme=dark; lang=en Error somevalidbase64== domain.tld https 609 0.318 - TLSv1.3 TLS_AES_128_GCM_SHA256 Error HTTP/1.1 - - 50294 0.318 Error text/html -

‎packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1334,6 +1334,102 @@
13341334
},
13351335
"version": "7.0"
13361336
}
1337+
},
1338+
{
1339+
"@timestamp": "2024-07-13T15:29:45.000Z",
1340+
"aws": {
1341+
"cloudfront": {
1342+
"cookies": "sessionid=abc123; theme=dark; lang=en",
1343+
"domain": "xxxxxxxxxxxxx.cloudfront.net",
1344+
"edge_detailed_result_type": "Error",
1345+
"edge_location": "EWR53-C1",
1346+
"edge_response_result_type": "Error",
1347+
"edge_result_type": "Error",
1348+
"time_to_first_byte": 0.318
1349+
}
1350+
},
1351+
"cloud": {
1352+
"provider": "aws"
1353+
},
1354+
"destination": {
1355+
"address": "domain.tld",
1356+
"domain": "domain.tld"
1357+
},
1358+
"ecs": {
1359+
"version": "8.11.0"
1360+
},
1361+
"event": {
1362+
"category": [
1363+
"web"
1364+
],
1365+
"duration": 318000000,
1366+
"id": "somevalidbase64==",
1367+
"kind": "event",
1368+
"original": "2024-07-13\t15:29:45\tEWR53-C1\t198083\t127.0.0.1\tGET\txxxxxxxxxxxxx.cloudfront.net\t/en(test)\t404\thttps://domain.tld/\tUser-Agent:%20Mozilla/4.0%20(compatible;%20MSIE%207.0;%20Windows%20NT%205.1;%20360SE)\t-\tsessionid=abc123; theme=dark; lang=en\tError\tsomevalidbase64==\tdomain.tld\thttps\t609\t0.318\t-\tTLSv1.3\tTLS_AES_128_GCM_SHA256\tError\tHTTP/1.1\t-\t-\t50294\t0.318\tError\ttext/html\t-",
1369+
"outcome": "failure",
1370+
"type": [
1371+
"access"
1372+
]
1373+
},
1374+
"http": {
1375+
"request": {
1376+
"bytes": 609,
1377+
"id": "somevalidbase64==",
1378+
"method": "GET",
1379+
"referrer": "https://domain.tld/"
1380+
},
1381+
"response": {
1382+
"bytes": 198083,
1383+
"mime_type": "text/html",
1384+
"status_code": 404
1385+
},
1386+
"version": "1.1"
1387+
},
1388+
"network": {
1389+
"protocol": "https",
1390+
"type": "ipv4"
1391+
},
1392+
"related": {
1393+
"hosts": [
1394+
"xxxxxxxxxxxxx.cloudfront.net",
1395+
"domain.tld"
1396+
],
1397+
"ip": [
1398+
"127.0.0.1"
1399+
]
1400+
},
1401+
"source": {
1402+
"address": "127.0.0.1",
1403+
"ip": "127.0.0.1",
1404+
"port": 50294
1405+
},
1406+
"tags": [
1407+
"preserve_original_event"
1408+
],
1409+
"tls": {
1410+
"cipher": "TLS_AES_128_GCM_SHA256",
1411+
"version": "1.3",
1412+
"version_protocol": "tls"
1413+
},
1414+
"url": {
1415+
"domain": "domain.tld",
1416+
"full": "https://domain.tld/en(test)",
1417+
"path": "/en(test)",
1418+
"scheme": "https"
1419+
},
1420+
"user_agent": {
1421+
"device": {
1422+
"name": "Other"
1423+
},
1424+
"name": "IE",
1425+
"original": "User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
1426+
"os": {
1427+
"full": "Windows XP",
1428+
"name": "Windows",
1429+
"version": "XP"
1430+
},
1431+
"version": "7.0"
1432+
}
13371433
}
13381434
]
13391435
}

‎packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ processors:
164164
- rename:
165165
field: _tmp.cs_cookie
166166
target_field: aws.cloudfront.cookies
167-
if: ctx._tmp?.cs_cookies != null && ctx._tmp.cs_cookies != '-'
167+
if: ctx._tmp?.cs_cookie != null && ctx._tmp.cs_cookie != '-'
168168
# x-edge-result-type
169169
- rename:
170170
field: _tmp.x_edge_result_type

‎packages/aws/data_stream/cloudfront_logs/fields/fields.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,7 @@
3232
type: long
3333
description: |-
3434
When the response contains the HTTP Content-Range header, this field contains the range end value.
35+
- name: cookies
36+
type: keyword
37+
description: |-
38+
This key is used to capture the cookies specifically.

‎packages/aws/docs/cloudfront.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ Refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ec
5555
| Field | Description | Type |
5656
|---|---|---|
5757
| @timestamp | Event timestamp. | date |
58+
| aws.cloudfront.cookies | This key is used to capture the cookies specifically. | keyword |
5859
| aws.cloudfront.domain | The domain name of the CloudFront distribution (for example, d111111abcdef8.cloudfront.net). | keyword |
5960
| aws.cloudfront.edge_detailed_result_type | When the value of the x-edge-result-type field is Error, this field contains the specific type of error. When the object was served to the viewer from the Origin Shield cache, this field contains OriginShieldHit. In all other cases, this field contains the same value as x-edge-result-type. | keyword |
6061
| aws.cloudfront.edge_location | The edge location that served the request. Each edge location is identified by a three-letter code and an arbitrarily assigned number (for example, DFW3). The three-letter code typically corresponds with the International Air Transport Association (IATA) airport code for an airport near the edge location’s geographic location. | keyword |

‎packages/aws/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 3.3.2
22
name: aws
33
title: AWS
4-
version: "5.3.0"
4+
version: "5.3.1"
55
description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
66
type: integration
77
categories:

0 commit comments

Comments
 (0)