Skip to content

Commit bd9254f

Browse files
awahab07awahab07
authored
[Dataset Quality] Fix failing es-promotion test on 8.x (elastic#205188)
Newer builds of ES ingest `event.ingested` and `tags` fields into APM indices making the Dataset Quality tests fail which rely on ingested field counts against set number of field limits. The PR addresses this by explicitly ingesting the fields beforehand. After the merge to main, the PR will be backported to `8.x` to unblock es-promotion.
1 parent 39d8d7d commit bd9254f

File tree

5 files changed

+31
-16
lines changed

5 files changed

+31
-16
lines changed

‎packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ export type ApmFields = Fields<{
126126
'error.id': string;
127127
'error.type': string;
128128
'error.culprit': string;
129-
'event.ingested': number;
129+
'event.ingested': string;
130130
'event.name': string;
131131
'event.action': string;
132132
'event.outcome': string;

‎packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export type LogDocument = Fields &
3636
'error.message'?: string;
3737
'event.original'?: string;
3838
'event.dataset': string;
39+
'event.ingested': string;
3940
'log.level'?: string;
4041
'host.name'?: string;
4142
'container.id'?: string;

‎x-pack/test/functional/apps/dataset_quality/data/logs_data.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,3 +217,7 @@ export const MORE_THAN_1024_CHARS =
217217
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
218218
export const ANOTHER_1024_CHARS =
219219
'grape fig tangerine tangerine kiwi lemon papaya cherry nectarine papaya mango cherry nectarine fig cherry fig grape mango mango quince fig strawberry mango quince date kiwi quince raspberry apple kiwi banana quince fig papaya grape mango cherry banana mango cherry lemon cherry tangerine fig quince quince papaya tangerine grape strawberry banana kiwi grape mango papaya nectarine banana nectarine kiwi papaya lemon apple lemon orange fig cherry grape apple nectarine papaya orange fig papaya date mango papaya mango cherry tangerine papaya apple banana papaya cherry strawberry grape raspberry lemon date papaya mango kiwi cherry fig banana banana apple date strawberry mango tangerine date lemon kiwi quince date orange orange papaya date apple fig tangerine quince tangerine date papaya banana banana orange raspberry papaya apple nectarine lemon raspberry raspberry mango cherry kiwi cherry cherry nectarine cherry date strawberry banana orange mango mango tangerine quince papaya papaya kiwi papaya strawberry date mango';
220+
221+
export const CONSISTENT_TAGS = [
222+
'this_is_here_to_remove_variance_introduced_by_the_geoip_processor',
223+
];

‎x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
getInitialTestLogs,
1616
ANOTHER_1024_CHARS,
1717
MORE_THAN_1024_CHARS,
18+
CONSISTENT_TAGS,
1819
} from './data';
1920
import { logsSynthMappings } from './custom_mappings/custom_synth_mappings';
2021
import { logsNginxMappings } from './custom_mappings/custom_integration_mappings';
@@ -187,7 +188,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
187188
.timestamp(timestamp)
188189
);
189190
}),
190-
// Ingest Degraded Logs with 26 fields in Apm DataSet
191+
// Ingest Degraded Logs with 27 fields in Apm DataSet
191192
timerange(moment(to).subtract(count, 'minute'), moment(to))
192193
.interval('1m')
193194
.rate(1)
@@ -205,7 +206,9 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
205206
.defaults({
206207
'service.name': serviceName,
207208
'trace.id': generateShortId(),
209+
'event.ingested': new Date().toISOString(),
208210
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
211+
tags: CONSISTENT_TAGS, // To account for ES inserted error tags
209212
})
210213
.timestamp(timestamp)
211214
);
@@ -227,7 +230,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
227230

228231
// Set Limit of 26
229232
await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, {
230-
'mapping.total_fields.limit': 25,
233+
'mapping.total_fields.limit': 26,
231234
});
232235

233236
await synthtrace.index([
@@ -279,7 +282,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
279282
.timestamp(timestamp)
280283
);
281284
}),
282-
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
285+
// Ingest Degraded Logs with 29 fields in Apm APP DataSet
283286
timerange(moment(to).subtract(count, 'minute'), moment(to))
284287
.interval('1m')
285288
.rate(1)
@@ -299,6 +302,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
299302
'trace.id': generateShortId(),
300303
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
301304
'cloud.project.id': generateShortId(),
305+
'event.ingested': new Date().toISOString(),
306+
tags: CONSISTENT_TAGS, // To account for ES inserted error tags
302307
})
303308
.timestamp(timestamp)
304309
);
@@ -330,13 +335,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
330335
}
331336
);
332337

333-
// Set Limit of 27
338+
// Set Limit of 28
334339
await PageObjects.datasetQuality.setDataStreamSettings(
335340
PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({
336341
dataset: apmAppDatasetName,
337342
}) + '-000002',
338343
{
339-
'mapping.total_fields.limit': 27,
344+
'mapping.total_fields.limit': 28,
340345
}
341346
);
342347

@@ -389,7 +394,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
389394
.timestamp(timestamp)
390395
);
391396
}),
392-
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
397+
// Ingest Degraded Logs with 29 fields in Apm APP DataSet
393398
timerange(moment(to).subtract(count, 'minute'), moment(to))
394399
.interval('1m')
395400
.rate(1)
@@ -409,6 +414,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
409414
'trace.id': generateShortId(),
410415
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
411416
'cloud.project.id': generateShortId(),
417+
'event.ingested': new Date().toISOString(),
418+
tags: CONSISTENT_TAGS, // To account for ES inserted error tags
412419
})
413420
.timestamp(timestamp)
414421
);

‎x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
102102
});
103103
});
104104

105-
describe('testing root cause for ignored fields', () => {
105+
describe('detecting root cause for ignored fields', () => {
106106
before(async () => {
107107
// Create custom component template
108108
await synthtrace.createComponentTemplate(
@@ -134,14 +134,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
134134
// Install Nginx Integration and ingest logs for it
135135
await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg);
136136

137-
// Create custom component template to avoid issues with LogsDB
137+
// Create custom component template for Nginx to avoid issues with LogsDB
138138
await synthtrace.createComponentTemplate(
139139
customComponentTemplateNameNginx,
140140
logsNginxMappings(nginxAccessDatasetName)
141141
);
142142

143143
await synthtrace.index([
144-
// Ingest Degraded Logs with 25 fields
144+
// Ingest Degraded Logs with 25 fields in degraded DataSet
145145
timerange(moment(to).subtract(count, 'minute'), moment(to))
146146
.interval('1m')
147147
.rate(1)
@@ -187,7 +187,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
187187
.timestamp(timestamp)
188188
);
189189
}),
190-
// Ingest Degraded Logs with 26 fields in Apm DataSet
190+
// Ingest Degraded Logs with 27 fields in Apm DataSet
191191
timerange(moment(to).subtract(count, 'minute'), moment(to))
192192
.interval('1m')
193193
.rate(1)
@@ -206,6 +206,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
206206
'service.name': serviceName,
207207
'trace.id': generateShortId(),
208208
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
209+
'event.ingested': new Date().toISOString(),
209210
// this works around a geoip limitation in CI
210211
tags: CONSISTENT_TAGS,
211212
})
@@ -229,7 +230,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
229230

230231
// Set Limit of 26
231232
await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, {
232-
'mapping.total_fields.limit': 25,
233+
'mapping.total_fields.limit': 26,
233234
});
234235

235236
await synthtrace.index([
@@ -281,7 +282,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
281282
.timestamp(timestamp)
282283
);
283284
}),
284-
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
285+
// Ingest Degraded Logs with 29 fields in Apm APP DataSet
285286
timerange(moment(to).subtract(count, 'minute'), moment(to))
286287
.interval('1m')
287288
.rate(1)
@@ -303,6 +304,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
303304
'cloud.project.id': generateShortId(),
304305
// this works around a geoip limitation in CI
305306
tags: CONSISTENT_TAGS,
307+
'event.ingested': new Date().toISOString(),
306308
})
307309
.timestamp(timestamp)
308310
);
@@ -334,13 +336,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
334336
}
335337
);
336338

337-
// Set Limit of 27
339+
// Set Limit of 28
338340
await PageObjects.datasetQuality.setDataStreamSettings(
339341
PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({
340342
dataset: apmAppDatasetName,
341343
}) + '-000002',
342344
{
343-
'mapping.total_fields.limit': 27,
345+
'mapping.total_fields.limit': 28,
344346
}
345347
);
346348

@@ -393,7 +395,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
393395
.timestamp(timestamp)
394396
);
395397
}),
396-
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
398+
// Ingest Degraded Logs with 29 fields in Apm APP DataSet
397399
timerange(moment(to).subtract(count, 'minute'), moment(to))
398400
.interval('1m')
399401
.rate(1)
@@ -413,6 +415,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
413415
'trace.id': generateShortId(),
414416
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
415417
'cloud.project.id': generateShortId(),
418+
'event.ingested': new Date().toISOString(),
416419
// this works around a geoip limitation in CI
417420
tags: CONSISTENT_TAGS,
418421
})

0 commit comments

Comments
 (0)