diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsIndexingIT.java new file mode 100644 index 0000000000000..178cfe856a9f5 --- /dev/null +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsIndexingIT.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xcontent.XContentType; + +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.UUID; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class LogsIndexingIT extends ESSingleNodeTestCase { + + public static final String MAPPING_TEMPLATE = """ + { + "_doc":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "metricset": { + "type": "keyword" + } + } + } + }"""; + + private static final String DOC = """ + { + "@timestamp": "$time", + "metricset": "pod", + "k8s": { + "pod": { + "name": "dog", + "uid":"$uuid", + "ip": "10.10.55.3", + "network": { + "tx": 1434595272, + "rx": 530605511 + } + } + } + } + """; + + @Override + protected Collection> getPlugins() { + return List.of(DataStreamsPlugin.class, InternalSettingsPlugin.class); + } + + public void testIndexSearchAndRetrieval() throws Exception { + String[] uuis = { + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + UUID.randomUUID().toString() }; + + String dataStreamName = "k8s"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName + "*")) + .template( + new Template( + Settings.builder() + .put("index.mode", "logsdb") + .put("index.routing_path", "metricset,k8s.pod.uid") + .put("index.number_of_replicas", 0) + // Reduce sync interval to speedup this integraton test, + // otherwise by default it will take 30 seconds before minimum retained seqno is updated: + .put("index.soft_deletes.retention_lease.sync_interval", "100ms") + .build(), + new CompressedXContent(MAPPING_TEMPLATE), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest).actionGet(); + + // index some data + int numBulkRequests = randomIntBetween(128, 1024); + ; + int numDocsPerBulk = randomIntBetween(16, 256); + String indexName = null; + { + Instant time = Instant.now(); + for (int i = 0; i < numBulkRequests; i++) { + BulkRequest bulkRequest = new BulkRequest(dataStreamName); + for (int j = 0; j < numDocsPerBulk; j++) { + var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); + indexRequest.source( + DOC.replace("$time", formatInstant(time)).replace("$uuid", uuis[j % uuis.length]), + XContentType.JSON + ); + bulkRequest.add(indexRequest); + time = time.plusMillis(1); + } + var bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat(bulkResponse.hasFailures(), is(false)); + indexName = bulkResponse.getItems()[0].getIndex(); + } + client().admin().indices().refresh(new RefreshRequest(dataStreamName)).actionGet(); + } + + // Check the search api can synthesize _id + final String idxName = indexName; + var searchRequest = new SearchRequest(dataStreamName); + searchRequest.source().trackTotalHits(true); + assertResponse(client().search(searchRequest), searchResponse -> { + assertThat(searchResponse.getHits().getTotalHits().value(), equalTo((long) numBulkRequests * numDocsPerBulk)); + String id = searchResponse.getHits().getHits()[0].getId(); + assertThat(id, notNullValue()); + + // Check that the _id is gettable: + var getResponse = client().get(new GetRequest(idxName).id(id)).actionGet(); + assertThat(getResponse.isExists(), is(true)); + assertThat(getResponse.getId(), equalTo(id)); + }); + } + + static String formatInstant(Instant instant) { + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); + } + +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 4f34cbd3cc475..439b51297fe18 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -54,15 +54,14 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.OnScriptError; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; @@ -808,13 +807,13 @@ private static Response prepareRamIndex( BytesReference document = request.contextSetup.document; XContentType xContentType = request.contextSetup.xContentType; - SourceToParse sourceToParse = (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) + SourceToParse sourceToParse = (indexService.getIndexSettings().usesRoutingPath()) ? new SourceToParse( null, document, xContentType, indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) - ? TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE + ? RoutingPathHashFieldMapper.DUMMY_ENCODED_VALUE : null ) : new SourceToParse("_id", document, xContentType); diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 6c77186089644..3175a60e2cfb9 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -43,9 +43,9 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.TextFieldFamilySyntheticSourceTestSetup; import org.elasticsearch.index.mapper.TextFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -333,7 +333,7 @@ public void testStoreParameterDefaults() throws IOException { }); DocumentMapper mapper = createMapperService(getVersion(), indexSettings, () -> true, mapping).documentMapper(); - var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { + var source = source(RoutingPathHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { b.field("field", "1234"); if (timeSeriesIndexMode) { b.field("@timestamp", "2000-10-10T23:40:53.384Z"); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index d0f89b1b8b6cb..d07ebeed13263 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -191,7 +191,7 @@ missing sort field: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "unknown index sort field:[host_name]" } + - match: { error.reason: "unknown index sort field:[host_name] required by [index.mode=logsdb]" } --- non-default sort settings: @@ -479,7 +479,7 @@ override sort field using nested field type: - is_false: error --- -routing path not allowed in logs mode: +routing path allowed in logs mode: - requires: test_runner_features: [ capabilities ] capabilities: @@ -489,7 +489,6 @@ routing path not allowed in logs mode: reason: "Support for 'logsdb' index mode capability required" - do: - catch: bad_request indices.create: index: test body: @@ -514,9 +513,13 @@ routing path not allowed in logs mode: message: type: text - - match: { error.root_cause.0.type: "illegal_argument_exception" } - - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "[index.routing_path] requires [index.mode=time_series]" } + - do: + indices.get_settings: + index: test + + - is_true: test + - match: { test.settings.index.mode: "logsdb" } + - match: { test.settings.index.routing_path: [ "host.name", "agent_id"] } --- start time not allowed in logs mode: diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 4dcc7c73c280e..cfa0fb9187140 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -81,7 +81,7 @@ public static boolean isFailureStoreFeatureFlagEnabled() { public static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd"); public static final String TIMESTAMP_FIELD_NAME = "@timestamp"; // Timeseries indices' leaf readers should be sorted by desc order of their timestamp field, as it allows search time optimizations - public static Comparator TIMESERIES_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> { + public static Comparator TIME_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> { try { PointValues points = r.getPointValues(TIMESTAMP_FIELD_NAME); if (points != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 3fb3c182f89cd..24642056e22c3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -25,7 +25,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParser.Token; @@ -290,7 +290,7 @@ public int indexShard( checkNoRouting(routing); int hash = hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); if (trackTimeSeriesRoutingHash) { - routingHashSetter.accept(TimeSeriesRoutingHashFieldMapper.encode(hash)); + routingHashSetter.accept(RoutingPathHashFieldMapper.encode(hash)); } return hashToShardId(hash); } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java index 08a8e28457159..bb750cf192759 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java @@ -149,7 +149,7 @@ public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, * A special variant of loading docid and version in case of time series indices. *

* Makes use of the fact that timestamp is part of the id, the existence of @timestamp field and - * that segments are sorted by {@link org.elasticsearch.cluster.metadata.DataStream#TIMESERIES_LEAF_READERS_SORTER}. + * that segments are sorted by {@link org.elasticsearch.cluster.metadata.DataStream#TIME_LEAF_READERS_SORTER}. * This allows this method to know whether there is no document with the specified id without loading the docid for * the specified id. * diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index f5f923f3657f8..87712afdc391f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -26,6 +26,8 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.LogsIdExtractingIdFieldMapper; +import org.elasticsearch.index.mapper.LogsIdFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; @@ -34,9 +36,9 @@ import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.RoutingFields; import org.elasticsearch.index.mapper.RoutingPathFields; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import java.io.IOException; @@ -63,6 +65,7 @@ public enum IndexMode { STANDARD("standard") { @Override void validateWithOtherSettings(Map, Object> settings) { + IndexMode.validateRoutingPathSetting(settings); IndexMode.validateTimeSeriesSettings(settings); } @@ -90,13 +93,13 @@ public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { } @Override - public MetadataFieldMapper timeSeriesIdFieldMapper() { + public MetadataFieldMapper indexModeIdFieldMapper() { // non time-series indices must not have a TimeSeriesIdFieldMapper return null; } @Override - public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { + public MetadataFieldMapper routingHashFieldMapper() { // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper return null; } @@ -190,13 +193,13 @@ private static String routingRequiredBad() { } @Override - public MetadataFieldMapper timeSeriesIdFieldMapper() { + public MetadataFieldMapper indexModeIdFieldMapper() { return TimeSeriesIdFieldMapper.INSTANCE; } @Override - public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { - return TimeSeriesRoutingHashFieldMapper.INSTANCE; + public MetadataFieldMapper routingHashFieldMapper() { + return RoutingPathHashFieldMapper.INSTANCE; } public IdFieldMapper idFieldMapperWithoutFieldData() { @@ -262,12 +265,12 @@ public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { @Override public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { - return new ProvidedIdFieldMapper(fieldDataEnabled); + return LogsIdExtractingIdFieldMapper.INSTANCE; } @Override public IdFieldMapper idFieldMapperWithoutFieldData() { - return ProvidedIdFieldMapper.NO_FIELD_DATA; + return LogsIdExtractingIdFieldMapper.INSTANCE; } @Override @@ -276,20 +279,12 @@ public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { } @Override - public MetadataFieldMapper timeSeriesIdFieldMapper() { - // non time-series indices must not have a TimeSeriesIdFieldMapper - return null; + public MetadataFieldMapper indexModeIdFieldMapper() { + return LogsIdFieldMapper.INSTANCE; } - @Override - public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { - // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper - return null; - } - - @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { - return RoutingFields.Noop.INSTANCE; + public MetadataFieldMapper routingHashFieldMapper() { + return RoutingPathHashFieldMapper.INSTANCE; } @Override @@ -347,14 +342,12 @@ public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { } @Override - public MetadataFieldMapper timeSeriesIdFieldMapper() { - // non time-series indices must not have a TimeSeriesIdFieldMapper + public MetadataFieldMapper indexModeIdFieldMapper() { return null; } @Override - public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { - // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper + public MetadataFieldMapper routingHashFieldMapper() { return null; } @@ -389,8 +382,11 @@ public SourceFieldMapper.Mode defaultSourceMode() { private static final String HOST_NAME = "host.name"; - private static void validateTimeSeriesSettings(Map, Object> settings) { + private static void validateRoutingPathSetting(Map, Object> settings) { settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); + } + + private static void validateTimeSeriesSettings(Map, Object> settings) { settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME); settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME); } @@ -510,22 +506,28 @@ public String getName() { /** * Return an instance of the {@link TimeSeriesIdFieldMapper} that generates - * the _tsid field. The field mapper will be added to the list of the metadata + * the id field. The field mapper will be added to the list of the metadata * field mappers for the index. */ - public abstract MetadataFieldMapper timeSeriesIdFieldMapper(); + public abstract MetadataFieldMapper indexModeIdFieldMapper(); /** - * Return an instance of the {@link TimeSeriesRoutingHashFieldMapper} that generates + * Return an instance of the {@link RoutingPathHashFieldMapper} that generates * the _ts_routing_hash field. The field mapper will be added to the list of the metadata * field mappers for the index. */ - public abstract MetadataFieldMapper timeSeriesRoutingHashFieldMapper(); + public abstract MetadataFieldMapper routingHashFieldMapper(); /** - * How {@code time_series_dimension} fields are handled by indices in this mode. + * How routing fields are handled by indices in this mode. */ - public abstract RoutingFields buildRoutingFields(IndexSettings settings); + public RoutingFields buildRoutingFields(IndexSettings settings) { + if (settings.usesRoutingPath()) { + IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); + return new RoutingPathFields(routing.builder()); + } + return RoutingFields.Noop.INSTANCE; + } /** * @return Whether timestamps should be validated for being withing the time range of an index. diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 5bea838f9d70c..9f097b8ae3628 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -1182,6 +1182,11 @@ public IndexMode getMode() { return mode; } + public boolean usesRoutingPath() { + return mode == IndexMode.TIME_SERIES + || (mode == IndexMode.LOGSDB && IndexMetadata.INDEX_ROUTING_PATH.get(settings).isEmpty() == false); + } + /** * Returns the node settings. The settings returned from {@link #getSettings()} are a merged version of the * index settings and the node settings where node settings are overwritten by index settings. diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 2811c7493a277..49cae1490c202 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; +import org.elasticsearch.index.mapper.LogsIdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.search.MultiValueMode; @@ -107,6 +108,14 @@ public final class IndexSortConfig { TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; } + public static final FieldSortSpec[] LOGS_SORT; + + static { + FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); + timeStampSpec.order = SortOrder.DESC; + LOGS_SORT = new FieldSortSpec[] { new FieldSortSpec(LogsIdFieldMapper.NAME), timeStampSpec }; + } + private static String validateMissingValue(String missing) { if ("_last".equals(missing) == false && "_first".equals(missing) == false) { throw new IllegalArgumentException("Illegal missing value:[" + missing + "], " + "must be one of [_last, _first]"); @@ -140,9 +149,16 @@ public IndexSortConfig(IndexSettings indexSettings) { final Settings settings = indexSettings.getSettings(); this.indexMode = indexSettings.getMode(); - if (this.indexMode == IndexMode.TIME_SERIES) { - this.sortSpecs = TIME_SERIES_SORT; - return; + if (indexSettings.usesRoutingPath()) { + if (this.indexMode == IndexMode.TIME_SERIES) { + this.sortSpecs = TIME_SERIES_SORT; + return; + } + + if (this.indexMode == IndexMode.LOGSDB) { + this.sortSpecs = LOGS_SORT; + return; + } } List fields = INDEX_SORT_FIELD_SETTING.get(settings); @@ -223,11 +239,9 @@ public Sort buildIndexSort( FieldSortSpec sortSpec = sortSpecs[i]; final MappedFieldType ft = fieldTypeLookup.apply(sortSpec.field); if (ft == null) { - String err = "unknown index sort field:[" + sortSpec.field + "]"; - if (this.indexMode == IndexMode.TIME_SERIES) { - err += " required by [" + IndexSettings.MODE.getKey() + "=time_series]"; - } - throw new IllegalArgumentException(err); + throw new IllegalArgumentException( + "unknown index sort field:[" + sortSpec.field + "] required by [" + IndexSettings.MODE.getKey() + "=" + indexMode + "]" + ); } if (Objects.equals(ft.name(), sortSpec.field) == false) { throw new IllegalArgumentException("Cannot use alias [" + sortSpec.field + "] as an index sort field"); @@ -252,6 +266,7 @@ public Sort buildIndexSort( validateIndexSortField(sortFields[i]); } return new Sort(sortFields); + } private static void validateIndexSortField(SortField sortField) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 9c2a08a69002c..8554defa62a25 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -21,8 +21,11 @@ import org.elasticsearch.index.codec.postings.ES812PostingsFormat; import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.LogsIdFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; /** @@ -72,7 +75,7 @@ boolean useBloomFilter(String field) { // but based on dimension fields and timestamp field, so during indexing // version/seq_no/term needs to be looked up and having a bloom filter // can speed this up significantly. - return indexSettings.getMode() == IndexMode.TIME_SERIES + return (indexSettings.usesRoutingPath()) && IdFieldMapper.NAME.equals(field) && IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.get(indexSettings.getSettings()); } else { @@ -110,7 +113,10 @@ boolean useTSDBDocValuesFormat(final String field) { private boolean excludeFields(String fieldName) { // Avoid using tsdb codec for fields like _seq_no, _primary_term. // But _tsid and _ts_routing_hash should always use the tsdb codec. - return fieldName.startsWith("_") && fieldName.equals("_tsid") == false && fieldName.equals("_ts_routing_hash") == false; + return fieldName.startsWith("_") + && fieldName.equals(TimeSeriesIdFieldMapper.NAME) == false + && fieldName.equals(LogsIdFieldMapper.NAME) == false + && fieldName.equals(RoutingPathHashFieldMapper.NAME) == false; } private boolean isTimeSeriesModeIndex() { diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 8d43252d178ee..07d2309d288b8 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -73,7 +73,6 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -1020,8 +1019,8 @@ private VersionValue resolveDocVersion(final Operation op, boolean loadSeqNo) th assert incrementIndexVersionLookup(); // used for asserting in tests final VersionsAndSeqNoResolver.DocIdAndVersion docIdAndVersion; try (Searcher searcher = acquireSearcher("load_version", SearcherScope.INTERNAL)) { - if (engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - assert engineConfig.getLeafSorter() == DataStream.TIMESERIES_LEAF_READERS_SORTER; + if (engineConfig.getIndexSettings().usesRoutingPath()) { + assert engineConfig.getLeafSorter() == DataStream.TIME_LEAF_READERS_SORTER; docIdAndVersion = VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion( searcher.getIndexReader(), op.uid(), @@ -2710,7 +2709,7 @@ private IndexWriterConfig getIndexWriterConfig() { iwc.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD); mergePolicy = new RecoverySourcePruneMergePolicy( SourceFieldMapper.RECOVERY_SOURCE_NAME, - engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES, + engineConfig.getIndexSettings().usesRoutingPath(), softDeletesPolicy::getRetentionQuery, new SoftDeletesRetentionMergePolicy( Lucene.SOFT_DELETES_FIELD, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 10484a1c26098..c23b50420d8b4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; @@ -154,21 +155,23 @@ public void validate(IndexSettings settings, boolean checkLimits) { } } } - List routingPaths = settings.getIndexMetadata().getRoutingPaths(); - for (String path : routingPaths) { - for (String match : mappingLookup.getMatchingFieldNames(path)) { - mappingLookup.getFieldType(match).validateMatchedRoutingPath(path); - } - for (String objectName : mappingLookup.objectMappers().keySet()) { - // object type is not allowed in the routing paths - if (path.equals(objectName)) { - throw new IllegalArgumentException( - "All fields that match routing_path must be configured with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] " - + "and without the [script] parameter. [" - + objectName - + "] was [object]." - ); + if (settings.getMode() == IndexMode.TIME_SERIES) { + List routingPaths = settings.getIndexMetadata().getRoutingPaths(); + for (String path : routingPaths) { + for (String match : mappingLookup.getMatchingFieldNames(path)) { + mappingLookup.getFieldType(match).validateMatchedRoutingPath(path); + } + for (String objectName : mappingLookup.objectMappers().keySet()) { + // object type is not allowed in the routing paths + if (path.equals(objectName)) { + throw new IllegalArgumentException( + "All fields that match routing_path must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] " + + "and without the [script] parameter. [" + + objectName + + "] was [object]." + ); + } } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 82004356ceb57..e66f2fbe58be7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -145,6 +145,8 @@ private void internalParseDocument(MetadataFieldMapper[] metadataFieldsMappers, executeIndexTimeScripts(context); + context.getRoutingFieldsForLogsMode(); + for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) { metadataMapper.postParse(context); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 51e4e9f4c1b5e..e7baeb81b68fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -13,6 +13,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; @@ -274,7 +275,7 @@ public final SourceToParse sourceToParse() { } public final String routing() { - return mappingParserContext.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? null : sourceToParse.routing(); + return mappingParserContext.getIndexSettings().usesRoutingPath() ? null : sourceToParse.routing(); } /** @@ -825,6 +826,37 @@ public final DynamicTemplate findDynamicTemplate(String fieldName, DynamicTempla return null; } + /** + * Identify the fields that match the routing path, for indexes in logs mode. These fields are equivalent to TSDB dimensions. + */ + public final void getRoutingFieldsForLogsMode() { + if (indexSettings().getMode() == IndexMode.LOGSDB + && indexSettings().getIndexRouting() instanceof IndexRouting.ExtractFromSource dimensionRouting) { + for (var mapper : mappingLookup().fieldMappers()) { + if (mapper instanceof FieldMapper fieldMapper && dimensionRouting.matchesField(fieldMapper.fullPath())) { + String name = fieldMapper.fullPath(); + var field = rootDoc().getField(name); + if (field != null) { + var binaryValue = field.binaryValue(); + if (binaryValue != null) { + getRoutingFields().addString(name, binaryValue); + } else { + var stringValue = field.stringValue(); + if (stringValue != null) { + getRoutingFields().addString(name, stringValue); + } else { + var numericValueValue = field.numericValue(); + if (numericValueValue != null) { + getRoutingFields().addLong(name, numericValueValue.longValue()); + } + } + } + } + } + } + } + } + // XContentParser that wraps an existing parser positioned on a value, // and a field name, and returns a stream that looks like { 'field' : 'value' } private static class CopyToParser extends FilterXContentParserWrapper { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 741252f98473b..0554700a481bf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -17,6 +17,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import java.io.IOException; @@ -25,7 +26,7 @@ /** * Responsible for loading the _id from stored fields or for TSDB synthesizing the _id from the routing, _tsid and @timestamp fields. */ -public sealed interface IdLoader permits IdLoader.TsIdLoader, IdLoader.StoredIdLoader { +public sealed interface IdLoader permits IdLoader.SyntheticIdLoader, IdLoader.StoredIdLoader { /** * @return returns an {@link IdLoader} instance the loads the _id from stored field. @@ -37,8 +38,8 @@ static IdLoader fromLeafStoredFieldLoader() { /** * @return returns an {@link IdLoader} instance that syn synthesizes _id from routing, _tsid and @timestamp fields. */ - static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { - return new TsIdLoader(indexRouting, routingPaths); + static IdLoader createSyntheticIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths, IndexMode indexMode) { + return new SyntheticIdLoader(indexRouting, routingPaths, indexMode); } Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException; @@ -56,14 +57,16 @@ sealed interface Leaf permits StoredLeaf, TsIdLeaf { } - final class TsIdLoader implements IdLoader { + final class SyntheticIdLoader implements IdLoader { private final IndexRouting.ExtractFromSource indexRouting; private final List routingPaths; + private final IndexMode indexMode; - TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { + SyntheticIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths, IndexMode indexMode) { this.routingPaths = routingPaths; this.indexRouting = indexRouting; + this.indexMode = indexMode; } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { @@ -92,32 +95,35 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] String[] ids = new String[docIdsInLeaf.length]; // Each document always has exactly one tsid and one timestamp: - SortedDocValues tsIdDocValues = DocValues.getSorted(reader, TimeSeriesIdFieldMapper.NAME); + SortedDocValues indexModeIdDocValues = DocValues.getSorted( + reader, + (indexMode == IndexMode.TIME_SERIES) ? TimeSeriesIdFieldMapper.NAME : LogsIdFieldMapper.NAME + ); SortedNumericDocValues timestampDocValues = DocValues.getSortedNumeric(reader, DataStream.TIMESTAMP_FIELD_NAME); - SortedDocValues routingHashDocValues = builders == null - ? DocValues.getSorted(reader, TimeSeriesRoutingHashFieldMapper.NAME) - : null; + SortedDocValues routingHashDocValues = builders == null ? DocValues.getSorted(reader, RoutingPathHashFieldMapper.NAME) : null; for (int i = 0; i < docIdsInLeaf.length; i++) { int docId = docIdsInLeaf[i]; - boolean found = tsIdDocValues.advanceExact(docId); + boolean found = indexModeIdDocValues.advanceExact(docId); assert found; - BytesRef tsid = tsIdDocValues.lookupOrd(tsIdDocValues.ordValue()); + BytesRef indexModeId = indexModeIdDocValues.lookupOrd(indexModeIdDocValues.ordValue()); found = timestampDocValues.advanceExact(docId); assert found; assert timestampDocValues.docValueCount() == 1; long timestamp = timestampDocValues.nextValue(); if (builders != null) { var routingBuilder = builders[i]; - ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); + ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, indexModeId, timestamp, new byte[16]); } else { found = routingHashDocValues.advanceExact(docId); assert found; BytesRef routingHashBytes = routingHashDocValues.lookupOrd(routingHashDocValues.ordValue()); - int routingHash = TimeSeriesRoutingHashFieldMapper.decode( + int routingHash = RoutingPathHashFieldMapper.decode( Uid.decodeId(routingHashBytes.bytes, routingHashBytes.offset, routingHashBytes.length) ); - ids[i] = TsidExtractingIdFieldMapper.createId(routingHash, tsid, timestamp); + ids[i] = (indexMode == IndexMode.TIME_SERIES) + ? TsidExtractingIdFieldMapper.createId(routingHash, indexModeId, timestamp) + : LogsIdExtractingIdFieldMapper.createId(routingHash, indexModeId, timestamp); } } return new TsIdLeaf(docIdsInLeaf, ids); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LogsIdExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LogsIdExtractingIdFieldMapper.java new file mode 100644 index 0000000000000..46ba513781e10 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/LogsIdExtractingIdFieldMapper.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.hash.MurmurHash3; +import org.elasticsearch.common.hash.MurmurHash3.Hash128; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; + +import java.util.Base64; +import java.util.Locale; + +/** + * A mapper for the {@code _id} field that builds the {@code _id} from the + * {@code _logs_id} and {@code @timestamp}. + */ +public class LogsIdExtractingIdFieldMapper extends IdFieldMapper { + /** + * Maximum length of the {@code _tsid} in the {@link #documentDescription}. + */ + static final int DESCRIPTION_LIMIT = 1000; + + public static final LogsIdExtractingIdFieldMapper INSTANCE = new LogsIdExtractingIdFieldMapper(); + + private LogsIdExtractingIdFieldMapper() { + super(new AbstractIdFieldType() { + @Override + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + throw new IllegalArgumentException("Fielddata is not supported on [_id] field in [time_series] indices"); + } + }); + } + + private static final long SEED = 0; + + public static void createField(DocumentParserContext context, BytesRef logsId) { + final IndexableField timestampField = context.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); + if (timestampField == null) { + throw new IllegalArgumentException( + "data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] is missing" + ); + } + long timestamp = timestampField.numericValue().longValue(); + String id; + if (context.sourceToParse().routing() != null) { + int routingHash = RoutingPathHashFieldMapper.decode(context.sourceToParse().routing()); + id = createId(routingHash, logsId, timestamp); + } else { + if (context.sourceToParse().id() == null) { + throw new IllegalArgumentException( + "_ts_routing_hash was null but must be set because index [" + + context.indexSettings().getIndexMetadata().getIndex().getName() + + "] is in time_series mode" + ); + } + // In Translog operations, the id has already been generated based on the routing hash while the latter is no longer available. + id = context.sourceToParse().id(); + } + if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "_id must be unset or set to [%s] but was [%s] because [%s] is in logs mode", + id, + context.sourceToParse().id(), + context.indexSettings().getIndexMetadata().getIndex().getName() + ) + ); + } + context.id(id); + BytesRef uidEncoded = Uid.encodeId(context.id()); + context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); + } + + public static String createId(int routingHash, BytesRef tsid, long timestamp) { + Hash128 hash = new Hash128(); + MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); + + byte[] bytes = new byte[20]; + ByteUtils.writeIntLE(routingHash, bytes, 0); + ByteUtils.writeLongLE(hash.h1, bytes, 4); + ByteUtils.writeLongBE(timestamp, bytes, 12); // Big Ending shrinks the inverted index by ~37% + + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + @Override + public String documentDescription(DocumentParserContext context) { + /* + * We don't yet have an _id because it'd be generated by the document + * parsing process. But we *might* have something more useful - the + * time series dimensions and the timestamp! If we have those, then + * include them in the description. If not, all we know is + * "a time series document". + */ + StringBuilder description = new StringBuilder("a logs document"); + IndexableField tsidField = context.doc().getField(TimeSeriesIdFieldMapper.NAME); + if (tsidField != null) { + description.append(" with logs_id ").append(logsIdDescription(tsidField)); + } + IndexableField timestampField = context.doc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); + if (timestampField != null) { + String timestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(timestampField.numericValue().longValue()); + description.append(" at [").append(timestamp).append(']'); + } + return description.toString(); + } + + @Override + public String documentDescription(ParsedDocument parsedDocument) { + IndexableField logsIdField = parsedDocument.rootDoc().getField(LogsIdFieldMapper.NAME); + long timestamp = parsedDocument.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH).numericValue().longValue(); + String timestampStr = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(timestamp); + return "[" + parsedDocument.id() + "][" + logsIdDescription(logsIdField) + "@" + timestampStr + "]"; + } + + private static String logsIdDescription(IndexableField field) { + String encoded = RoutingPathFields.encode(field.binaryValue()).toString(); + if (encoded.length() <= DESCRIPTION_LIMIT) { + return encoded; + } + return encoded.substring(0, DESCRIPTION_LIMIT) + "...}"; + } + + @Override + public String reindexId(String id) { + // null the _id so we recalculate it on write + return null; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LogsIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LogsIdFieldMapper.java new file mode 100644 index 0000000000000..2ea553f9614e4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/LogsIdFieldMapper.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.plain.SortedOrdinalsIndexFieldData; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.Collections; + +/** + * Mapper for {@code _logs_id} field generated when the index is + * {@link IndexMode#LOGSDB used for logs} and it contains a routing path. + */ +public class LogsIdFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_logs_id"; + public static final LogsIdFieldType FIELD_TYPE = new LogsIdFieldType(); + public static final LogsIdFieldMapper INSTANCE = new LogsIdFieldMapper(); + + @Override + public FieldMapper.Builder getMergeBuilder() { + return new Builder().init(this); + } + + public static class Builder extends MetadataFieldMapper.Builder { + + protected Builder() { + super(NAME); + } + + @Override + protected Parameter[] getParameters() { + return EMPTY_PARAMETERS; + } + + @Override + public LogsIdFieldMapper build() { + return INSTANCE; + } + } + + public static final TypeParser PARSER = new FixedTypeParser(c -> { + if (c.getIndexSettings().usesRoutingPath()) { + return c.getIndexSettings().getMode().indexModeIdFieldMapper(); + } + return null; + }); + + public static final class LogsIdFieldType extends MappedFieldType { + private LogsIdFieldType() { + super(NAME, false, false, true, TextSearchInfo.NONE, Collections.emptyMap()); + } + + @Override + public String typeName() { + return NAME; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); + } + + @Override + public DocValueFormat docValueFormat(String format, ZoneId timeZone) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return DocValueFormat.LOGS_ID; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + failIfNoDocValues(); + // TODO don't leak the id binary format into the script + return new SortedOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new DelegateDocValuesField( + new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), + n + ) + ); + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + throw new IllegalArgumentException("[" + NAME + "] is not searchable"); + } + } + + private LogsIdFieldMapper() { + super(FIELD_TYPE); + } + + @Override + public void postParse(DocumentParserContext context) throws IOException { + assert fieldType().isIndexed() == false; + if (context.indexSettings().usesRoutingPath()) { + final RoutingPathFields routingPathFields = (RoutingPathFields) context.getRoutingFields(); + final BytesRef id = routingPathFields.buildHash().toBytesRef(); + context.doc().add(new SortedDocValuesField(fieldType().name(), id)); + LogsIdExtractingIdFieldMapper.createField(context, id); + } + } + + @Override + protected String contentType() { + return NAME; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 5743baeec536d..92ca3e77f91c3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -51,7 +51,7 @@ public Set getFeatures() { FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT, IndexSettings.IGNORE_ABOVE_INDEX_LEVEL_SETTING, SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX, - TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF, + RoutingPathHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF, FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT, DenseVectorFieldMapper.BBQ_FORMAT ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java index 73baca1bf3fdb..78d99154fd29c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; @@ -225,6 +226,24 @@ private void add(String fieldName, BytesReference encoded) throws IOException { } } + static Object encode(StreamInput in) { + try { + return base64Encode(in.readSlicedBytesReference().toBytesRef()); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to read tsid"); + } + } + + public static Object encode(final BytesRef bytesRef) { + return base64Encode(bytesRef); + } + + private static String base64Encode(final BytesRef bytesRef) { + byte[] bytes = new byte[bytesRef.length]; + System.arraycopy(bytesRef.bytes, bytesRef.offset, bytes, 0, bytesRef.length); + return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(bytes); + } + public static Map decodeAsMap(BytesRef bytesRef) { try (StreamInput in = new BytesArray(bytesRef).streamInput()) { int size = in.readVInt(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathHashFieldMapper.java similarity index 83% rename from server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java rename to server/src/main/java/org/elasticsearch/index/mapper/RoutingPathHashFieldMapper.java index 6336ed831b78d..854de396b6fa4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathHashFieldMapper.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -36,25 +35,28 @@ /** * Mapper for the {@code _ts_routing_hash} field. * - * The field contains the routing hash, as calculated in coordinating nodes for docs in time-series indexes. + * The field contains the routing hash, as calculated in coordinating nodes for docs in time-series and logs indexes. * It's stored to be retrieved and added as a prefix when reconstructing the _id field in search queries. - * The prefix can then used for routing Get and Delete requests (by doc id) to the right shard. + * The prefix can then be used for routing Get and Delete requests (by doc id) to the right shard. */ -public class TimeSeriesRoutingHashFieldMapper extends MetadataFieldMapper { +public class RoutingPathHashFieldMapper extends MetadataFieldMapper { public static final String NAME = "_ts_routing_hash"; - public static final TimeSeriesRoutingHashFieldMapper INSTANCE = new TimeSeriesRoutingHashFieldMapper(); + public static final RoutingPathHashFieldMapper INSTANCE = new RoutingPathHashFieldMapper(); - public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingHashFieldMapper()); + public static final TypeParser PARSER = new FixedTypeParser(c -> { + if (c.getIndexSettings().usesRoutingPath()) { + return c.getIndexSettings().getMode().routingHashFieldMapper(); + } + return null; + }); static final NodeFeature TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF = new NodeFeature("tsdb.ts_routing_hash_doc_value_parse_byte_ref"); - public static DocValueFormat TS_ROUTING_HASH_DOC_VALUE_FORMAT = TimeSeriesRoutingHashFieldType.DOC_VALUE_FORMAT; - - static final class TimeSeriesRoutingHashFieldType extends MappedFieldType { + static final class DimensionRoutingHashFieldType extends MappedFieldType { - private static final TimeSeriesRoutingHashFieldType INSTANCE = new TimeSeriesRoutingHashFieldType(); - static final DocValueFormat DOC_VALUE_FORMAT = new DocValueFormat() { + private static final DimensionRoutingHashFieldType INSTANCE = new DimensionRoutingHashFieldType(); + private static final DocValueFormat DOC_VALUE_FORMAT = new DocValueFormat() { @Override public String getWriteableName() { @@ -78,7 +80,7 @@ public BytesRef parseBytesRef(Object value) { } }; - private TimeSeriesRoutingHashFieldType() { + private DimensionRoutingHashFieldType() { super(NAME, false, false, true, TextSearchInfo.NONE, Collections.emptyMap()); } @@ -116,13 +118,13 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } } - private TimeSeriesRoutingHashFieldMapper() { - super(TimeSeriesRoutingHashFieldType.INSTANCE); + private RoutingPathHashFieldMapper() { + super(DimensionRoutingHashFieldType.INSTANCE); } @Override public void postParse(DocumentParserContext context) { - if (context.indexSettings().getMode() == IndexMode.TIME_SERIES + if ((context.indexSettings().usesRoutingPath()) && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { String routingHash = context.sourceToParse().routing(); if (routingHash == null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 8af3c3e6ec270..73c05bd5d7152 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -68,7 +68,7 @@ public TimeSeriesIdFieldMapper build() { } } - public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesIdFieldMapper()); + public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().indexModeIdFieldMapper()); public static final class TimeSeriesIdFieldType extends MappedFieldType { private TimeSeriesIdFieldType() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 821b11410f93b..b5f8ab5570a01 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -66,7 +66,7 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); } else if (context.sourceToParse().routing() != null) { - int routingHash = TimeSeriesRoutingHashFieldMapper.decode(context.sourceToParse().routing()); + int routingHash = RoutingPathHashFieldMapper.decode(context.sourceToParse().routing()); id = createId(routingHash, tsid, timestamp); } else { if (context.sourceToParse().id() == null) { @@ -162,7 +162,7 @@ public String documentDescription(ParsedDocument parsedDocument) { } private static String tsidDescription(IndexableField tsidField) { - String tsid = TimeSeriesIdFieldMapper.encodeTsid(tsidField.binaryValue()).toString(); + String tsid = RoutingPathFields.encode(tsidField.binaryValue()).toString(); if (tsid.length() <= DESCRIPTION_TSID_LIMIT) { return tsid; } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index ee24b8d9a9e91..cc9573b7bea09 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -185,7 +185,7 @@ import java.util.function.LongUnaryOperator; import java.util.function.Supplier; -import static org.elasticsearch.cluster.metadata.DataStream.TIMESERIES_LEAF_READERS_SORTER; +import static org.elasticsearch.cluster.metadata.DataStream.TIME_LEAF_READERS_SORTER; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.seqno.RetentionLeaseActions.RETAIN_ALL; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -3528,7 +3528,7 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) { replicationTracker::getRetentionLeases, this::getOperationPrimaryTerm, snapshotCommitSupplier, - isTimeBasedIndex ? TIMESERIES_LEAF_READERS_SORTER : null, + isTimeBasedIndex ? TIME_LEAF_READERS_SORTER : null, relativeTimeInNanosSupplier, indexCommitListener, routingEntry().isPromotableToPrimary(), diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 340bff4e1c852..785ecf5c4791f 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.mapper.IpScriptFieldType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.KeywordScriptFieldType; +import org.elasticsearch.index.mapper.LogsIdFieldMapper; import org.elasticsearch.index.mapper.LongScriptFieldType; import org.elasticsearch.index.mapper.LookupRuntimeFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -58,12 +59,12 @@ import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.RuntimeField; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -261,8 +262,9 @@ private static Map initBuiltInMetadataMa // (so will benefit from "fields: []" early termination builtInMetadataMappers.put(IdFieldMapper.NAME, IdFieldMapper.PARSER); builtInMetadataMappers.put(RoutingFieldMapper.NAME, RoutingFieldMapper.PARSER); + builtInMetadataMappers.put(LogsIdFieldMapper.NAME, LogsIdFieldMapper.PARSER); builtInMetadataMappers.put(TimeSeriesIdFieldMapper.NAME, TimeSeriesIdFieldMapper.PARSER); - builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER); + builtInMetadataMappers.put(RoutingPathHashFieldMapper.NAME, RoutingPathHashFieldMapper.PARSER); builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); builtInMetadataMappers.put(IndexModeFieldMapper.NAME, IndexModeFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 706f788e8a310..fd1dc57d6d44d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -776,7 +776,7 @@ private synchronized IndexService createIndexService( mapperRegistry, indicesFieldDataCache, namedWriteableRegistry, - idFieldMappers.apply(idxSettings.getMode()), + idxSettings.usesRoutingPath() ? idFieldMappers.apply(idxSettings.getMode()) : idFieldMappers.apply(IndexMode.STANDARD), valuesSourceRegistry, indexFoldersDeletionListeners, snapshotCommitSuppliers diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 8ac35f7c40caa..367203d6f832a 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -28,7 +28,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; @@ -933,7 +932,7 @@ public SourceLoader newSourceLoader() { @Override public IdLoader newIdLoader() { - if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { + if (indexService.getIndexSettings().usesRoutingPath()) { IndexRouting.ExtractFromSource indexRouting = null; List routingPaths = null; if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { @@ -954,7 +953,7 @@ public IdLoader newIdLoader() { } } } - return IdLoader.createTsIdLoader(indexRouting, routingPaths); + return IdLoader.createSyntheticIdLoader(indexRouting, routingPaths, indexService.getIndexSettings().getMode()); } else { return IdLoader.fromLeafStoredFieldLoader(); } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 51f52326907eb..efc13601ac436 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -787,4 +787,48 @@ private BytesRef parseBytesRefMap(Object value) { } } }; + + DocValueFormat LOGS_ID = new LogsIdDocValueFormat(); + + /** + * DocValues format for logs id. + */ + class LogsIdDocValueFormat implements DocValueFormat { + private static final Base64.Decoder BASE64_DECODER = Base64.getUrlDecoder(); + + private LogsIdDocValueFormat() {} + + @Override + public String getWriteableName() { + return "logs_id"; + } + + @Override + public void writeTo(StreamOutput out) {} + + @Override + public String toString() { + return "logs_id"; + } + + /** + * @param value The logs id as a {@link BytesRef} + * @return the Base 64 encoded id + */ + @Override + public Object format(BytesRef value) { + return TimeSeriesIdFieldMapper.encodeTsid(value); + } + + @Override + public BytesRef parseBytesRef(Object value) { + if (value instanceof BytesRef valueAsBytesRef) { + return valueAsBytesRef; + } + if (value instanceof String valueAsString) { + return new BytesRef(BASE64_DECODER.decode(valueAsString)); + } + throw new IllegalArgumentException("Cannot parse logs_id object [" + value + "]"); + } + }; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index fd39a95bdb75d..7af8849d151cc 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -275,7 +275,6 @@ import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; -import static org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_DOC_VALUE_FORMAT; /** * Sets up things that can be done at search time like queries, aggregations, and suggesters. @@ -1023,7 +1022,7 @@ private void registerValueFormats() { registerValueFormat(DocValueFormat.DENSE_VECTOR.getWriteableName(), in -> DocValueFormat.DENSE_VECTOR); registerValueFormat(DocValueFormat.UNSIGNED_LONG_SHIFTED.getWriteableName(), in -> DocValueFormat.UNSIGNED_LONG_SHIFTED); registerValueFormat(DocValueFormat.TIME_SERIES_ID.getWriteableName(), in -> DocValueFormat.TIME_SERIES_ID); - registerValueFormat(TS_ROUTING_HASH_DOC_VALUE_FORMAT.getWriteableName(), in -> TS_ROUTING_HASH_DOC_VALUE_FORMAT); + registerValueFormat(DocValueFormat.LOGS_ID.getWriteableName(), in -> DocValueFormat.LOGS_ID); } /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java index 5faf6e0aaaedf..d413fd2c88821 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java @@ -96,7 +96,7 @@ public int compareKey(Bucket other) { @Override protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { - if (format == DocValueFormat.TIME_SERIES_ID) { + if (format == DocValueFormat.TIME_SERIES_ID || format == DocValueFormat.LOGS_ID) { return builder.field(CommonFields.KEY.getPreferredName(), format.format(termBytes)); } return builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index 7f257172638c2..1a642d9aec9f4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -165,7 +165,7 @@ public void testDefaultsForTimeSeriesIndex() throws IOException { }); DocumentMapper mapper = createMapperService(getVersion(), indexSettings, () -> true, mapping).documentMapper(); - var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { + var source = source(RoutingPathHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { b.field("field", Base64.getEncoder().encodeToString(randomByteArrayOfLength(10))); b.field("@timestamp", "2000-10-10T23:40:53.384Z"); b.field("dimension", "dimension1"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 83553503c3c5e..ff80954c4306a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -271,7 +271,7 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { ParsedDocument doc = mapper.parse(source(null, b -> { b.array("field", true, false); b.field("@timestamp", Instant.now()); - }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + }, RoutingPathHashFieldMapper.encode(randomInt()))); assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } @@ -279,7 +279,7 @@ public void testDimensionMultiValuedFieldNonTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + }), IndexMode.STANDARD); ParsedDocument doc = mapper.parse(source(b -> { b.array("field", true, false); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 907a1a15721dc..bf91836bd85eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -124,7 +124,8 @@ private static Set builtInMetadataFields() { Set builtInMetadataFields = new HashSet<>(IndicesModule.getBuiltInMetadataFields()); // Index is not a time-series index, and it will not contain _tsid and _ts_routing_hash fields. builtInMetadataFields.remove(TimeSeriesIdFieldMapper.NAME); - builtInMetadataFields.remove(TimeSeriesRoutingHashFieldMapper.NAME); + builtInMetadataFields.remove(LogsIdFieldMapper.NAME); + builtInMetadataFields.remove(RoutingPathHashFieldMapper.NAME); return builtInMetadataFields; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 083efccceec16..8b46ddf86d020 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -28,6 +28,7 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -46,7 +47,7 @@ public class IdLoaderTests extends ESTestCase { private final int routingHash = randomInt(); public void testSynthesizeIdSimple() throws Exception { - var idLoader = IdLoader.createTsIdLoader(null, null); + var idLoader = IdLoader.createSyntheticIdLoader(null, null, IndexMode.TIME_SERIES); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( @@ -68,7 +69,7 @@ public void testSynthesizeIdSimple() throws Exception { } public void testSynthesizeIdMultipleSegments() throws Exception { - var idLoader = IdLoader.createTsIdLoader(null, null); + var idLoader = IdLoader.createSyntheticIdLoader(null, null, IndexMode.TIME_SERIES); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs1 = List.of( @@ -138,13 +139,14 @@ public void testSynthesizeIdMultipleSegments() throws Exception { } public void testSynthesizeIdRandom() throws Exception { - var idLoader = IdLoader.createTsIdLoader(null, null); + var idLoader = IdLoader.createSyntheticIdLoader(null, null, IndexMode.TIME_SERIES); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); Set expectedIDs = new HashSet<>(); List randomDocs = new ArrayList<>(); int numberOfTimeSeries = randomIntBetween(8, 64); for (int i = 0; i < numberOfTimeSeries; i++) { + long routingId = 0; int numberOfDimensions = randomIntBetween(1, 6); List dimensions = new ArrayList<>(numberOfDimensions); for (int j = 1; j <= numberOfDimensions; j++) { @@ -156,6 +158,7 @@ public void testSynthesizeIdRandom() throws Exception { value = randomAlphaOfLength(4); } dimensions.add(new Dimension(fieldName, value)); + routingId = value.hashCode(); } int numberOfSamples = randomIntBetween(1, 16); for (int j = 0; j < numberOfSamples; j++) { @@ -200,7 +203,7 @@ private void prepareIndexReader( } Sort sort = new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), - new SortField(TimeSeriesRoutingHashFieldMapper.NAME, SortField.Type.STRING, false), + new SortField(RoutingPathHashFieldMapper.NAME, SortField.Type.STRING, false), new SortedNumericSortField(DataStreamTimestampFieldMapper.DEFAULT_PATH, SortField.Type.LONG, true) ); config.setIndexSort(sort); @@ -231,12 +234,7 @@ private static void indexDoc(IndexWriter iw, Doc doc, int routingHash) throws IO } BytesRef tsid = routingFields.buildHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); - fields.add( - new SortedDocValuesField( - TimeSeriesRoutingHashFieldMapper.NAME, - Uid.encodeId(TimeSeriesRoutingHashFieldMapper.encode(routingHash)) - ) - ); + fields.add(new SortedDocValuesField(RoutingPathHashFieldMapper.NAME, Uid.encodeId(RoutingPathHashFieldMapper.encode(routingHash)))); iw.addDocument(fields); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 86c1157259790..9346a481f70ae 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -262,7 +262,7 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { ParsedDocument doc = mapper.parse(source(null, b -> { b.array("field", "192.168.1.1", "192.168.1.1"); b.field("@timestamp", Instant.now()); - }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + }, RoutingPathHashFieldMapper.encode(randomInt()))); assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } @@ -270,7 +270,7 @@ public void testDimensionMultiValuedFieldNonTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + }), IndexMode.STANDARD); ParsedDocument doc = mapper.parse(source(b -> { b.array("field", "192.168.1.1", "192.168.1.1"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 052bf995bdd48..2ccbbb875a1e1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -387,7 +387,7 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { ParsedDocument doc = mapper.parse(source(null, b -> { b.array("field", "1234", "45678"); b.field("@timestamp", Instant.now()); - }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + }, RoutingPathHashFieldMapper.encode(randomInt()))); assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } @@ -395,7 +395,7 @@ public void testDimensionMultiValuedFieldNonTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + }), IndexMode.STANDARD); ParsedDocument doc = mapper.parse(source(b -> { b.array("field", "1234", "45678"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingPathHashFieldMapperTests.java similarity index 88% rename from server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/RoutingPathHashFieldMapperTests.java index 0220a85727bcc..0b3d0171c0e1e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingPathHashFieldMapperTests.java @@ -23,11 +23,11 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -public class TimeSeriesRoutingHashFieldMapperTests extends MetadataMapperTestCase { +public class RoutingPathHashFieldMapperTests extends MetadataMapperTestCase { @Override protected String fieldName() { - return TimeSeriesRoutingHashFieldMapper.NAME; + return RoutingPathHashFieldMapper.NAME; } @Override @@ -57,7 +57,7 @@ private static ParsedDocument parseDocument(int hash, DocumentMapper docMapper, return docMapper.parse(source(null, b -> { f.accept(b); b.field("@timestamp", "2021-10-01"); - }, TimeSeriesRoutingHashFieldMapper.encode(hash))); + }, RoutingPathHashFieldMapper.encode(hash))); } private static ParsedDocument parseDocument(String id, DocumentMapper docMapper, CheckedConsumer f) @@ -70,8 +70,8 @@ private static ParsedDocument parseDocument(String id, DocumentMapper docMapper, } private static int getRoutingHash(ParsedDocument document) { - BytesRef value = document.rootDoc().getBinaryValue(TimeSeriesRoutingHashFieldMapper.NAME); - return TimeSeriesRoutingHashFieldMapper.decode(Uid.decodeId(value.bytes)); + BytesRef value = document.rootDoc().getBinaryValue(RoutingPathHashFieldMapper.NAME); + return RoutingPathHashFieldMapper.decode(Uid.decodeId(value.bytes)); } @SuppressWarnings("unchecked") @@ -92,7 +92,7 @@ public void testRetrievedFromIdInTimeSeriesMode() throws Exception { })); int hash = randomInt(); - ParsedDocument doc = parseDocument(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, docMapper, b -> b.field("a", "value")); + ParsedDocument doc = parseDocument(RoutingPathHashFieldMapper.DUMMY_ENCODED_VALUE, docMapper, b -> b.field("a", "value")); assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); assertEquals(0, getRoutingHash(doc)); } @@ -102,7 +102,7 @@ public void testDisabledInStandardMode() throws Exception { getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), mapping(b -> {}) ).documentMapper(); - assertThat(docMapper.metadataMapper(TimeSeriesRoutingHashFieldMapper.class), is(nullValue())); + assertThat(docMapper.metadataMapper(RoutingPathHashFieldMapper.class), is(nullValue())); ParsedDocument doc = docMapper.parse(source("id", b -> b.field("field", "value"), null)); assertThat(doc.rootDoc().getBinaryValue("_ts_routing_hash"), is(nullValue())); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index df6d9380fd141..8c7c874af32f2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -424,21 +424,33 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { public void testRecoverySourceWithLogs() throws IOException { { - Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).build(); - MapperService mapperService = createMapperService(settings, mapping(b -> {})); + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") + .build(); + MapperService mapperService = createMapperService(settings, fieldMapping(b -> { b.field("type", "keyword"); })); DocumentMapper docMapper = mapperService.documentMapper(); - ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); + ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) + ); } { Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") .put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false) .build(); - MapperService mapperService = createMapperService(settings, mapping(b -> {})); + MapperService mapperService = createMapperService(settings, fieldMapping(b -> { + b.field("type", "keyword"); + b.field("time_series_dimension", true); + })); DocumentMapper docMapper = mapperService.documentMapper(); - ParsedDocument doc = docMapper.parse(source(b -> b.field("@timestamp", "2012-02-13"))); + ParsedDocument doc = docMapper.parse( + source("123", b -> b.field("@timestamp", "2012-02-13").field("field", randomAlphaOfLength(5)), null) + ); assertNull(doc.rootDoc().getField("_recovery_source")); } } @@ -605,23 +617,45 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { } public void testRecoverySourceWithLogsCustom() throws IOException { - XContentBuilder mappings = topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject()); + String mappings = """ + { + "_doc" : { + "_source" : { + "mode" : "synthetic" + }, + "properties": { + "field": { + "type": "keyword" + } + } + } + } + """; { - Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).build(); + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") + .build(); MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); - ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); + ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); assertNotNull(doc.rootDoc().getField("_recovery_source")); - assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); + assertThat( + doc.rootDoc().getField("_recovery_source").binaryValue(), + equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) + ); } { Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") .put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false) .build(); MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); - ParsedDocument doc = docMapper.parse(source(b -> b.field("@timestamp", "2012-02-13"))); + ParsedDocument doc = docMapper.parse( + source("123", b -> b.field("@timestamp", "2012-02-13").field("field", randomAlphaOfLength(5)), null) + ); assertNull(doc.rootDoc().getField("_recovery_source")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 7f9474f5bab83..65d652b050b41 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -292,7 +292,7 @@ public void testStoreParameterDefaults() throws IOException { }); DocumentMapper mapper = createMapperService(getVersion(), indexSettings, () -> true, mapping).documentMapper(); - var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { + var source = source(RoutingPathHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { b.field("field", "1234"); if (timeSeriesIndexMode) { b.field("@timestamp", "2000-10-10T23:40:53.384Z"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 9d56938f185de..f761487af3916 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -110,7 +110,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ); assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); + assertEquals(RoutingPathFields.encode(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); } public void testDisabledInStandardMode() throws Exception { @@ -155,7 +155,7 @@ public void testStrings() throws IOException { docMapper, b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWE"); + assertEquals(RoutingPathFields.encode(new BytesArray(tsid).streamInput()), "AWE"); } @SuppressWarnings("unchecked") @@ -168,7 +168,7 @@ public void testUnicodeKeys() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); - Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + Object tsid = RoutingPathFields.encode(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); assertEquals(tsid, "A-I"); } @@ -179,7 +179,7 @@ public void testKeywordTooLong() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); + assertEquals(RoutingPathFields.encode(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } @SuppressWarnings("unchecked") @@ -190,7 +190,7 @@ public void testKeywordTooLongUtf8() throws IOException { String theWordLong = "長い"; ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); + assertEquals(RoutingPathFields.encode(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } public void testKeywordNull() throws IOException { @@ -229,7 +229,7 @@ public void testLong() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", 1234).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(RoutingPathFields.encode(new BytesArray(tsid).streamInput()), "AWFs"); } public void testLongInvalidString() throws IOException { @@ -281,7 +281,7 @@ public void testInteger() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(RoutingPathFields.encode(new BytesArray(tsid).streamInput()), "AWFs"); } public void testIntegerInvalidString() throws IOException { @@ -337,7 +337,7 @@ public void testShort() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Short.MIN_VALUE).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(RoutingPathFields.encode(new BytesArray(tsid).streamInput()), "AWFs"); } public void testShortInvalidString() throws IOException { @@ -393,7 +393,7 @@ public void testByte() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(RoutingPathFields.encode(new BytesArray(tsid).streamInput()), "AWFs"); } public void testByteInvalidString() throws IOException { @@ -449,7 +449,7 @@ public void testIp() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", "255.255.255.1").endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); + assertEquals(RoutingPathFields.encode(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); } public void testIpInvalidString() throws IOException { @@ -484,7 +484,7 @@ public void testVeryLarge() throws IOException { } }); - Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + Object tsid = RoutingPathFields.encode(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); assertEquals( tsid, "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd" @@ -720,7 +720,7 @@ public void testParseWithDynamicMapping() { "@timestamp": 1609459200000, "dim": "6a841a21", "value": 100 - }"""), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE); + }"""), XContentType.JSON, RoutingPathHashFieldMapper.DUMMY_ENCODED_VALUE); Engine.Index index = IndexShard.prepareIndex( mapper, source, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index a4a588930dcae..3661276cbfdf8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -718,7 +718,7 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C id, BytesReference.bytes(builder), builder.contentType(), - TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH) + RoutingPathHashFieldMapper.encode(ROUTING_HASH) ); return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 82dc0683fa98e..af314eafbc439 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -31,8 +31,8 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import org.elasticsearch.xcontent.XContentBuilder; @@ -210,7 +210,7 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { ParsedDocument doc = mapper.parse(source(null, b -> { b.array("field.key1", "value1", "value2"); b.field("@timestamp", Instant.now()); - }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + }, RoutingPathHashFieldMapper.encode(randomInt()))); assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } @@ -218,7 +218,7 @@ public void testDimensionMultiValuedFieldNonTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimensions", List.of("key1", "key2", "field3.key3")); - }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + }), IndexMode.STANDARD); ParsedDocument doc = mapper.parse(source(b -> { b.array("field.key1", "value1", "value2"); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index ab65d56557ad9..b1fff1a4a293f 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.IndexModeFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.LogsIdFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperRegistry; @@ -29,12 +30,12 @@ import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.RuntimeField; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.plugins.FieldPredicate; import org.elasticsearch.plugins.MapperPlugin; @@ -85,8 +86,9 @@ public Map getMetadataMappers() { IgnoredFieldMapper.NAME, IdFieldMapper.NAME, RoutingFieldMapper.NAME, + LogsIdFieldMapper.NAME, TimeSeriesIdFieldMapper.NAME, - TimeSeriesRoutingHashFieldMapper.NAME, + RoutingPathHashFieldMapper.NAME, IndexFieldMapper.NAME, IndexModeFieldMapper.NAME, SourceFieldMapper.NAME, diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index d039c265c98ae..84f650da1caac 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -56,8 +56,8 @@ import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; import org.elasticsearch.index.seqno.RetentionLeases; @@ -522,7 +522,7 @@ public Engine.Index createIndexOp(int docIdent) { { "@timestamp": %s, "dim": "dim" - }""", docIdent)), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE); + }""", docIdent)), XContentType.JSON, RoutingPathHashFieldMapper.DUMMY_ENCODED_VALUE); return IndexShard.prepareIndex( mapper, source, diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index a474c1dc38c50..6784e8974fe2f 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -479,7 +479,7 @@ public void testNewIdLoaderWithTsdb() throws Exception { .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") .build(); try (DefaultSearchContext context = createDefaultSearchContext(settings)) { - assertThat(context.newIdLoader(), instanceOf(IdLoader.TsIdLoader.class)); + assertThat(context.newIdLoader(), instanceOf(IdLoader.SyntheticIdLoader.class)); context.indexShard().getThreadPool().shutdown(); } } @@ -503,7 +503,7 @@ public void testNewIdLoaderWithTsdbAndRoutingPathMatch() throws Exception { }); try (DefaultSearchContext context = createDefaultSearchContext(settings, mappings)) { - assertThat(context.newIdLoader(), instanceOf(IdLoader.TsIdLoader.class)); + assertThat(context.newIdLoader(), instanceOf(IdLoader.SyntheticIdLoader.class)); context.indexShard().getThreadPool().shutdown(); } } diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index e81066a731d2e..426343366ab01 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.mapper.RoutingPathFields; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -395,11 +394,10 @@ public void testParseTsid() throws IOException { public void testFormatAndParseTsRoutingHash() throws IOException { BytesRef tsRoutingHashInput = new BytesRef("cn4exQ"); - DocValueFormat docValueFormat = TimeSeriesRoutingHashFieldMapper.INSTANCE.fieldType().docValueFormat(null, ZoneOffset.UTC); - Object formattedValue = docValueFormat.format(tsRoutingHashInput); + Object formattedValue = DocValueFormat.TIME_SERIES_ID.format(tsRoutingHashInput); // the format method takes BytesRef as input and outputs a String assertThat(formattedValue, instanceOf(String.class)); // the parse method will output the BytesRef input - assertThat(docValueFormat.parseBytesRef(formattedValue), is(tsRoutingHashInput)); + assertThat(DocValueFormat.TIME_SERIES_ID.parseBytesRef(formattedValue), is(tsRoutingHashInput)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index a297f5d13254b..c9dc8c3bd67de 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -83,7 +83,7 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { ParsedDocument doc = mapper.parse(source(null, b -> { b.array("field", randomNumber(), randomNumber(), randomNumber()); b.field("@timestamp", Instant.now()); - }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + }, RoutingPathHashFieldMapper.encode(randomInt()))); assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } @@ -91,7 +91,7 @@ public void testDimensionMultiValuedFieldNonTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + }), IndexMode.STANDARD); ParsedDocument doc = mapper.parse(source(b -> { b.array("field", randomNumber(), randomNumber(), randomNumber()); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index f554a84048fde..3ed0714f7821a 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -20,8 +20,8 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberTypeOutOfRangeSpec; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.RoutingPathHashFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesParams; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.WholeNumberFieldMapperTests; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xcontent.XContentBuilder; @@ -272,7 +272,7 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { ParsedDocument doc = mapper.parse(source(null, b -> { b.array("field", randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); b.field("@timestamp", Instant.now()); - }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + }, RoutingPathHashFieldMapper.encode(randomInt()))); assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } @@ -280,7 +280,7 @@ public void testDimensionMultiValuedFieldNonTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + }), IndexMode.STANDARD); ParsedDocument doc = mapper.parse(source(b -> { b.array("field", randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());