Skip to content

Commit 5660d43

Browse files
breskebybreskeby
authored
Merge branch '8.19' into update-docker-default-base
2 parents bd095c9 + b0beb54 commit 5660d43

File tree

209 files changed

+4644
-833
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

209 files changed

+4644
-833
lines changed

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
4646

4747
private Boolean apmServerEnabled = false;
4848

49-
private List<String> plugins = List.of();
49+
private List<String> plugins;
5050

5151
private Boolean preserveData = false;
5252

@@ -115,7 +115,12 @@ public void setPlugins(String plugins) {
115115
}
116116
}
117117

118+
public void setPlugins(List<String> plugins) {
119+
this.plugins = plugins;
120+
}
121+
118122
@Input
123+
@Optional
119124
public List<String> getPlugins() {
120125
return plugins;
121126
}

‎docs/changelog/126581.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
pr: 126581
2+
summary: "Optimize shared blob cache evictions on shard removal
3+
Shared blob cache evictions occur on the cluster applier thread when shards are
4+
removed from a node. These can be expensive if a large number of shards are
5+
being removed. This change uses the context of the removal to avoid unnecessary
6+
evictions that might hold up the applier thread.
7+
"
8+
area: Snapshot/Restore
9+
type: enhancement
10+
issues: []

‎docs/changelog/127532.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 127532
2+
summary: Fix case insensitive comparisons to ""
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 127431

‎docs/changelog/127767.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127767
2+
summary: Integrate `OpenAi` Chat Completion in `SageMaker`
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/127966.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127966
2+
summary: "[ML] Add Rerank support to the Inference Plugin"
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/128163.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
pr: 128163
2+
summary: Make `skip_unavailable` catch all errors
3+
area: ES|QL
4+
type: breaking
5+
issues: [ ]
6+
breaking:
7+
title: Cluster setting "skip_unavailable" catches all runtime errors
8+
area: ES|QL
9+
details: "If `skip_unavailable` is set to `true`, the runtime errors from this cluster\
10+
\ do not lead to a failure of the query. Instead, the cluster is set to `skipped`\
11+
\ or `partial` status, and the query execution continues. This is a breaking change\
12+
\ from previous versions, where `skip_unavailable` only applied to errors related\
13+
\ to a cluster being unavailable."
14+
impact: "The errors on remote clusters, e.g. missing indices, will not lead to a\
15+
\ failure of the query. Instead, the cluster is set to `skipped` or `partial` status\
16+
\ in the response metadata."
17+
notable: false

‎docs/changelog/128218.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 128218
2+
summary: Improve exception for trained model deployment scale up timeout
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/128241.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 128241
2+
summary: Adding VoyageAI's v3.5 models
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/128473.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 128473
2+
summary: Conditionally force sequential reading in `LuceneSyntheticSourceChangesSnapshot`
3+
area: Logs
4+
type: enhancement
5+
issues: []

‎docs/changelog/128584.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 128584
2+
summary: '`InferenceService` support aliases'
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/internal/DistributedArchitectureGuide.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ to communicate with Elasticsearch.
9090

9191
(Sketch of important classes? Might inform more sections to add for details.)
9292

93-
(A NodeB can coordinate a search across several other nodes, when NodeB itself does not have the data, and then return a result to the caller. Explain this coordinating role)
93+
(A node can coordinate a search across several other nodes, when the node itself does not have the data, and then return a result to the caller. Explain this coordinating role)
9494

9595
### Node Roles
9696

‎docs/reference/release-notes/8.17.0.asciidoc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ Machine Learning::
8181
* Mitigate IOSession timeouts {es-pull}115414[#115414] (issues: {es-issue}114385[#114385], {es-issue}114327[#114327], {es-issue}114105[#114105], {es-issue}114232[#114232])
8282
* Propagate scoring function through random sampler {es-pull}116957[#116957] (issue: {es-issue}110134[#110134])
8383
* Wait for the worker service to shutdown before closing task processor {es-pull}117920[#117920] (issue: {es-issue}117563[#117563])
84+
* Increase the upper limits for the Boost.JSON SAX parser {ml-pull}2809[#2809]
85+
* Correct handling of config updates {ml-pull}2821[#2821]
8486

8587
Mapping::
8688
* Address mapping and compute engine runtime field issues {es-pull}117792[#117792] (issue: {es-issue}117644[#117644])

‎docs/reference/release-notes/8.18.0.asciidoc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,10 @@ Machine Learning::
126126
* Set default similarity for Cohere model to cosine {es-pull}125370[#125370] (issue: {es-issue}122878[#122878])
127127
* Updating Inference Update API documentation to have the correct PUT method {es-pull}121048[#121048]
128128
* [Inference API] Fix output stream ordering in `InferenceActionProxy` {es-pull}124225[#124225]
129+
* Restrict file system access for PyTorch models {ml-pull}2851[#2851]
130+
* Update the PyTorch library to version 2.5.1. {ml-pull}2783[#2798], {ml-pull}2799[#2799]
131+
* Upgrade Boost libraries to version 1.86. {ml-pull}2780[#2780], {ml-pull}2779[#2779]
132+
129133

130134
Mapping::
131135
* Avoid serializing empty `_source` fields in mappings {es-pull}122606[#122606]

‎docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -60,23 +60,41 @@ measure the performance characteristics of your storage system.
6060
The default values for the parameters to this API are deliberately low to reduce
6161
the impact of running an analysis inadvertently and to provide a sensible
6262
starting point for your investigations. Run your first analysis with the default
63-
parameter values to check for simple problems. If successful, run a sequence of
64-
increasingly large analyses until you encounter a failure or you reach a
65-
`blob_count` of at least `2000`, a `max_blob_size` of at least `2gb`, a
66-
`max_total_data_size` of at least `1tb`, and a `register_operation_count` of at
67-
least `100`. Always specify a generous timeout, possibly `1h` or longer, to
68-
allow time for each analysis to run to completion. Perform the analyses using a
69-
multi-node cluster of a similar size to your production cluster so that it can
70-
detect any problems that only arise when the repository is accessed by many
71-
nodes at once.
63+
parameter values to check for simple problems. Some repositories may behave
64+
correctly when lightly loaded but incorrectly under production-like workloads.
65+
If the first analysis is successful, run a sequence of increasingly large
66+
analyses until you encounter a failure or you reach a `blob_count` of at least
67+
`2000`, a `max_blob_size` of at least `2gb`, a `max_total_data_size` of at least
68+
`1tb`, and a `register_operation_count` of at least `100`. Always specify a
69+
generous timeout, possibly `1h` or longer, to allow time for each analysis to
70+
run to completion. Some repositories may behave correctly when accessed by a
71+
small number of {es} nodes but incorrectly when accessed concurrently by a
72+
production-scale cluster. Perform the analyses using a multi-node cluster of a
73+
similar size to your production cluster so that it can detect any problems that
74+
only arise when the repository is accessed by many nodes at once.
7275

7376
If the analysis fails then {es} detected that your repository behaved
74-
unexpectedly. This usually means you are using a third-party storage system
75-
with an incorrect or incompatible implementation of the API it claims to
76-
support. If so, this storage system is not suitable for use as a snapshot
77-
repository. You will need to work with the supplier of your storage system to
78-
address the incompatibilities that {es} detects. See
79-
<<self-managed-repo-types>> for more information.
77+
unexpectedly. This usually means you are using a third-party storage system with
78+
an incorrect or incompatible implementation of the API it claims to support. If
79+
so, this storage system is not suitable for use as a snapshot repository.
80+
Repository analysis triggers conditions that occur only rarely when taking
81+
snapshots in a production system. Snapshotting to unsuitable storage may appear
82+
to work correctly most of the time despite repository analysis failures. However
83+
your snapshot data is at risk if you store it in a snapshot repository that does
84+
not reliably pass repository analysis. You can demonstrate that the analysis
85+
failure is due to an incompatible storage implementation by verifying that
86+
Elasticsearch does not detect the same problem when analysing the reference
87+
implementation of the storage protocol you are using. For instance, if you are
88+
using storage that offers an API which the supplier claims to be compatible with
89+
AWS S3, verify that repositories in AWS S3 do not fail repository analysis. This
90+
allows you to demonstrate to your storage supplier that a repository analysis
91+
failure must only be caused by an incompatibility with AWS S3 and cannot be
92+
attributed to a problem in Elasticsearch. Please do not report Elasticsearch
93+
issues involving third-party storage systems unless you can demonstrate that the
94+
same issue exists when analysing a repository that uses the reference
95+
implementation of the same storage protocol. You will need to work with the
96+
supplier of your storage system to address the incompatibilities that {es}
97+
detects. See <<self-managed-repo-types>> for more information.
8098

8199
If the analysis is successful this API returns details of the testing process,
82100
optionally including how long each operation took. You can use this information

‎docs/reference/snapshot-restore/repository-s3.asciidoc

Lines changed: 19 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@ The following list contains the available client settings. Those that must be
105105
stored in the keystore are marked as "secure" and are *reloadable*; the other
106106
settings belong in the `elasticsearch.yml` file.
107107

108+
`region`::
109+
Specifies the region to use. When set, determines the signing region and regional endpoint to use, unless the endpoint
110+
is overridden via the `endpoint` setting. If not set, {es} will attempt to determine the region automatically using
111+
the AWS SDK.
112+
108113
`access_key` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
109114

110115
An S3 access key. If set, the `secret_key` setting must also be specified.
@@ -121,22 +126,23 @@ settings belong in the `elasticsearch.yml` file.
121126

122127
`endpoint`::
123128

124-
The S3 service endpoint to connect to. This defaults to `s3.amazonaws.com`
125-
but the
126-
https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region[AWS
127-
documentation] lists alternative S3 endpoints. If you are using an
128-
<<repository-s3-compatible-services,S3-compatible service>> then you should
129-
set this to the service's endpoint.
129+
The S3 service endpoint to connect to. This defaults to the regional endpoint corresponding to the configured `region`,
130+
but the https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region[AWS documentation] lists alternative S3
131+
endpoints. If you are using an <<repository-s3-compatible-services,S3-compatible service>> then you should set this
132+
to the service’s endpoint. The endpoint should specify the protocol and host name,
133+
e.g. `https://s3.ap-southeast-4.amazonaws.com`, `http://minio.local:9000`.
134+
+
135+
When using HTTPS, this repository type validates the repository’s certificate chain using the JVM-wide truststore.
136+
Ensure that the root certificate authority is in this truststore using the JVM’s `keytool` tool. If you have a custom
137+
certificate authority for your S3 repository and you use the {es} <<jvm-version,bundled JDK>>, then you will need to
138+
reinstall your CA certificate every time you upgrade {es}.
139+
130140

131141
`protocol`::
132142

133-
The protocol to use to connect to S3. Valid values are either `http` or
134-
`https`. Defaults to `https`. When using HTTPS, this repository type validates the
135-
repository's certificate chain using the JVM-wide truststore. Ensure that
136-
the root certificate authority is in this truststore using the JVM's
137-
`keytool` tool. If you have a custom certificate authority for your S3 repository
138-
and you use the {es} <<jvm-version,bundled JDK>>, then you will need to reinstall your
139-
CA certificate every time you upgrade {es}.
143+
The protocol to use to connect to S3. Valid values are either `http` or `https`. Defaults to `https`. Note that this
144+
setting is deprecated since 8.19 and is only used if `endpoint` is set to a URL that does not include a scheme.
145+
Users should migrate to including the scheme in the `endpoint` setting.
140146

141147
`proxy.host`::
142148

@@ -175,11 +181,6 @@ settings belong in the `elasticsearch.yml` file.
175181
The number of retries to use when an S3 request fails. The default value is
176182
`3`.
177183

178-
`use_throttle_retries`::
179-
180-
Whether retries should be throttled (i.e. should back off). Must be `true`
181-
or `false`. Defaults to `true`.
182-
183184
`path_style_access`::
184185

185186
Whether to force the use of the path style access pattern. If `true`, the
@@ -205,23 +206,6 @@ pattern then you should set this setting to `true` when upgrading.
205206
https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#disableChunkedEncoding--[AWS
206207
Java SDK documentation] for details. Defaults to `false`.
207208

208-
`region`::
209-
210-
Allows specifying the signing region to use. Specificing this setting manually should not be necessary for most use cases. Generally,
211-
the SDK will correctly guess the signing region to use. It should be considered an expert level setting to support S3-compatible APIs
212-
that require https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html[v4 signatures] and use a region other than the
213-
default `us-east-1`. Defaults to empty string which means that the SDK will try to automatically determine the correct signing region.
214-
215-
`signer_override`::
216-
217-
Allows specifying the name of the signature algorithm to use for signing requests by the S3 client. Specifying this setting should not
218-
be necessary for most use cases. It should be considered an expert level setting to support S3-compatible APIs that do not support the
219-
signing algorithm that the SDK automatically determines for them.
220-
See the
221-
https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ClientConfiguration.html#setSignerOverride-java.lang.String-[AWS
222-
Java SDK documentation] for details. Defaults to empty string which means that no signing algorithm override will be used.
223-
224-
225209
[[repository-s3-repository]]
226210
==== Repository settings
227211

‎libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentString.java

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,6 @@
2121

2222
import java.util.Locale;
2323

24-
import static org.elasticsearch.repositories.s3.S3Service.REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING;
25-
import static org.elasticsearch.repositories.s3.S3Service.REPOSITORY_S3_CAS_TTL_SETTING;
26-
2724
@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
2825
@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
2926
public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase {
@@ -42,10 +39,6 @@ public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS
4239
.keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY)
4340
.keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY)
4441
.setting("s3.client." + CLIENT + ".endpoint", minioFixture::getAddress)
45-
// Skip listing of pre-existing uploads during a CAS because MinIO sometimes leaks them; also reduce the delay before proceeding
46-
// TODO do not set these if running a MinIO version in which https://github.com/minio/minio/issues/21189 is fixed
47-
.setting(REPOSITORY_S3_CAS_TTL_SETTING.getKey(), "-1")
48-
.setting(REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING.getKey(), "100ms")
4942
.build();
5043

5144
@ClassRule

‎muted-tests.yml

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -458,18 +458,9 @@ tests:
458458
- class: org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderIT
459459
method: testSearchWhileRelocating
460460
issue: https://github.com/elastic/elasticsearch/issues/127188
461-
- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
462-
method: test {p0=search.vectors/41_knn_search_byte_quantized/Test index configured rescore vector score consistency}
463-
issue: https://github.com/elastic/elasticsearch/issues/128077
464461
- class: org.elasticsearch.compute.operator.ChangePointOperatorTests
465462
method: testSimpleFinishClose
466463
issue: https://github.com/elastic/elasticsearch/issues/127106
467464
- class: org.elasticsearch.search.CCSDuelIT
468465
method: testTerminateAfter
469466
issue: https://github.com/elastic/elasticsearch/issues/126085
470-
- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
471-
method: test {yaml=search.vectors/41_knn_search_byte_quantized/Test index configured rescore vector score consistency}
472-
issue: https://github.com/elastic/elasticsearch/issues/128353
473-
- class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT
474-
method: testEqualityAndOther {semantic_text}
475-
issue: https://github.com/elastic/elasticsearch/issues/128414

‎plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
import org.apache.lucene.search.IndexSearcher;
1313
import org.apache.lucene.util.CharsRefBuilder;
14-
import org.elasticsearch.xcontent.Text;
14+
import org.elasticsearch.common.text.Text;
1515
import org.elasticsearch.search.suggest.Suggest;
1616
import org.elasticsearch.search.suggest.Suggester;
1717

‎plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
import org.elasticsearch.common.io.stream.StreamInput;
1313
import org.elasticsearch.common.io.stream.StreamOutput;
14-
import org.elasticsearch.xcontent.Text;
14+
import org.elasticsearch.common.text.Text;
1515
import org.elasticsearch.search.suggest.Suggest;
1616
import org.elasticsearch.xcontent.ParseField;
1717
import org.elasticsearch.xcontent.XContentBuilder;

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -884,22 +884,3 @@ setup:
884884
- match: { hits.hits.1._score: $default_rescore1 }
885885
- match: { hits.hits.2._score: $override_score2 }
886886
- match: { hits.hits.2._score: $default_rescore2 }
887-
888-
- do:
889-
headers:
890-
Content-Type: application/json
891-
search:
892-
rest_total_hits_as_int: true
893-
index: int8_rescore_zero_hnsw
894-
body:
895-
knn:
896-
field: vector
897-
query_vector: [-0.5, 90.0, -10, 14.8]
898-
k: 3
899-
num_candidates: 3
900-
901-
# Compare scores as hit IDs may change depending on how things are distributed
902-
- match: { hits.total: 3 }
903-
- match: { hits.hits.0._score: $raw_score0 }
904-
- match: { hits.hits.1._score: $raw_score1 }
905-
- match: { hits.hits.2._score: $raw_score2 }

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -919,22 +919,3 @@ setup:
919919
- match: { hits.hits.1._score: $default_rescore1 }
920920
- match: { hits.hits.2._score: $override_score2 }
921921
- match: { hits.hits.2._score: $default_rescore2 }
922-
923-
- do:
924-
headers:
925-
Content-Type: application/json
926-
search:
927-
rest_total_hits_as_int: true
928-
index: int4_rescore_zero_hnsw
929-
body:
930-
knn:
931-
field: vector
932-
query_vector: [-0.5, 90.0, -10, 14.8]
933-
k: 3
934-
num_candidates: 3
935-
936-
# Compare scores as hit IDs may change depending on how things are distributed
937-
- match: { hits.total: 3 }
938-
- match: { hits.hits.0._score: $raw_score0 }
939-
- match: { hits.hits.1._score: $raw_score1 }
940-
- match: { hits.hits.2._score: $raw_score2 }

0 commit comments

Comments
 (0)