Merge branch '9.2' into backport/9.2/pr-138046

Author: benchaplin

docs/changelog/137702.yaml

@@ -0,0 +1,6 @@
+pr: 137702
+summary: Handle index deletion while querying in ES|QL
+area: ES|QL
+type: bug
+issues:
+ - 135863

docs/changelog/137992.yaml

@@ -0,0 +1,5 @@
+pr: 137992
+summary: Prevent passing a pipeline to a logs stream bulk index request body
+area: Data streams
+type: bug
+issues: []

docs/changelog/138132.yaml

@@ -0,0 +1,5 @@
+pr: 138132
+summary: Fix integer overflow in block memory estimation
+area: ES|QL
+type: bug
+issues: []

docs/reference/search-connectors/release-notes.md

@@ -13,13 +13,26 @@ If you are an Enterprise Search user and want to upgrade to Elastic 9.0, refer t
 It includes detailed steps, tooling, and resources to help you transition to supported alternatives in 9.x, such as Elasticsearch, the Open Web Crawler, and self-managed connectors.
 :::
 
+## 9.2.1 [connectors-9.2.1-release-notes]
+There are no new features, enhancements, fixes, known issues, or deprecations associated with this release.
+
 ## 9.2.0 [connectors-9.2.0-release-notes]
 
 ### Features and enhancements [connectors-9.2.0-features-enhancements]
 * Refactored pagination from OFFSET-based to keyset (primary-key) pagination in the MySQL connector. This delivers 3×+ faster syncs on large tables and modest gains on smaller ones. [#3719](https://github.com/elastic/connectors/pull/3719).
 
 * Updated the Jira connector to use the new `/rest/api/3/search/jql` endpoint, ensuring compatibility with Jira’s latest API. [#3710](https://github.com/elastic/connectors/pull/3710).
 
+## 9.1.7 [connectors-9.1.7-release-notes]
+There are no new features, enhancements, fixes, known issues, or deprecations associated with this release.
+
+## 9.1.6 [connectors-9.1.6-release-notes]
+
+### Features and enhancements [connectors-9.1.6-features-enhancements]
+* Idle Github connectors no longer excessively query set-up repositories, which reduces the number of calls to GitHub each connector makes and makes users less likely to hit GitHub API quotas. [#3708](https://github.com/elastic/connectors/pull/3708)
+
+* In the Sharepoint Online connector, /contentstorage/ URLs are no longer synced. [#3630](https://github.com/elastic/connectors/pull/3630)
+
 ## 9.1.5 [connectors-9.1.5-release-notes]
 
 ### Features and enhancements [connectors-9.1.5-features-enhancements]

docs/release-notes/breaking-changes.md

@@ -12,6 +12,37 @@ If you are migrating from a version prior to version 9.0, you must first upgrade
 
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
+## 9.2.1 [elasticsearch-9.2.1-breaking-changes]
+
+There are no breaking changes associated with this release.
+
+## 9.1.7 [elasticsearch-9.1.7-breaking-changes]
+
+There are no breaking changes associated with this release.
+
+## 9.1.6 [elasticsearch-9.1.6-breaking-changes]
+
+There are no breaking changes associated with this release.
+
+## 9.2.0 [elasticsearch-9.2.0-breaking-changes]
+
+Ingest Node:
+* Simulate API: Return 400 on invalid processor(s) [#130325](https://github.com/elastic/elasticsearch/pull/130325) (issue: [#120731](https://github.com/elastic/elasticsearch/issues/120731))
+
+Mapping:
+* Don't enable norms for fields of type text when the index mode is LogsDB or TSDB [#131317](https://github.com/elastic/elasticsearch/pull/131317)
+
+Vector Search:
+* Enable `exclude_source_vectors` by default for new indices [#131907](https://github.com/elastic/elasticsearch/pull/131907)
+
+## 9.0.8 [elasticsearch-9.0.8-breaking-changes]
+
+There are no breaking changes associated with this release.
+
+## 9.1.5 [elasticsearch-9.1.5-breaking-changes]
+
+There are no breaking changes associated with this release.
+
 ## 9.1.4 [elasticsearch-9.1.4-breaking-changes]
 
 There are no breaking changes associated with this release.

docs/release-notes/deprecations.md

@@ -16,6 +16,31 @@ To give you insight into what deprecated features you’re using, {{es}}:
 
 % ## Next version [elasticsearch-nextversion-deprecations]
 
+## 9.1.7 [elasticsearch-9.1.7-deprecations]
+
+There are no deprecations associated with this release.
+
+## 9.2.1 [elasticsearch-9.2.1-deprecations]
+
+There are no deprecations associated with this release.
+
+## 9.1.6 [elasticsearch-9.1.6-deprecations]
+
+There are no deprecations associated with this release.
+
+## 9.2.0 [elasticsearch-9.2.0-deprecations]
+
+Cluster Coordination:
+* Remove `PeerFinder` request timeout [#134365](https://github.com/elastic/elasticsearch/pull/134365)
+
+## 9.0.8 [elasticsearch-9.0.8-deprecations]
+
+There are no deprecations associated with this release.
+
+## 9.1.5 [elasticsearch-9.1.5-deprecations]
+
+There are no deprecations associated with this release.
+
 ## 9.1.4 [elasticsearch-9.1.4-deprecations]
 
 There are no deprecations associated with this release.

docs/release-notes/index.md

@@ -177,7 +177,9 @@ public void startTrace(TraceContext traceContext, Traceable traceable, String sp
 
             // A span can have a parent span, which here is modelled though a parent span context.
             // Setting this is important for seeing a complete trace in the APM UI.
-            final Context parentContext = getParentContext(traceContext);
+            // Attempt to fetch a local parent context first, otherwise look for a remote parent
+            final Context localParentContext = traceContext.getTransient(Task.PARENT_APM_TRACE_CONTEXT);
+            final Context parentContext = localParentContext != null ? localParentContext : getRemoteParentContext(traceContext);
             if (parentContext != null) {
                 spanBuilder.setParent(parentContext);
             }
@@ -188,21 +190,21 @@ public void startTrace(TraceContext traceContext, Traceable traceable, String sp
             if (startTime != null) {
                 spanBuilder.setStartTimestamp(startTime);
             }
+
             final Span span = spanBuilder.startSpan();
-            // If the agent decided not to record this span (e.g., due to transaction_max_spans), isRecording() will be false.
-            if (span.isRecording() == false) {
-                logger.trace("Span [{}] [{}] will not be recorded, e.g. due to transaction_max_spans reached", spanId, spanName);
-                // It's good practice to end the no-op span immediately to release any resources.
-                span.end();
-                // Returning null from computeIfAbsent means no value will be inserted into the map.
-                return null;
+            // If not a root span (meaning a local parent exists) and the agent decided not to record the span, discard it immediately.
+            // Root spans (transactions), however, have to be kept to correctly report their duration.
+            if (localParentContext != null && span.isRecording() == false) {
+                logger.trace("Span [{}] [{}] will not be recorded due to transaction_max_spans reached", spanId, spanName);
+                span.end(); // end span immediately to release any resources.
+                return null; // return null to discard and not record in map of spans
             }
 
-            // If we are here, the span is real and being recorded.
-            logger.trace("Successfully started tracing [{}] [{}]", spanId, spanName);
             final Context contextForNewSpan = Context.current().with(span);
-
-            updateThreadContext(traceContext, services, contextForNewSpan);
+            if (span.isRecording()) {
+                logger.trace("Recording trace [{}] [{}]", spanId, spanName);
+                updateThreadContext(traceContext, services, contextForNewSpan);
+            }
 
             return contextForNewSpan;
         });
@@ -240,30 +242,26 @@ private static void updateThreadContext(TraceContext traceContext, APMServices s
         });
     }
 
-    private Context getParentContext(TraceContext traceContext) {
+    private Context getRemoteParentContext(TraceContext traceContext) {
         // https://github.com/open-telemetry/opentelemetry-java/discussions/2884#discussioncomment-381870
         // If you just want to propagate across threads within the same process, you don't need context propagators (extract/inject).
         // You can just pass the Context object directly to another thread (it is immutable and thus thread-safe).
 
-        // Attempt to fetch a local parent context first, otherwise look for a remote parent
-        Context parentContext = traceContext.getTransient(Task.PARENT_APM_TRACE_CONTEXT);
-        if (parentContext == null) {
-            final String traceParentHeader = traceContext.getTransient(Task.PARENT_TRACE_PARENT_HEADER);
-            final String traceStateHeader = traceContext.getTransient(Task.PARENT_TRACE_STATE);
-
-            if (traceParentHeader != null) {
-                final Map<String, String> traceContextMap = Maps.newMapWithExpectedSize(2);
-                // traceparent and tracestate should match the keys used by W3CTraceContextPropagator
-                traceContextMap.put(Task.TRACE_PARENT_HTTP_HEADER, traceParentHeader);
-                if (traceStateHeader != null) {
-                    traceContextMap.put(Task.TRACE_STATE, traceStateHeader);
-                }
-                parentContext = services.openTelemetry.getPropagators()
-                    .getTextMapPropagator()
-                    .extract(Context.current(), traceContextMap, new MapKeyGetter());
+        final String traceParentHeader = traceContext.getTransient(Task.PARENT_TRACE_PARENT_HEADER);
+        final String traceStateHeader = traceContext.getTransient(Task.PARENT_TRACE_STATE);
+
+        if (traceParentHeader != null) {
+            final Map<String, String> traceContextMap = Maps.newMapWithExpectedSize(2);
+            // traceparent and tracestate should match the keys used by W3CTraceContextPropagator
+            traceContextMap.put(Task.TRACE_PARENT_HTTP_HEADER, traceParentHeader);
+            if (traceStateHeader != null) {
+                traceContextMap.put(Task.TRACE_STATE, traceStateHeader);
             }
+            return services.openTelemetry.getPropagators()
+                .getTextMapPropagator()
+                .extract(Context.current(), traceContextMap, new MapKeyGetter());
         }
-        return parentContext;
+        return null;
     }
 
     /**
@@ -288,7 +286,7 @@ private Context getParentContext(TraceContext traceContext) {
     @Override
     public Releasable withScope(Traceable traceable) {
         final Context context = spans.get(traceable.getSpanId());
-        if (context != null) {
+        if (context != null && Span.fromContextOrNull(context).isRecording()) {
             return context.makeCurrent()::close;
         }
         return () -> {};
@@ -385,9 +383,10 @@ public void setAttribute(Traceable traceable, String key, String value) {
 
     @Override
     public void stopTrace(Traceable traceable) {
-        final var span = Span.fromContextOrNull(spans.remove(traceable.getSpanId()));
+        final String spanId = traceable.getSpanId();
+        final var span = Span.fromContextOrNull(spans.remove(spanId));
         if (span != null) {
-            logger.trace("Finishing trace [{}]", traceable);
+            logger.trace("Finishing trace [{}]", spanId);
             span.end();
         }
     }

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.telemetry.apm.internal.APMAgentSettings;
+import org.elasticsearch.telemetry.tracing.TraceContext;
 import org.elasticsearch.telemetry.tracing.Traceable;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -42,6 +43,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -87,22 +89,29 @@ public void test_onTraceStarted_startsTrace() {
         Settings settings = Settings.builder().put(APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true).build();
         APMTracer apmTracer = buildTracer(settings);
 
-        apmTracer.startTrace(new ThreadContext(settings), TRACEABLE1, "name1", null);
+        ThreadContext traceContext = new ThreadContext(settings);
+        apmTracer.startTrace(traceContext, TRACEABLE1, "name1", null);
 
+        assertThat(traceContext.getTransient(Task.APM_TRACE_CONTEXT), notNullValue());
         assertThat(apmTracer.getSpans(), aMapWithSize(1));
         assertThat(apmTracer.getSpans(), hasKey(TRACEABLE1.getSpanId()));
     }
 
     /**
-     * Check that when a trace is started, but it is not recorded, e.g. due to sampling, the tracer does not record it either.
+     * Check that when a root trace is started, but it is not recorded, e.g. due to sampling,
+     * the tracer tracks it but doesn't start tracing.
      */
-    public void test_onTraceStarted_ifNotRecorded_doesNotStartTrace() {
+    public void test_onTraceStarted_ifNotRecorded_doesNotStartTracing() {
         Settings settings = Settings.builder().put(APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true).build();
         APMTracer apmTracer = buildTracer(settings);
 
-        apmTracer.startTrace(new ThreadContext(settings), TRACEABLE1, "name1_discard", null);
+        ThreadContext traceContext = new ThreadContext(settings);
+        apmTracer.startTrace(traceContext, TRACEABLE1, "name1_discard", null);
 
-        assertThat(apmTracer.getSpans(), anEmptyMap());
+        assertThat(traceContext.getTransient(Task.APM_TRACE_CONTEXT), nullValue());
+        // the root span (transaction) is tracked
+        assertThat(apmTracer.getSpans(), aMapWithSize(1));
+        assertThat(apmTracer.getSpans(), hasKey(TRACEABLE1.getSpanId()));
     }
 
     /**
@@ -116,8 +125,11 @@ public void test_onNestedTraceStarted_ifNotRecorded_doesNotStartTrace() {
         apmTracer.startTrace(traceContext, TRACEABLE1, "name1", null);
         try (var ignore1 = traceContext.newTraceContext()) {
             apmTracer.startTrace(traceContext, TRACEABLE2, "name2_discard", null);
+            assertThat(traceContext.getTransient(Task.APM_TRACE_CONTEXT), nullValue());
+
             try (var ignore2 = traceContext.newTraceContext()) {
                 apmTracer.startTrace(traceContext, TRACEABLE3, "name3_discard", null);
+                assertThat(traceContext.getTransient(Task.APM_TRACE_CONTEXT), nullValue());
             }
         }
         assertThat(apmTracer.getSpans(), aMapWithSize(1));
@@ -131,12 +143,13 @@ public void test_onTraceStartedWithStartTime_startsTrace() {
         Settings settings = Settings.builder().put(APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true).build();
         APMTracer apmTracer = buildTracer(settings);
 
-        ThreadContext threadContext = new ThreadContext(settings);
+        TraceContext traceContext = new ThreadContext(settings);
         // 1_000_000L because of "toNanos" conversions that overflow for large long millis
         Instant spanStartTime = Instant.ofEpochMilli(randomLongBetween(0, Long.MAX_VALUE / 1_000_000L));
-        threadContext.putTransient(Task.TRACE_START_TIME, spanStartTime);
-        apmTracer.startTrace(threadContext, TRACEABLE1, "name1", null);
+        traceContext.putTransient(Task.TRACE_START_TIME, spanStartTime);
+        apmTracer.startTrace(traceContext, TRACEABLE1, "name1", null);
 
+        assertThat(traceContext.getTransient(Task.APM_TRACE_CONTEXT), notNullValue());
         assertThat(apmTracer.getSpans(), aMapWithSize(1));
         assertThat(apmTracer.getSpans(), hasKey(TRACEABLE1.getSpanId()));
         assertThat(((SpyAPMTracer) apmTracer).getSpanStartTime("name1"), is(spanStartTime));
@@ -151,6 +164,7 @@ public void test_onTraceStopped_stopsTrace() {
 
         apmTracer.startTrace(new ThreadContext(settings), TRACEABLE1, "name1", null);
         apmTracer.stopTrace(TRACEABLE1);
+        apmTracer.stopTrace(TRACEABLE2); // stopping a non-existent trace is a noop
 
         assertThat(apmTracer.getSpans(), anEmptyMap());
     }

modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracerTests.java

@@ -0,0 +1,25 @@
+---
+teardown:
+  - do:
+      streams.logs_disable: { }
+
+---
+"Check User Can't Provide Pipeline to Logs Stream":
+  - do:
+      streams.logs_enable: { }
+  - is_true: acknowledged
+
+  - do:
+      streams.status: { }
+  - is_true: logs.enabled
+
+  - do:
+      bulk:
+        index: logs
+        body: |
+          { "create": {"pipeline": "noop"}}
+          { "message": "hello streams!"}
+  - match: { errors: true }
+  - match: { items.0.create.status: 400 }
+  - match: { items.0.create.error.type: "illegal_argument_exception" }
+  - match: { items.0.create.error.reason: "Cannot provide a pipeline when writing to a stream however the [noop] pipeline was provided when writing to the [logs] stream" }