[8.19](backport #47873) cleanup(otel): move otelbeat packages to x-pack/otel

x-pack/filebeat/fbreceiver/receiver_leak_test.go

@@ -19,7 +19,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zaptest/observer"
 
-	"github.com/elastic/beats/v7/libbeat/otelbeat/oteltest"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltest"
 )
 
 func TestLeak(t *testing.T) {

x-pack/filebeat/fbreceiver/receiver_test.go

@@ -32,7 +32,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zaptest/observer"
 
-	"github.com/elastic/beats/v7/libbeat/otelbeat/oteltest"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltest"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 

x-pack/filebeat/input/gcppubsub/otel_test.go

@@ -18,10 +18,10 @@ import (
 	"github.com/gofrs/uuid/v5"
 	"github.com/stretchr/testify/require"
 
-	"github.com/elastic/beats/v7/libbeat/otelbeat/oteltest"
 	"github.com/elastic/beats/v7/libbeat/tests/integration"
 	"github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub/testutil"
-	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/oteltestcol"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltest"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltestcol"
 
 	"github.com/elastic/elastic-agent-libs/testing/estools"
 )

x-pack/filebeat/tests/integration/otel_test.go

@@ -32,10 +32,10 @@ import (
 
 	"github.com/gofrs/uuid/v5"
 
-	"github.com/elastic/beats/v7/libbeat/otelbeat/oteltest"
 	libbeattesting "github.com/elastic/beats/v7/libbeat/testing"
 	"github.com/elastic/beats/v7/libbeat/tests/integration"
-	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/oteltestcol"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltest"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltestcol"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-libs/testing/estools"
 	"github.com/elastic/go-elasticsearch/v8"

x-pack/libbeat/cmd/instance/beat.go

@@ -25,7 +25,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/publisher/pipeline"
 	"github.com/elastic/beats/v7/libbeat/publisher/processing"
 	"github.com/elastic/beats/v7/libbeat/version"
-	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/otelmanager"
+	"github.com/elastic/beats/v7/x-pack/otel/otelmanager"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/keystore"
 	"github.com/elastic/elastic-agent-libs/logp"

x-pack/libbeat/cmd/instance/beat_test.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/elastic/beats/v7/filebeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/management"
-	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/otelmanager"
+	"github.com/elastic/beats/v7/x-pack/otel/otelmanager"
 )
 
 func TestManager(t *testing.T) {

x-pack/libbeat/cmd/instance/receiver.go

@@ -17,9 +17,9 @@ import (
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
 	"github.com/elastic/beats/v7/libbeat/common/backoff"
 	"github.com/elastic/beats/v7/libbeat/management/status"
-	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/otelmanager"
-	otelstatus "github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/status"
 	_ "github.com/elastic/beats/v7/x-pack/libbeat/include"
+	"github.com/elastic/beats/v7/x-pack/otel/otelmanager"
+	otelstatus "github.com/elastic/beats/v7/x-pack/otel/status"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/monitoring"
 	metricreport "github.com/elastic/elastic-agent-system-metrics/report"

x-pack/libbeat/outputs/otelconsumer/otelconsumer.go

@@ -14,9 +14,9 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/otelbeat/otelmap"
 	"github.com/elastic/beats/v7/libbeat/outputs"
 	"github.com/elastic/beats/v7/libbeat/publisher"
+	"github.com/elastic/beats/v7/x-pack/otel/otelmap"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"

x-pack/metricbeat/mbreceiver/receiver_leak_test.go

@@ -21,7 +21,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zaptest/observer"
 
-	"github.com/elastic/beats/v7/libbeat/otelbeat/oteltest"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltest"
 )
 
 func TestLeak(t *testing.T) {

x-pack/metricbeat/mbreceiver/receiver_test.go

@@ -22,15 +22,14 @@ import (
 	"strings"
 	"testing"
 
-	"go.opentelemetry.io/collector/pdata/pcommon"
-
-	"github.com/elastic/beats/v7/libbeat/otelbeat/oteltest"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltest"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/component/componentstatus"
+	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/receiver"
 
 	"go.uber.org/zap"

x-pack/metricbeat/tests/integration/otel_test.go

@@ -24,7 +24,7 @@ import (
 
 	libbeattesting "github.com/elastic/beats/v7/libbeat/testing"
 	"github.com/elastic/beats/v7/libbeat/tests/integration"
-	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/oteltestcol"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltestcol"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-libs/testing/estools"
 )

libbeat/otelbeat/beatconverter/beatconverter.go → x-pack/otel/beatconverter/beatconverter.go

@@ -1,19 +1,6 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
 
 package beatconverter
 
@@ -24,8 +11,8 @@ import (
 	"go.opentelemetry.io/collector/confmap"
 
 	"github.com/elastic/beats/v7/libbeat/cloudid"
-	elasticsearchtranslate "github.com/elastic/beats/v7/libbeat/otelbeat/oteltranslate/outputs/elasticsearch"
 	"github.com/elastic/beats/v7/libbeat/outputs/elasticsearch"
+	elasticsearchtranslate "github.com/elastic/beats/v7/x-pack/otel/oteltranslate/outputs/elasticsearch"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 )

libbeat/otelbeat/beatconverter/beatconverter_test.go → x-pack/otel/beatconverter/beatconverter_test.go

@@ -1,19 +1,6 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
 
 package beatconverter
 

x-pack/otel/beatconverter/logging.go

@@ -0,0 +1,25 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package beatconverter
+
+import (
+	"fmt"
+	"strings"
+)
+
+func getOTelLogLevel(level string) (string, error) {
+	switch strings.ToLower(level) {
+	case "debug":
+		return "DEBUG", nil
+	case "info":
+		return "INFO", nil
+	case "warning":
+		return "WARN", nil
+	case "error", "critical":
+		return "ERROR", nil
+	default:
+		return "", fmt.Errorf("unrecognized level: %s", level)
+	}
+}

x-pack/libbeat/common/otelbeat/beatsuth_test.go → x-pack/otel/extension/beatsauthextension/beatsauth_test.go

@@ -4,7 +4,7 @@
 
 //go:build !requirefips
 
-package otelbeat
+package beatsauthextension_test
 
 import (
 	"bytes"
@@ -42,9 +42,9 @@ import (
 
 	"gopkg.in/yaml.v2"
 
-	"github.com/elastic/beats/v7/libbeat/otelbeat/beatconverter"
-	"github.com/elastic/beats/v7/libbeat/otelbeat/oteltest"
+	"github.com/elastic/beats/v7/x-pack/otel/beatconverter"
 	"github.com/elastic/beats/v7/x-pack/otel/extension/beatsauthextension"
+	"github.com/elastic/beats/v7/x-pack/otel/oteltest"
 	"github.com/elastic/elastic-agent-libs/testing/proxytest"
 	"github.com/elastic/elastic-agent-libs/transport/tlscommontest"
 	mockes "github.com/elastic/mock-es/pkg/api"
@@ -79,7 +79,6 @@ type options struct {
 
 // tests mutual TLS
 func TestMTLS(t *testing.T) {
-
 	// create server certificates
 	serverCerts, err := tlscommontest.GenSignedCert(caCert, x509.KeyUsageCertSign, false, "server", []string{"localhost"}, []net.IP{net.IPv4(127, 0, 0, 1)}, false)
 	if err != nil {
@@ -150,10 +149,12 @@ receivers:
 
 	// check if data has reached ES
 	assertReceivedLogRecord(t, metricReader)
+
+	require.NoError(t, exp.Shutdown(t.Context()), "error shutting down exporter")
+	require.NoError(t, beatsauth.Shutdown(t.Context()), "error shutting down extension")
 }
 
 func TestKeyPassPhrase(t *testing.T) {
-
 	// create server certificates
 	serverCerts, err := tlscommontest.GenSignedCert(caCert, x509.KeyUsageCertSign, false, "server", []string{"localhost"}, []net.IP{net.IPv4(127, 0, 0, 1)}, false)
 	if err != nil {
@@ -225,6 +226,9 @@ receivers:
 
 	// check if data has reached ES
 	assertReceivedLogRecord(t, metricReader)
+
+	require.NoError(t, exp.Shutdown(t.Context()), "error shutting down exporter")
+	require.NoError(t, beatsauth.Shutdown(t.Context()), "error shutting down extension")
 }
 
 // tests ca_trusted_fingerprint
@@ -290,12 +294,14 @@ receivers:
 
 	// check if data has reached ES
 	assertReceivedLogRecord(t, metricReader)
+
+	require.NoError(t, exp.Shutdown(t.Context()), "error shutting down exporter")
+	require.NoError(t, beatsauth.Shutdown(t.Context()), "error shutting down extension")
 }
 
 // tests verification mode
 // The test scenarios are taken from https://github.com/elastic/elastic-agent-libs/blob/main/transport/tlscommon/tls_config_test.go#L495
 func TestVerificationMode(t *testing.T) {
-
 	testcases := map[string]struct {
 		verificationMode string
 		expectingError   bool
@@ -477,12 +483,16 @@ receivers:
 
 			if test.expectingError {
 				require.Error(t, err, "expected error got none")
+				require.NoError(t, exp.Shutdown(t.Context()), "error shutting down exporter")
+				require.NoError(t, beatsauth.Shutdown(t.Context()), "error shutting down extension")
 				return
 			}
 
 			// check if data has reached ES
 			assertReceivedLogRecord(t, metricReader)
 
+			require.NoError(t, exp.Shutdown(t.Context()), "error shutting down exporter")
+			require.NoError(t, beatsauth.Shutdown(t.Context()), "error shutting down extension")
 		})
 	}
 
@@ -491,7 +501,6 @@ receivers:
 // TestProxyHTTPS tests proxy_url with http and https proxy server
 // It also tests proxy_disable configuration
 func TestProxyHTTP(t *testing.T) {
-
 	// caCert cert pool
 	certPool := x509.NewCertPool()
 	certPool.AddCert(caCert.Leaf)
@@ -589,6 +598,7 @@ receivers:
 
 			serverName, metricReader := startTestServer(t, test.serverTLSConfig)
 			proxy := proxytest.New(t, test.proxyOptions...)
+			defer proxy.Close()
 
 			if test.serverTLSConfig != nil {
 				require.NoErrorf(t, proxy.StartTLS(), "error starting proxy server")
@@ -637,6 +647,9 @@ receivers:
 			} else {
 				assert.Empty(t, proxy.ProxiedRequests(), "proxy should have captured at least 1 request")
 			}
+
+			require.NoError(t, exp.Shutdown(t.Context()), "error shutting down exporter")
+			require.NoError(t, beatsauth.Shutdown(t.Context()), "error shutting down extension")
 		})
 	}
 

libbeat/otelbeat/otelmap/otelmap.go → x-pack/otel/otelmap/otelmap.go

@@ -1,19 +1,6 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
 
 // Package otelmap provides utilities for converting between beats and otel map types.
 package otelmap

libbeat/otelbeat/otelmap/otelmap_test.go → x-pack/otel/otelmap/otelmap_test.go

@@ -1,19 +1,6 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
 
 package otelmap