Fix misrouting of /_license (and others) to ClickHouse (#1225)

In ElasticSearch there's a `GET /:index` endpoint. Quesma supports that
endpoint, however a `GET /_license` request erroneously matched to that
endpoint, routing it into ClickHouse instead of a fallback to
ElasticSearch.

Previously a couple similar endpoints (such as `/_nodes`, `/_xpack`,
`/_stats`) were hardcoded to be routed to Elastic, however that list was
not exhaustive and was prone to become obsolete if Elastic introduced a
new endpoint.

This PR changes the routing logic - now if an index name starts with `_`
(forbidden index name in Elastic and indicative of an endpoint name) it
is assumed that it is an internal path and should be routed to Elastic.

Author: avelanarius

ci/it/testcases/test_transparent_proxy.go

@@ -33,6 +33,7 @@ func (a *TransparentProxyIntegrationTestcase) RunTests(ctx context.Context, t *t
 	t.Run("test basic request", func(t *testing.T) { a.testBasicRequest(ctx, t) })
 	t.Run("test if cat health request reaches elasticsearch", func(t *testing.T) { a.testIfCatHealthRequestReachesElasticsearch(ctx, t) })
 	t.Run("test if index creation works", func(t *testing.T) { a.testIfIndexCreationWorks(ctx, t) })
+	t.Run("test internal endpoints", func(t *testing.T) { a.testInternalEndpoints(ctx, t) })
 	return nil
 }
 
@@ -64,3 +65,13 @@ func (a *TransparentProxyIntegrationTestcase) testIfIndexCreationWorks(ctx conte
 	assert.Contains(t, string(bodyBytes), "index_1")
 	assert.Contains(t, string(bodyBytes), "index_2")
 }
+
+func (a *TransparentProxyIntegrationTestcase) testInternalEndpoints(ctx context.Context, t *testing.T) {
+	for _, internalPath := range InternalPaths {
+		t.Run(internalPath, func(t *testing.T) {
+			resp, _ := a.RequestToQuesma(ctx, t, "GET", internalPath, nil)
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			assert.Equal(t, "Elasticsearch", resp.Header.Get("X-Elastic-Product"))
+		})
+	}
+}

ci/it/testcases/test_wildcard_clickhouse.go

@@ -33,6 +33,7 @@ func (a *WildcardClickhouseTestcase) RunTests(ctx context.Context, t *testing.T)
 	t.Run("test basic request", func(t *testing.T) { a.testBasicRequest(ctx, t) })
 	t.Run("test ingest+query works", func(t *testing.T) { a.testIngestQueryWorks(ctx, t) })
 	t.Run("test clickhouse table autodiscovery", func(t *testing.T) { a.testClickHouseTableAutodiscovery(ctx, t) })
+	t.Run("test internal endpoints", func(t *testing.T) { a.testInternalEndpoints(ctx, t) })
 	return nil
 }
 
@@ -94,3 +95,18 @@ func (a *WildcardClickhouseTestcase) testClickHouseTableAutodiscovery(ctx contex
 	// assert.Contains(t, string(bodyBytes), "Charlie")
 	// assert.Equal(t, "Elasticsearch", resp.Header.Get("X-Elastic-Product"))
 }
+
+// For full list, run "rg -o 'new Route\([^/{}"]*("/_[^/{}"]*(/[^{}/"]*)?")' | cut -d'"' -f2 | sort | uniq"
+// on the ES codebase.
+var InternalPaths = []string{"/_nodes", "/_xpack", "/_stats", "/_all/_stats", "/_license", "/_cat", "/_cat/health", "/_cluster/stats", "/_data_stream/_stats"}
+
+func (a *WildcardClickhouseTestcase) testInternalEndpoints(ctx context.Context, t *testing.T) {
+	for _, internalPath := range InternalPaths {
+		t.Run(internalPath, func(t *testing.T) {
+			resp, _ := a.RequestToQuesma(ctx, t, "GET", internalPath, nil)
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			assert.Equal(t, "Elasticsearch", resp.Header.Get("X-Quesma-Source"))
+			assert.Equal(t, "Elasticsearch", resp.Header.Get("X-Elastic-Product"))
+		})
+	}
+}

ci/it/testcases/test_wildcard_disabled.go

@@ -35,6 +35,7 @@ func (a *WildcardDisabledTestcase) RunTests(ctx context.Context, t *testing.T) e
 	t.Run("test ingest is disabled", func(t *testing.T) { a.testIngestIsDisabled(ctx, t) })
 	t.Run("test explicit index query enabled", func(t *testing.T) { a.testExplicitIndexQueryIsEnabled(ctx, t) })
 	t.Run("test explicit index ingest enabled", func(t *testing.T) { a.testExplicitIndexIngestIsEnabled(ctx, t) })
+	t.Run("test internal endpoints", func(t *testing.T) { a.testInternalEndpoints(ctx, t) })
 	return nil
 }
 
@@ -99,3 +100,14 @@ func (a *WildcardDisabledTestcase) testExplicitIndexIngestIsEnabled(ctx context.
 	assert.Equal(t, "Clickhouse", resp.Header.Get("X-Quesma-Source"))
 	assert.Equal(t, "Elasticsearch", resp.Header.Get("X-Elastic-Product"))
 }
+
+func (a *WildcardDisabledTestcase) testInternalEndpoints(ctx context.Context, t *testing.T) {
+	for _, internalPath := range InternalPaths {
+		t.Run(internalPath, func(t *testing.T) {
+			resp, _ := a.RequestToQuesma(ctx, t, "GET", internalPath, nil)
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			assert.Equal(t, "Elasticsearch", resp.Header.Get("X-Quesma-Source"))
+			assert.Equal(t, "Elasticsearch", resp.Header.Get("X-Elastic-Product"))
+		})
+	}
+}

quesma/elasticsearch/index.go

@@ -10,19 +10,17 @@ import (
 
 const (
 	internalIndexPrefix = "."
+	internalPathPrefix  = "_"
 )
 
 func IsIndexPattern(index string) bool {
 	return strings.ContainsAny(index, "*,")
 }
 
 func IsInternalIndex(index string) bool {
-	return strings.HasPrefix(index, internalIndexPrefix)
+	return strings.HasPrefix(index, internalIndexPrefix) || strings.HasPrefix(index, internalPathPrefix)
 }
 
-// InternalPaths is a list of paths that are considered internal and should not handled by Quesma
-var InternalPaths = []string{"/_nodes", "/_xpack"}
-
 func IsValidIndexName(name string) error {
 	const maxIndexNameLength = 256
 

quesma/frontend_connectors/elasticsearch_query.go

@@ -5,7 +5,6 @@ package frontend_connectors
 
 import (
 	"context"
-	"github.com/QuesmaOrg/quesma/quesma/elasticsearch"
 	"github.com/QuesmaOrg/quesma/quesma/processors/es_to_ch_common"
 	"github.com/QuesmaOrg/quesma/quesma/quesma/config"
 	quesma_api "github.com/QuesmaOrg/quesma/quesma/v2/core"
@@ -28,15 +27,6 @@ func NewElasticsearchQueryFrontendConnector(endpoint string, cfg *config.QuesmaC
 	}
 	router := quesma_api.NewPathRouter()
 
-	internalPaths := append(elasticsearch.InternalPaths, "/_stats")
-
-	for _, esInternalPath := range internalPaths {
-		router.Register(esInternalPath, quesma_api.Always(), func(ctx context.Context, req *quesma_api.Request, writer http.ResponseWriter) (*quesma_api.Result, error) {
-			metadata := quesma_api.MakeNewMetadata()
-			metadata[es_to_ch_common.Bypass] = true
-			return &quesma_api.Result{Meta: metadata, GenericResult: req.OriginalRequest}, nil
-		})
-	}
 	//TODO: Somehow this messes up the router, so we need to fix it
 	//router.Register(IndexPath, quesma_api.IsHTTPMethod("GET"), func(ctx context.Context, req *quesma_api.Request, writer http.ResponseWriter) (*quesma_api.Result, error) {
 	//	return &quesma_api.Result{Meta: metadata, GenericResult: req.OriginalRequest}, nil

quesma/quesma/router_test.go

@@ -46,14 +46,6 @@ func configureRouter(cfg *config.QuesmaConfiguration, sr schema.Registry, lm *cl
 
 	router := quesma_api.NewPathRouter()
 
-	// These are the endpoints that are not supported by Quesma
-	// These will redirect to the elastic cluster.
-	for _, path := range elasticsearch.InternalPaths {
-		router.Register(path, quesma_api.Never(), func(ctx context.Context, req *quesma_api.Request, _ http.ResponseWriter) (*quesma_api.Result, error) {
-			return nil, nil
-		})
-	}
-
 	// These are the endpoints that are supported by Quesma
 
 	// Warning:

quesma/quesma/router_v2.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"github.com/QuesmaOrg/quesma/quesma/backend_connectors"
 	"github.com/QuesmaOrg/quesma/quesma/clickhouse"
-	"github.com/QuesmaOrg/quesma/quesma/elasticsearch"
 	"github.com/QuesmaOrg/quesma/quesma/ingest"
 	"github.com/QuesmaOrg/quesma/quesma/logger"
 	"github.com/QuesmaOrg/quesma/quesma/painful"
@@ -32,14 +31,6 @@ func ConfigureIngestRouterV2(cfg *config.QuesmaConfiguration, dependencies quesm
 
 	router := quesma_api.NewPathRouter()
 
-	// These are the endpoints that are not supported by Quesma
-	// These will redirect to the elastic cluster.
-	for _, path := range elasticsearch.InternalPaths {
-		router.Register(path, quesma_api.Never(), func(ctx context.Context, req *quesma_api.Request, _ http.ResponseWriter) (*quesma_api.Result, error) {
-			return nil, nil
-		})
-	}
-
 	router.Register(routes.ExecutePainlessScriptPath, and(method("POST"), matchAgainstIndexNameInScriptRequestBody(tableResolver)), func(ctx context.Context, req *quesma_api.Request, _ http.ResponseWriter) (*quesma_api.Result, error) {
 
 		var scriptRequest painful.ScriptRequest
@@ -120,14 +111,6 @@ func ConfigureSearchRouterV2(cfg *config.QuesmaConfiguration, dependencies quesm
 
 	router := quesma_api.NewPathRouter()
 
-	// These are the endpoints that are not supported by Quesma
-	// These will redirect to the elastic cluster.
-	for _, path := range elasticsearch.InternalPaths {
-		router.Register(path, quesma_api.Never(), func(ctx context.Context, req *quesma_api.Request, _ http.ResponseWriter) (*quesma_api.Result, error) {
-			return nil, nil
-		})
-	}
-
 	// These are the endpoints that are supported by Quesma
 
 	// Warning: