Merging to release-5.8: [TT-12238] Javascript regex issue on OAS API (#7923)

gateway/api.go

@@ -1182,11 +1182,21 @@ func (gw *Gateway) handleGetAPIOAS(apiID string, modePublic bool) (interface{},
 	defer gw.apisMu.RUnlock()
 
 	obj, code := gw.handleGetAPI(apiID, true)
-	if apiOAS, ok := obj.(*oas.OAS); ok && modePublic {
-		apiOAS.RemoveTykExtension()
+	if apiOAS, ok := obj.(*oas.OAS); ok {
+		// We have to operate on oas clone in order to preserve original state after any manipulations on schema.
+		oasClone, _ := apiOAS.Clone() // nolint:errcheck
+		if modePublic {
+			oasClone.RemoveTykExtension()
+		}
+
+		visitor := schema.NewVisitor()
+		visitor.AddSchemaManipulation(schema.RestoreUnicodeEscapesFromRE2Manipulation)
+		visitor.ProcessOAS(oasClone)
+
+		obj = oasClone
 	}
-	return obj, code
 
+	return obj, code
 }
 
 func (gw *Gateway) handleAddApi(r *http.Request, fs afero.Fs, oasEndpoint bool) (interface{}, int) {
@@ -1365,7 +1375,12 @@ func (gw *Gateway) writeOASAndAPIDefToFile(fs afero.Fs, apiDef *apidef.APIDefini
 		return
 	}
 
-	err, errCode = gw.writeToFile(fs, oasObj, apiDef.APIID+"-oas")
+	oasDeepCopy, _ := oasObj.Clone() // nolint:errcheck
+	visitor := schema.NewVisitor()
+	visitor.AddSchemaManipulation(schema.RestoreUnicodeEscapesFromRE2Manipulation)
+	visitor.ProcessOAS(oasDeepCopy)
+
+	err, errCode = gw.writeToFile(fs, oasDeepCopy, apiDef.APIID+"-oas")
 	if err != nil {
 		return
 	}
@@ -3571,6 +3586,11 @@ func extractOASObjFromReq(reqBody io.Reader) ([]byte, *oas.OAS, error) {
 	visitor.AddSchemaManipulation(schema.TransformUnicodeEscapesToRE2Manipulation)
 	visitor.ProcessOAS(&oasObj)
 
+	reqBodyInBytes, err = json.Marshal(&oasObj)
+	if err != nil {
+		return nil, nil, ErrRequestMalformed
+	}
+
 	return reqBodyInBytes, &oasObj, nil
 }
 

gateway/api_definition.go

@@ -44,6 +44,7 @@ import (
 	"github.com/TykTechnologies/tyk/config"
 	"github.com/TykTechnologies/tyk/header"
 	"github.com/TykTechnologies/tyk/internal/model"
+	"github.com/TykTechnologies/tyk/pkg/schema"
 	"github.com/TykTechnologies/tyk/regexp"
 	"github.com/TykTechnologies/tyk/rpc"
 	"github.com/TykTechnologies/tyk/storage"
@@ -744,7 +745,13 @@ func (a APIDefinitionLoader) loadDefFromFilePath(filePath string) (*APISpec, err
 		loader.ReadFromURIFunc = openapi3.ReadFromFile
 		oasDoc, err := loader.LoadFromFile(a.GetOASFilepath(filePath))
 		if err == nil {
-			nestDef.OAS = &oas.OAS{T: *oasDoc}
+			oasObj := &oas.OAS{T: *oasDoc}
+
+			visitor := schema.NewVisitor()
+			visitor.AddSchemaManipulation(schema.TransformUnicodeEscapesToRE2Manipulation)
+			visitor.ProcessOAS(oasObj)
+
+			nestDef.OAS = oasObj
 		}
 	}
 

gateway/api_definition_test.go

@@ -11,6 +11,8 @@
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@@ -615,7 +617,7 @@
 }
 
 func TestOldMockResponse(t *testing.T) {
 	test.Racy(t) // TODO: TT-5225
 
 	ts := StartTest(nil)
 	defer ts.Close()
@@ -1374,7 +1376,7 @@
 }
 
 func TestEnforcedTimeout(t *testing.T) {
 	test.Flaky(t) // TODO TT-5222
 
 	ts := StartTest(nil)
 	defer ts.Close()
@@ -1912,7 +1914,7 @@
 }
 
 // TestFromDashboardServiceNetworkErrors tests various network error scenarios for API definitions
 func TestFromDashboardServiceNetworkErrors(t *testing.T) {
 	testCases := []struct {
 		name          string
 		serverFunc    func() *httptest.Server
@@ -2639,3 +2641,110 @@
 		assert.Equal(t, EndPointNotAllowed, status)
 	})
 }
+
+func TestLoadDefFromFilePath(t *testing.T) {
+	ts := StartTest(nil)
+	defer ts.Close()
+
+	loader := APIDefinitionLoader{Gw: ts.Gw}
+
+	t.Run("load classic definition", func(t *testing.T) {
+		apiName := "Test API"
+		apiID := "test-api-1"
+		def := apidef.APIDefinition{
+			Name:  apiName,
+			APIID: apiID,
+			Proxy: apidef.ProxyConfig{
+				ListenPath: "/test-api-1",
+			},
+		}
+		data, err := json.Marshal(def)
+		assert.NoError(t, err)
+
+		tmpFile, err := os.CreateTemp("", "api_def_*.json")
+		assert.NoError(t, err)
+		defer os.Remove(tmpFile.Name())
+
+		_, err = tmpFile.Write(data)
+		assert.NoError(t, err)
+		tmpFile.Close()
+
+		spec, err := loader.loadDefFromFilePath(tmpFile.Name())
+		assert.NoError(t, err)
+		assert.NotNil(t, spec)
+		assert.Equal(t, apiName, spec.Name)
+		assert.Equal(t, apiID, spec.APIID)
+	})
+
+	t.Run("load OAS definition", func(t *testing.T) {
+		tmpDir, err := os.MkdirTemp("", "oas_test")
+		assert.NoError(t, err)
+		defer os.RemoveAll(tmpDir)
+
+		apiFilePath := filepath.Join(tmpDir, "test-api.json")
+		oasFilePath := filepath.Join(tmpDir, "test-api-oas.json")
+
+		// Test whether schema validator successfully applies schema manipulation to oas doc.
+		expectedPatternAfterLoad := "{\"^[\\\\x{0000}-\\\\x{017f}]*$\"}"
+		schema := openapi3.NewSchema()
+		schema.Pattern = "{\"^[\\\\u0000-\\\\u017f]*$\"}"
+		oasDoc := &oas.OAS{
+			T: openapi3.T{
+				Components: &openapi3.Components{
+					Schemas: openapi3.Schemas{
+						"Schema1": openapi3.NewSchemaRef("", schema),
+					},
+				},
+			},
+		}
+		oasData, err := json.Marshal(oasDoc)
+		assert.NoError(t, err)
+
+		err = os.WriteFile(oasFilePath, oasData, 0644)
+		assert.NoError(t, err)
+
+		apiName := "Test OAS API"
+		apiID := "test-oas-api-1"
+		def := apidef.APIDefinition{
+			Name:  apiName,
+			APIID: apiID,
+			IsOAS: true,
+			Proxy: apidef.ProxyConfig{
+				ListenPath: "/test-oas-api-1",
+			},
+		}
+		data, err := json.Marshal(def)
+		assert.NoError(t, err)
+
+		err = os.WriteFile(apiFilePath, data, 0644)
+		assert.NoError(t, err)
+
+		spec, err := loader.loadDefFromFilePath(apiFilePath)
+		assert.NoError(t, err)
+		assert.NotNil(t, spec)
+		assert.Equal(t, apiName, spec.Name)
+		assert.Equal(t, apiID, spec.APIID)
+		assert.NotNil(t, spec.OAS)
+		assert.Equal(t, expectedPatternAfterLoad, spec.OAS.Components.Schemas["Schema1"].Value.Pattern)
+	})
+
+	t.Run("file not found", func(t *testing.T) {
+		spec, err := loader.loadDefFromFilePath("non_existent_file.json")
+		assert.Error(t, err)
+		assert.Nil(t, spec)
+	})
+
+	t.Run("invalid json", func(t *testing.T) {
+		tmpFile, err := os.CreateTemp("", "api_def_*.json")
+		assert.NoError(t, err)
+		defer os.Remove(tmpFile.Name())
+
+		_, err = tmpFile.Write([]byte("{invalid json}"))
+		assert.NoError(t, err)
+		tmpFile.Close()
+
+		spec, err := loader.loadDefFromFilePath(tmpFile.Name())
+		assert.Error(t, err)
+		assert.Nil(t, spec)
+	})
+}