feat: Add OpenAI API Type to OpenAIDefaults (#2453)

Co-authored-by: Wendong Li <[email protected]>

Author: levscaut

cognitive/src/main/python/synapse/ml/services/openai/OpenAIDefaults.py

@@ -128,3 +128,12 @@ def get_reasoning_effort(self):
 
     def reset_reasoning_effort(self):
         self.defaults.resetReasoningEffort()
+
+    def set_api_type(self, api_type):
+        self.defaults.setApiType(api_type)
+
+    def get_api_type(self):
+        return getOption(self.defaults.getApiType())
+
+    def reset_api_type(self):
+        self.defaults.resetApiType()

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAI.scala

@@ -107,6 +107,7 @@ case object OpenAISeedKey extends GlobalKey[Either[Int, String]]
 case object OpenAITopPKey extends GlobalKey[Either[Double, String]]
 case object OpenAIVerbosityKey extends GlobalKey[Either[String, String]]
 case object OpenAIReasoningEffortKey extends GlobalKey[Either[String, String]]
+case object OpenAIApiTypeKey extends GlobalKey[String]
 
 // scalastyle:off number.of.methods
 trait HasOpenAITextParams extends HasOpenAISharedParams {

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIDefaults.scala

@@ -144,6 +144,23 @@ object OpenAIDefaults {
     GlobalParams.resetGlobalParam(OpenAIReasoningEffortKey)
   }
 
+  def setApiType(v: String): Unit = {
+    val options = Seq("responses", "chat_completions")
+    require(
+      options.contains(v),
+      s"ApiType must be in ${options.mkString(", ")}, got: $v"
+    )
+    GlobalParams.setGlobalParam(OpenAIApiTypeKey, v)
+  }
+
+  def getApiType: Option[String] = {
+    GlobalParams.getGlobalParam(OpenAIApiTypeKey)
+  }
+
+  def resetApiType(): Unit = {
+    GlobalParams.resetGlobalParam(OpenAIApiTypeKey)
+  }
+
   private def extractLeft[T](optEither: Option[Either[T, String]]): Option[T] = {
     optEither match {
       case Some(Left(v)) => Some(v)

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPrompt.scala

@@ -8,7 +8,7 @@ import com.microsoft.azure.synapse.ml.core.spark.Functions
 import com.microsoft.azure.synapse.ml.io.binary.BinaryFileReader
 import com.microsoft.azure.synapse.ml.io.http.{ConcurrencyParams, HasErrorCol, HasURL}
 import com.microsoft.azure.synapse.ml.logging.{FeatureNames, SynapseMLLogging}
-import com.microsoft.azure.synapse.ml.param.{HasGlobalParams, StringStringMapParam}
+import com.microsoft.azure.synapse.ml.param.{GlobalParams, HasGlobalParams, StringStringMapParam}
 import com.microsoft.azure.synapse.ml.services._
 import com.microsoft.azure.synapse.ml.services.aifoundry.{AIFoundryChatCompletion, HasAIFoundryTextParamsExtended}
 import HasReturnUsage.{UsageFieldMapping, UsageMappings}
@@ -125,15 +125,22 @@ class OpenAIPrompt(override val uid: String) extends Transformer
     this, "apiType", "The OpenAI API type to use: 'chat_completions' or 'responses'",
     isValid = ParamValidators.inArray(Array("chat_completions", "responses")))
 
+  GlobalParams.registerParam(apiType, OpenAIApiTypeKey)
+
   def getApiType: String = $(apiType)
 
   def setApiType(value: String): this.type = set(apiType, value)
 
   val columnTypes = new StringStringMapParam(
     this, "columnTypes", "A map from column names to their types. Supported types are 'text' and 'path'.")
   private def validateColumnType(value: String) = {
-    require(value.equalsIgnoreCase("text") || value.equalsIgnoreCase("path"),
+    if (value.equalsIgnoreCase("path") || value.equalsIgnoreCase("text")) {
+      logWarning(s"Column type '$value' is deprecated. Please use lowercase 'path' or 'text' instead.")
+    }
+    require(value == "text" || value == "path",
       s"Unsupported column type: $value. Supported types are 'text' and 'path'.")
+    require(value != "responses" || this.getApiType == "responses",
+      s"Column type 'path' is only supported when apiType is set to 'responses'.")
   }
 
   def getColumnTypes: Map[String, String] = $(columnTypes)

cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIDefaultsSuite.scala

@@ -87,6 +87,7 @@ class OpenAIDefaultsSuite extends Flaky with OpenAIAPIKey {
     OpenAIDefaults.setEmbeddingDeploymentName("text-embedding-ada-002")
     OpenAIDefaults.setVerbosity("medium")
     OpenAIDefaults.setReasoningEffort("medium")
+    OpenAIDefaults.setApiType("responses")
 
     assert(OpenAIDefaults.getDeploymentName.contains(deploymentName))
     assert(OpenAIDefaults.getSubscriptionKey.contains(openAIAPIKey))
@@ -99,6 +100,7 @@ class OpenAIDefaultsSuite extends Flaky with OpenAIAPIKey {
     assert(OpenAIDefaults.getEmbeddingDeploymentName.contains("text-embedding-ada-002"))
     assert(OpenAIDefaults.getVerbosity.contains("medium"))
     assert(OpenAIDefaults.getReasoningEffort.contains("medium"))
+    assert(OpenAIDefaults.getApiType.contains("responses"))
   }
 
   test("Test Resetters") {
@@ -113,6 +115,7 @@ class OpenAIDefaultsSuite extends Flaky with OpenAIAPIKey {
     OpenAIDefaults.setEmbeddingDeploymentName("text-embedding-ada-002")
     OpenAIDefaults.setVerbosity("medium")
     OpenAIDefaults.setReasoningEffort("medium")
+    OpenAIDefaults.setApiType("responses")
 
     OpenAIDefaults.resetDeploymentName()
     OpenAIDefaults.resetSubscriptionKey()
@@ -125,6 +128,7 @@ class OpenAIDefaultsSuite extends Flaky with OpenAIAPIKey {
     OpenAIDefaults.resetEmbeddingDeploymentName()
     OpenAIDefaults.resetVerbosity()
     OpenAIDefaults.resetReasoningEffort()
+    OpenAIDefaults.resetApiType()
 
     assert(OpenAIDefaults.getDeploymentName.isEmpty)
     assert(OpenAIDefaults.getSubscriptionKey.isEmpty)
@@ -137,6 +141,7 @@ class OpenAIDefaultsSuite extends Flaky with OpenAIAPIKey {
     assert(OpenAIDefaults.getEmbeddingDeploymentName.isEmpty)
     assert(OpenAIDefaults.getVerbosity.isEmpty)
     assert(OpenAIDefaults.getReasoningEffort.isEmpty)
+    assert(OpenAIDefaults.getApiType.isEmpty)
   }
 
   test("Test Parameter Validation") {