Add support for default_profile in [__settings__] section

NEXT_CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Release v0.104.0
 
 ### New Features and Improvements
+* Support `default_profile` in `[__settings__]` section of `.databrickscfg` for consistent default profile resolution across CLI and SDKs.
 
 ### Bug Fixes
 * Fixed Databricks CLI authentication to detect when the cached token's scopes don't match the SDK's configured scopes. Previously, a scope mismatch was silently ignored, causing requests to use wrong permissions. The SDK now raises an error with instructions to re-authenticate.

databricks-sdk-java/src/main/java/com/databricks/sdk/core/ConfigLoader.java

@@ -19,6 +19,7 @@
 @InternalApi
 public class ConfigLoader {
   private static final Logger LOG = LoggerFactory.getLogger(ConfigLoader.class);
+  private static final String SETTINGS_SECTION = "__settings__";
 
   private static final List<ConfigAttributeAccessor> accessors = attributeAccessors();
 
@@ -92,22 +93,25 @@ static void loadFromConfig(DatabricksConfig cfg) throws IllegalAccessException {
     INIConfiguration ini = parseDatabricksCfg(configFile, isDefaultConfig);
     if (ini == null) return;
 
-    String profile = cfg.getProfile();
-    boolean hasExplicitProfile = !isNullOrEmpty(profile);
-    if (!hasExplicitProfile) {
-      profile = "DEFAULT";
-    }
+    String[] resolved = resolveProfile(cfg.getProfile(), ini, configFile.toString());
+    String profile = resolved[0];
+    boolean isFallback = "true".equals(resolved[1]);
+
     SubnodeConfiguration section = ini.getSection(profile);
     boolean sectionNotPresent = section == null || section.isEmpty();
-    if (sectionNotPresent && !hasExplicitProfile) {
-      LOG.info("{} has no {} profile configured", configFile, profile);
-      return;
-    }
     if (sectionNotPresent) {
+      if (isFallback) {
+        LOG.info("{} has no {} profile configured", configFile, profile);
+        return;
+      }
       String msg = String.format("resolve: %s has no %s profile configured", configFile, profile);
       throw new DatabricksException(msg);
     }
 
+    if (!isFallback) {
+      cfg.setProfile(profile);
+    }
+
     for (ConfigAttributeAccessor accessor : accessors) {
       String value = section.getString(accessor.getName());
       if (!isNullOrEmpty(accessor.getValueFromConfig(cfg))) {
@@ -117,6 +121,51 @@ static void loadFromConfig(DatabricksConfig cfg) throws IllegalAccessException {
     }
   }
 
+  /**
+   * Resolves which profile to use from the config file.
+   *
+   * <p>Resolution order:
+   *
+   * <ol>
+   *   <li>Explicit profile (flag, env var, or programmatic config) with isFallback=false
+   *   <li>{@code [__settings__].default_profile} with isFallback=false
+   *   <li>{@code "DEFAULT"} with isFallback=true
+   * </ol>
+   *
+   * @return a two-element array: [profileName, "true"/"false" for isFallback]
+   * @throws DatabricksException if the resolved profile is the reserved __settings__ section
+   */
+  static String[] resolveProfile(String requestedProfile, INIConfiguration ini, String configFile) {
+    if (!isNullOrEmpty(requestedProfile)) {
+      if (SETTINGS_SECTION.equals(requestedProfile)) {
+        throw new DatabricksException(
+            String.format(
+                "%s: %s is a reserved section name and cannot be used as a profile",
+                configFile, SETTINGS_SECTION));
+      }
+      return new String[] {requestedProfile, "false"};
+    }
+
+    SubnodeConfiguration settings = ini.getSection(SETTINGS_SECTION);
+    if (settings != null && !settings.isEmpty()) {
+      String defaultProfile = settings.getString("default_profile");
+      if (defaultProfile != null) {
+        defaultProfile = defaultProfile.trim();
+      }
+      if (!isNullOrEmpty(defaultProfile)) {
+        if (SETTINGS_SECTION.equals(defaultProfile)) {
+          throw new DatabricksException(
+              String.format(
+                  "%s: %s is a reserved section name and cannot be used as a profile",
+                  configFile, SETTINGS_SECTION));
+        }
+        return new String[] {defaultProfile, "false"};
+      }
+    }
+
+    return new String[] {"DEFAULT", "true"};
+  }
+
   private static INIConfiguration parseDatabricksCfg(String configFile, boolean isDefaultConfig) {
     INIConfiguration iniConfig = new INIConfiguration();
     try (FileReader reader = new FileReader(configFile)) {

databricks-sdk-java/src/test/java/com/databricks/sdk/DefaultProfileTest.java

@@ -0,0 +1,147 @@
+package com.databricks.sdk;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import com.databricks.sdk.core.ConfigResolving;
+import com.databricks.sdk.core.DatabricksConfig;
+import com.databricks.sdk.core.DatabricksException;
+import com.databricks.sdk.core.http.HttpClient;
+import com.databricks.sdk.core.utils.TestOSUtils;
+import org.junit.jupiter.api.Test;
+
+public class DefaultProfileTest implements ConfigResolving {
+
+  private DatabricksConfig createConfigWithMockClient() {
+    HttpClient mockClient = mock(HttpClient.class);
+    return new DatabricksConfig().setHttpClient(mockClient);
+  }
+
+  /** Test 1: default_profile resolves correctly and is written back to config */
+  @Test
+  public void testDefaultProfileResolvesCorrectly() {
+    StaticEnv env = new StaticEnv().with("HOME", TestOSUtils.resource("/testdata/default_profile"));
+    DatabricksConfig config = createConfigWithMockClient();
+    resolveConfig(config, env);
+    config.authenticate();
+
+    assertEquals("pat", config.getAuthType());
+    assertEquals("https://my-workspace.cloud.databricks.com", config.getHost());
+    assertEquals("my-workspace", config.getProfile());
+  }
+
+  /** Test 2: default_profile takes precedence over [DEFAULT] */
+  @Test
+  public void testDefaultProfileTakesPrecedenceOverDefault() {
+    StaticEnv env =
+        new StaticEnv().with("HOME", TestOSUtils.resource("/testdata/default_profile_precedence"));
+    DatabricksConfig config = createConfigWithMockClient();
+    resolveConfig(config, env);
+    config.authenticate();
+
+    assertEquals("pat", config.getAuthType());
+    assertEquals("https://my-workspace.cloud.databricks.com", config.getHost());
+  }
+
+  /** Test 3: Legacy fallback when no [__settings__] */
+  @Test
+  public void testLegacyFallbackWhenNoSettings() {
+    StaticEnv env = new StaticEnv().with("HOME", TestOSUtils.resource("/testdata"));
+    DatabricksConfig config = createConfigWithMockClient();
+    resolveConfig(config, env);
+    config.authenticate();
+
+    assertEquals("pat", config.getAuthType());
+    assertEquals("https://dbc-XXXXXXXX-YYYY.cloud.databricks.com", config.getHost());
+  }
+
+  /** Test 4: Legacy fallback when default_profile is empty */
+  @Test
+  public void testLegacyFallbackWhenDefaultProfileEmpty() {
+    StaticEnv env =
+        new StaticEnv()
+            .with("HOME", TestOSUtils.resource("/testdata/default_profile_empty_settings"));
+    DatabricksConfig config = createConfigWithMockClient();
+    resolveConfig(config, env);
+    config.authenticate();
+
+    assertEquals("pat", config.getAuthType());
+    assertEquals("https://default.cloud.databricks.com", config.getHost());
+  }
+
+  /** Test 5: default_profile = __settings__ is rejected */
+  @Test
+  public void testSettingsSelfReferenceIsRejected() {
+    StaticEnv env =
+        new StaticEnv()
+            .with("HOME", TestOSUtils.resource("/testdata/default_profile_settings_self_ref"));
+    DatabricksConfig config = createConfigWithMockClient();
+
+    DatabricksException ex =
+        assertThrows(
+            DatabricksException.class,
+            () -> {
+              resolveConfig(config, env);
+              config.authenticate();
+            });
+    assertTrue(
+        ex.getMessage().contains("reserved section name"),
+        "Error should reject __settings__ as a profile target: " + ex.getMessage());
+  }
+
+  /** Test 6: Explicit --profile overrides default_profile */
+  @Test
+  public void testExplicitProfileOverridesDefaultProfile() {
+    StaticEnv env =
+        new StaticEnv()
+            .with("DATABRICKS_CONFIG_PROFILE", "other")
+            .with("HOME", TestOSUtils.resource("/testdata/default_profile_explicit_override"));
+    DatabricksConfig config = createConfigWithMockClient();
+    resolveConfig(config, env);
+    config.authenticate();
+
+    assertEquals("pat", config.getAuthType());
+    assertEquals("https://other.cloud.databricks.com", config.getHost());
+  }
+
+  @Test
+  public void testExplicitSettingsSectionProfileIsRejected() {
+    StaticEnv env =
+        new StaticEnv()
+            .with("DATABRICKS_CONFIG_PROFILE", "__settings__")
+            .with("HOME", TestOSUtils.resource("/testdata/default_profile"));
+    DatabricksConfig config = createConfigWithMockClient();
+
+    DatabricksException ex =
+        assertThrows(
+            DatabricksException.class,
+            () -> {
+              resolveConfig(config, env);
+              config.authenticate();
+            });
+    assertTrue(
+        ex.getMessage().contains("reserved section name"),
+        "Error should reject __settings__ as a profile target: " + ex.getMessage());
+  }
+
+  /** Test 7: default_profile pointing to nonexistent section */
+  @Test
+  public void testDefaultProfileNonexistentSection() {
+    StaticEnv env =
+        new StaticEnv().with("HOME", TestOSUtils.resource("/testdata/default_profile_nonexistent"));
+    DatabricksConfig config = createConfigWithMockClient();
+
+    DatabricksException ex =
+        assertThrows(
+            DatabricksException.class,
+            () -> {
+              resolveConfig(config, env);
+              config.authenticate();
+            });
+    assertTrue(
+        ex.getMessage().contains("deleted-profile"),
+        "Error should mention the missing profile name: " + ex.getMessage());
+  }
+}

databricks-sdk-java/src/test/resources/testdata/default_profile/.databrickscfg

@@ -0,0 +1,6 @@
+[__settings__]
+default_profile = my-workspace
+
+[my-workspace]
+host = https://my-workspace.cloud.databricks.com
+token = dapiXYZ

databricks-sdk-java/src/test/resources/testdata/default_profile_empty_settings/.databrickscfg

@@ -0,0 +1,5 @@
+[__settings__]
+
+[DEFAULT]
+host = https://default.cloud.databricks.com
+token = dapiXYZ

databricks-sdk-java/src/test/resources/testdata/default_profile_explicit_override/.databrickscfg

@@ -0,0 +1,10 @@
+[__settings__]
+default_profile = my-workspace
+
+[my-workspace]
+host = https://my-workspace.cloud.databricks.com
+token = dapiXYZ
+
+[other]
+host = https://other.cloud.databricks.com
+token = dapiOTHER

databricks-sdk-java/src/test/resources/testdata/default_profile_nonexistent/.databrickscfg

@@ -0,0 +1,6 @@
+[__settings__]
+default_profile = deleted-profile
+
+[my-workspace]
+host = https://my-workspace.cloud.databricks.com
+token = dapiXYZ

databricks-sdk-java/src/test/resources/testdata/default_profile_precedence/.databrickscfg

@@ -0,0 +1,10 @@
+[__settings__]
+default_profile = my-workspace
+
+[DEFAULT]
+host = https://default.cloud.databricks.com
+token = dapiOLD
+
+[my-workspace]
+host = https://my-workspace.cloud.databricks.com
+token = dapiXYZ

databricks-sdk-java/src/test/resources/testdata/default_profile_settings_self_ref/.databrickscfg

@@ -0,0 +1,6 @@
+[__settings__]
+default_profile = __settings__
+
+[DEFAULT]
+host = https://default.cloud.databricks.com
+token = dapiXYZ