SONARPY-3803: Fix IPython Notebook parsing when an unsupported cell language is specified (#861)

GitOrigin-RevId: 881c30a7a345a4b76abb562a5da18965e90d75f1

Author: joke1196

python-frontend/src/main/java/org/sonar/plugins/python/IpynbNotebookParser.java

@@ -27,11 +27,15 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonar.python.EscapeCharPositionInfo;
 import org.sonar.python.IPythonLocation;
 
 public class IpynbNotebookParser {
 
+  private static final Logger LOG = LoggerFactory.getLogger(IpynbNotebookParser.class);
+
   public static final String SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER = "#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER";
 
   private static final Set<String> ACCEPTED_LANGUAGE = Set.of("python", "ipython");
@@ -53,25 +57,85 @@ private IpynbNotebookParser(PythonInputFile inputFile) {
   private int lastPythonLine = 0;
 
   public Optional<GeneratedIPythonFile> parse() throws IOException {
-    // If the language is not present, we assume it is a Python notebook
-    var isPythonNotebook = parseLanguage().map(ACCEPTED_LANGUAGE::contains).orElse(true);
+    var language = parseLanguage();
+    boolean isPythonNotebook = language.map(ACCEPTED_LANGUAGE::contains).orElse(true);
+
+    if (isPythonNotebook) {
+      return Optional.of(parseNotebook());
+    }
 
-    return Boolean.TRUE.equals(isPythonNotebook) ? Optional.of(parseNotebook()) : Optional.empty();
+    if(LOG.isDebugEnabled()){
+      LOG.debug("Skipping notebook '{}': unsupported language '{}'", inputFile.wrappedFile().filename(), language.orElse("unknown"));
+    }
+    return Optional.empty();
   }
 
+  /**
+   * Parses the notebook's top-level metadata to find the language.
+   * Only checks metadata.kernelspec.language and metadata.language_info.name,
+   * ignoring any language fields in cell metadata.
+   */
   public Optional<String> parseLanguage() throws IOException {
     String content = inputFile.wrappedFile().contents();
     JsonFactory factory = new JsonFactory();
+    List<String> foundLanguages = new ArrayList<>();
+
     try (JsonParser jParser = factory.createParser(content)) {
       while (!jParser.isClosed()) {
         JsonToken jsonToken = jParser.nextToken();
-        if (JsonToken.FIELD_NAME.equals(jsonToken) && "language".equals(jParser.currentName())) {
+        if (JsonToken.FIELD_NAME.equals(jsonToken) && "metadata".equals(jParser.currentName()) && jParser.getParsingContext().getParent().inRoot()) {
           jParser.nextToken();
-          return Optional.ofNullable(jParser.getValueAsString());
+          extractLanguagesFromMetadata(jParser, foundLanguages);
+          break;
         }
       }
     }
-    return Optional.empty();
+
+    // Return an accepted language if found, otherwise the first language found (for rejection), or empty
+    return foundLanguages.stream()
+      .filter(ACCEPTED_LANGUAGE::contains)
+      .findFirst()
+      .or(() -> foundLanguages.stream().findFirst());
+  }
+
+  /**
+   * Extracts language values from the top-level metadata object.
+   * Looks for kernelspec.language and language_info.name.
+   */
+  private static void extractLanguagesFromMetadata(JsonParser jParser, List<String> foundLanguages) throws IOException {
+    while (jParser.nextToken() != JsonToken.END_OBJECT) {
+      if (JsonToken.FIELD_NAME.equals(jParser.currentToken())) {
+        String fieldName = jParser.currentName();
+        if ("kernelspec".equals(fieldName)) {
+          jParser.nextToken();
+          extractFieldFromObject(jParser, "language", foundLanguages);
+        } else if ("language_info".equals(fieldName)) {
+          jParser.nextToken();
+          extractFieldFromObject(jParser, "name", foundLanguages);
+        } else {
+          jParser.nextToken();
+          skipNestedObjects(jParser);
+        }
+      }
+    }
+  }
+
+  /**
+   * Extracts the value of a specific field from a JSON object.
+   */
+  private static void extractFieldFromObject(JsonParser jParser, String targetField, List<String> foundValues) throws IOException {
+    while (jParser.nextToken() != JsonToken.END_OBJECT) {
+      if (JsonToken.FIELD_NAME.equals(jParser.currentToken()) && targetField.equals(jParser.currentName())) {
+        jParser.nextToken();
+        String value = jParser.getValueAsString();
+        if (value != null) {
+          foundValues.add(value);
+        }
+      } else {
+        jParser.nextToken();
+        skipNestedObjects(jParser);
+      }
+    }
   }
 
   public GeneratedIPythonFile parseNotebook() throws IOException {

python-frontend/src/test/java/org/sonar/plugins/python/IpynbNotebookParserTest.java

@@ -23,16 +23,23 @@
 import java.util.List;
 import java.util.Map;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.slf4j.event.Level;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
 import org.sonar.api.internal.apachecommons.lang3.StringUtils;
+import org.sonar.api.testfixtures.log.LogTesterJUnit5;
 import org.sonar.python.IPythonLocation;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.sonar.plugins.python.NotebookTestUtils.mapToColumnMappingList;
 
 class IpynbNotebookParserTest {
+
+  @RegisterExtension
+  public LogTesterJUnit5 logTester = new LogTesterJUnit5().setLevel(Level.DEBUG);
+
   private final File baseDir = new File("src/test/resources/org/sonar/plugins/python").getAbsoluteFile();
 
   public static PythonInputFile createInputFile(File baseDir, String name, InputFile.Status status, InputFile.Type type) {
@@ -151,6 +158,22 @@ void testParseMojoNotebook() {
     var resultOptional = IpynbNotebookParser.parseNotebook(inputFile);
 
     assertThat(resultOptional).isEmpty();
+    assertThat(logTester.logs(Level.DEBUG)).contains("Skipping notebook 'notebook_mojo.ipynb': unsupported language 'mojo'");
+  }
+
+  @Test
+  void testParseNotebookWithCellLanguage() throws IOException {
+    var inputFile = createInputFile(baseDir, "notebook_with_cell_language.ipynb", InputFile.Status.CHANGED, InputFile.Type.MAIN);
+
+    var resultOptional = IpynbNotebookParser.parseNotebook(inputFile);
+
+    // Should be parsed as Python despite "sparksql" language in cell metadata
+    assertThat(resultOptional).isPresent();
+
+    var result = resultOptional.get();
+    // Should contain content from all code cells (including the SQL magic cell which is still a code cell)
+    assertThat(result.contents()).contains("x = 1");
+    assertThat(result.contents()).contains("y = 2");
   }
 
   @Test

python-frontend/src/test/resources/org/sonar/plugins/python/notebook_with_cell_language.ipynb

@@ -0,0 +1,51 @@
+{
+  "metadata": {
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "custom_field": {
+      "some_key": "some_value"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2,
+  "cells": [
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "x = 1\n",
+        "print(x)"
+      ],
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "microsoft": {
+          "language": "sparksql"
+        }
+      },
+      "source": [
+        "%%sql\n",
+        "SELECT * FROM table"
+      ],
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "y = 2"
+      ],
+      "outputs": [],
+      "execution_count": null
+    }
+  ]
+}