[jssrc2cpg] Fixes for json exceptions and missing identifier/local links (#5127)

Author: max-leuthaeuser

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForDeclarationsCreator.scala

@@ -85,8 +85,10 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
   ): Ast = {
     from match {
       case Some(value) =>
+        val identNode = identifierNode(declaration, value)
+        scope.addVariableReference(name, identNode)
         val call = createFieldAccessCallAst(
-          identifierNode(declaration, value, Seq.empty),
+          identNode,
           createFieldIdentifierNode(name, declaration.lineNumber, declaration.columnNumber),
           declaration.lineNumber,
           declaration.columnNumber
@@ -99,9 +101,11 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
           declaration.columnNumber
         )
       case None =>
+        val identNode = identifierNode(declaration, name)
+        scope.addVariableReference(name, identNode)
         createAssignmentCallAst(
           exportCallAst,
-          Ast(identifierNode(declaration, name)),
+          Ast(identNode),
           s"${codeOf(exportCallAst.nodes.head)} = $name",
           declaration.lineNumber,
           declaration.columnNumber
@@ -262,7 +266,6 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) {
         createExportAssignmentCallAst(name, exportCallAst, assignment, None)
       }
     }
-
     setArgumentIndices(declAsts)
     blockAst(createBlockNode(assignment), declAsts)
   }

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/parser/BabelJsonParser.scala

@@ -4,7 +4,9 @@ import io.joern.x2cpg.astgen.BaseParserResult
 import io.shiftleft.utils.IOUtils
 import ujson.Value.Value
 
-import java.nio.file.{Path, Paths}
+import java.nio.file.Path
+import java.nio.file.Paths
+import scala.util.Try
 
 object BabelJsonParser {
 
@@ -13,25 +15,28 @@ object BabelJsonParser {
     fullPath: String,
     json: Value,
     fileContent: String,
-    typeMap: Map[Int, String]
+    typeMap: Map[Int, String],
+    fileLoc: Int
   ) extends BaseParserResult
 
-  def readFile(rootPath: Path, file: Path): ParseResult = {
-    val typeMapPath = Paths.get(file.toString.replace(".json", ".typemap"))
-    val typeMap = if (typeMapPath.toFile.exists()) {
-      val typeMapJsonContent = IOUtils.readEntireFile(typeMapPath)
-      val typeMapJson        = ujson.read(typeMapJsonContent)
-      typeMapJson.obj.map { case (k, v) => k.toInt -> v.str }.toMap
-    } else {
-      Map.empty[Int, String]
+  def readFile(rootPath: Path, file: Path): Option[ParseResult] = {
+    Try {
+      val typeMapPath = Paths.get(file.toString.replace(".json", ".typemap"))
+      val typeMap = if (typeMapPath.toFile.exists()) {
+        val typeMapJsonContent = IOUtils.readEntireFile(typeMapPath)
+        val typeMapJson        = ujson.read(typeMapJsonContent)
+        typeMapJson.obj.map { case (k, v) => k.toInt -> v.str }.toMap
+      } else {
+        Map.empty[Int, String]
+      }
+      val jsonContent       = IOUtils.readEntireFile(file)
+      val json              = ujson.read(jsonContent)
+      val filename          = json("relativeName").str
+      val fullPath          = Paths.get(rootPath.toString, filename)
+      val sourceFileContent = IOUtils.readEntireFile(fullPath)
+      val fileLoc           = sourceFileContent.lines().count().toInt
+      ParseResult(filename, fullPath.toString, json, sourceFileContent, typeMap, fileLoc)
     }
-
-    val jsonContent       = IOUtils.readEntireFile(file)
-    val json              = ujson.read(jsonContent)
-    val filename          = json("relativeName").str
-    val fullPath          = Paths.get(rootPath.toString, filename)
-    val sourceFileContent = IOUtils.readEntireFile(fullPath)
-    ParseResult(filename, fullPath.toString, json, sourceFileContent, typeMap)
-  }
+  }.toOption
 
 }

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala

@@ -7,15 +7,19 @@ import io.joern.jssrc2cpg.utils.AstGenRunner.AstGenRunnerResult
 import io.joern.x2cpg.ValidationMode
 import io.joern.x2cpg.datastructures.Global
 import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
-import io.joern.x2cpg.utils.{Report, TimeUtils}
+import io.joern.x2cpg.utils.Report
+import io.joern.x2cpg.utils.TimeUtils
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.passes.ForkJoinParallelCpgPass
 import io.shiftleft.utils.IOUtils
-import org.slf4j.{Logger, LoggerFactory}
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 
 import java.nio.file.Paths
-import scala.util.{Failure, Success, Try}
 import scala.jdk.CollectionConverters.*
+import scala.util.Failure
+import scala.util.Success
+import scala.util.Try
 
 class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: Config, report: Report = new Report())(
   implicit withSchemaValidation: ValidationMode
@@ -45,20 +49,24 @@ class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config:
 
   override def runOnPart(diffGraph: DiffGraphBuilder, input: (String, String)): Unit = {
     val (rootPath, jsonFilename) = input
+    val parseResultMaybe         = BabelJsonParser.readFile(Paths.get(rootPath), Paths.get(jsonFilename))
     val ((gotCpg, filename), duration) = TimeUtils.time {
-      val parseResult = BabelJsonParser.readFile(Paths.get(rootPath), Paths.get(jsonFilename))
-      val fileLOC     = IOUtils.readLinesInFile(Paths.get(parseResult.fullPath)).size
-      report.addReportInfo(parseResult.filename, fileLOC, parsed = true)
-      Try {
-        val localDiff = new AstCreator(config, global, parseResult).createAst()
-        diffGraph.absorb(localDiff)
-      } match {
-        case Failure(exception) =>
-          logger.warn(s"Failed to generate a CPG for: '${parseResult.fullPath}'", exception)
-          (false, parseResult.filename)
-        case Success(_) =>
-          logger.debug(s"Generated a CPG for: '${parseResult.fullPath}'")
-          (true, parseResult.filename)
+      parseResultMaybe match {
+        case Some(parseResult) =>
+          report.addReportInfo(parseResult.filename, parseResult.fileLoc, parsed = true)
+          Try {
+            val localDiff = new AstCreator(config, global, parseResult).createAst()
+            diffGraph.absorb(localDiff)
+          } match {
+            case Failure(exception) =>
+              logger.warn(s"Failed to generate a CPG for: '${parseResult.fullPath}'", exception)
+              (false, parseResult.filename)
+            case Success(_) =>
+              logger.debug(s"Generated a CPG for: '${parseResult.fullPath}'")
+              (true, parseResult.filename)
+          }
+        case None =>
+          (false, jsonFilename.replace(".json", ""))
       }
     }
     report.updateReport(filename, cpg = gotCpg, duration)

joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/DependencyAstCreationPassTests.scala

@@ -49,6 +49,12 @@ class DependencyAstCreationPassTests extends AstJsSrc2CpgSuite {
   }
 
   "AST generation for dependencies" should {
+    "reference identifiers correctly" in {
+      val cpg = code("export const foo = bar();")
+      cpg.local.name("foo").referencingIdentifiers.size shouldBe 2
+      cpg.identifier.name("foo").size shouldBe 2
+    }
+
     "have no dependencies if none are declared at all" in {
       val cpg = code("var x = 1;")
       cpg.dependency.l.size shouldBe 0
@@ -335,7 +341,7 @@ class DependencyAstCreationPassTests extends AstJsSrc2CpgSuite {
         |export = function () {}; // anonymous
         |export = class ClassA {};
         |""".stripMargin)
-      cpg.local.code.l shouldBe List("foo", "bar", "func", "<lambda>0")
+      cpg.local.code.l shouldBe List("foo", "bar", "func", "<lambda>0", "ClassA")
       cpg.typeDecl.name.l should contain allElementsOf List("func", "ClassA")
       cpg.assignment.code.l shouldBe List(
         "var foo = 1",