[ruby] Add Support for ERB files

joern-cli/frontends/rubysrc2cpg/src/main/resources/application.conf

@@ -1,4 +1,4 @@
 rubysrc2cpg {
-    ruby_ast_gen_version: "0.33.0"
+    ruby_ast_gen_version: "0.39.0"
     joern_type_stubs_version: "0.6.0"
 }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala

@@ -6,6 +6,7 @@ import io.joern.rubysrc2cpg.parser.RubyJsonHelpers
 import io.joern.rubysrc2cpg.passes.Defines
 import io.joern.rubysrc2cpg.passes.GlobalTypes
 import io.joern.rubysrc2cpg.passes.Defines.{RubyOperators, prefixAsKernelDefined}
+import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants
 import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines}
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{
@@ -83,25 +84,37 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
   protected def astForNilBlock: Ast = blockAst(NewBlock(), List(astForNilLiteral))
 
   protected def astForDynamicLiteral(node: DynamicLiteral): Ast = {
-    val fmtValueAsts = node.expressions.map {
+    val fmtValueAsts = node.expressions.collect {
       case stmtList: StatementList if stmtList.size == 1 =>
-        val expressionAst = astForExpression(stmtList.statements.head)
-        val call = callNode(
-          node = stmtList,
-          code = stmtList.text,
-          name = Operators.formattedValue,
-          methodFullName = Operators.formattedValue,
-          dispatchType = DispatchTypes.STATIC_DISPATCH,
-          signature = None,
-          typeFullName = Some(node.typeFullName)
-        )
-        callAst(call, Seq(expressionAst))
+        stmtList.statements.head match {
+          case x: SimpleCall if x.span.text.startsWith("joern__") =>
+            val argAsts = x.arguments.map(astForExpression)
+            val (opName, callCode) = if (x.span.text.startsWith("joern__template_out_escape")) {
+              (RubyOperators.templateOutEscape, s"<%= ${x.arguments.headOption.map(_.span.text).getOrElse("")} %>")
+            } else {
+              (RubyOperators.templateOutRaw, s"<%== ${x.arguments.headOption.map(_.span.text).getOrElse("")} %>")
+            }
+            val opNode = callNode(x, callCode, opName, opName, DispatchTypes.STATIC_DISPATCH)
+            callAst(opNode, argAsts)
+          case x =>
+            val expressionAst = astForExpression(stmtList.statements.head)
+            val call = callNode(
+              node = stmtList,
+              code = stmtList.text,
+              name = Operators.formattedValue,
+              methodFullName = Operators.formattedValue,
+              dispatchType = DispatchTypes.STATIC_DISPATCH,
+              signature = None,
+              typeFullName = Some(node.typeFullName)
+            )
+            callAst(call, Seq(expressionAst))
+        }
       case stmtList: StatementList if stmtList.size > 1 =>
         logger.warn(
           s"Interpolations containing multiple statements are not supported yet: ${stmtList.text} ($relativeFileName), skipping"
         )
         astForUnknown(stmtList)
-      case node =>
+      case node if (!isLineFeed(node.text)) =>
         val call = callNode(
           node = node,
           code = node.text,
@@ -1087,4 +1100,6 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
   private def getUnaryOperatorName(op: String): Option[String] = UnaryOperatorNames.get(op)
 
   private def getAssignmentOperatorName(op: String): Option[String] = AssignmentOperatorNames.get(op)
+
+  private def isLineFeed(text: String): Boolean = text == "\n" || text == "\r" || text == "\r\n"
 }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPass.scala

@@ -28,7 +28,7 @@ class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg) {
     extensionFilter(".yml"),
     // XML files
     extensionFilter(".xml"),
-    // ERB files
-    extensionFilter(".erb")
+    // HTML.ERB files
+    pathEndFilter(".html.erb")
   )
 }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala

@@ -55,6 +55,8 @@ object Defines {
     val splat             = "<operator>.splat"
     val regexpMatch       = "=~"
     val regexpNotMatch    = "!~"
+    val templateOutRaw    = "<operator>.templateOutRaw"
+    val templateOutEscape = "<operator>.templateOutEscape"
 
     val regexMethods = Set("match") // TODO: Figure out how to model these, "sub", "gsub")
   }

joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPassTests.scala

@@ -45,16 +45,18 @@ class ConfigFileCreationPassTests extends RubyCode2CpgFixture {
     config.content should include("<p>bar<p>")
   }
 
-  "erb files should be included" in {
+  "html erb files should be included" in {
     val cpg = code(
       """
-        |<%= 1 + 2 %>
+        |<foo>
+        | <p><%= ENV['SOME_VAR'] %></p>
+        |</foo>
         |""".stripMargin,
-      "foo.erb"
+      "config.html.erb"
     )
 
-    val config = cpg.configFile.name("foo.erb").head
-    config.content should include("1 + 2")
+    val config = cpg.configFile.name("config.html.erb").head
+    config.content should include("<p><%= ENV['SOME_VAR'] %></p>")
   }
 
 }

joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ErbTests.scala

@@ -0,0 +1,215 @@
+package io.joern.rubysrc2cpg.querying
+
+import io.joern.rubysrc2cpg.passes.Defines.RubyOperators
+import io.joern.x2cpg.Defines
+import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants
+import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture
+import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators}
+import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, FieldIdentifier, Identifier, Literal}
+import io.shiftleft.semanticcpg.language.*
+
+class ErbTests extends RubyCode2CpgFixture {
+  "Complete ERB processing" should {
+    val cpg = code(
+      """
+        |app_name: <%= ENV['APP_NAME'] %>
+        |version: <%== ENV['APP_VERSION'] %>
+        |
+        |database:
+        |  host: <%= ENV['DB_HOST'] %>
+        |  port: <%== ENV['DB_PORT'] %>
+        |
+        |<% if ENV['USE_REDIS'] == 'true' %>
+        |redis:
+        |  host: <%= ENV['REDIS_HOST'] %>
+        |  port: <%== ENV['REDIS_PORT'] %>
+        |<% end %>
+        |""".stripMargin,
+      "test.erb"
+    )
+
+    "Contain a RETURN node" in {
+      inside(cpg.method.name(Constants.Main).methodReturn.toReturn.l) {
+        case erbReturn :: Nil =>
+          val List(formatStringReturn: Call) = erbReturn.astChildren.l: @unchecked
+          formatStringReturn.methodFullName shouldBe Operators.formatString
+        case xs => fail(s"Expected one method return, got ${xs.code.mkString("[", ",", "]")}")
+      }
+    }
+
+    "Contain a call to <operator>.conditional" in {
+      inside(cpg.call.name(Operators.conditional).l) {
+        case _ :: Nil =>
+        // Do nothing - contains call to operator.conditional
+        case _ => fail("AST Should contain one if statement")
+      }
+    }
+
+    "Contain a call to fmtString in the true branch of the <operator>.conditional" in {
+      inside(cpg.call.name(Operators.conditional).argument.l) {
+        case (condition: Call) :: (trueBranch: Block) :: _ :: Nil =>
+          condition.code shouldBe "ENV['USE_REDIS'] == 'true'"
+          condition.methodFullName shouldBe Operators.equals
+
+          inside(trueBranch.astChildren.isCall.l) {
+            case fmtStringCall :: Nil =>
+              fmtStringCall.methodFullName shouldBe Operators.formatString
+              fmtStringCall.argument.l.size shouldBe 4
+              val List(fmtVal1: Call, templateOutEscapeCall: Call, fmtVal3: Call, templateOutRawCall: Call) =
+                fmtStringCall.argument.l: @unchecked
+
+              fmtVal1.methodFullName shouldBe Operators.formattedValue
+              fmtVal1.code shouldBe
+                """redis:
+                  |  host: """.stripMargin
+
+              templateOutEscapeCall.methodFullName shouldBe RubyOperators.templateOutEscape
+              templateOutEscapeCall.code shouldBe "<%= ENV['REDIS_HOST'] %>"
+
+              fmtVal3.methodFullName shouldBe Operators.formattedValue
+              fmtVal3.code shouldBe "  port: "
+
+              templateOutRawCall.methodFullName shouldBe RubyOperators.templateOutRaw
+              templateOutRawCall.code shouldBe "<%== ENV['REDIS_PORT'] %>"
+            case xs => fail(s"Expected one call to fmtString, got [${xs.mkString(",")}]")
+          }
+        case xs => fail(s"Expected three arguments, got ${xs.size}: [${xs.mkString(",")}] instead")
+      }
+    }
+
+    "Contains calls to formattedValue" in {
+      cpg.call.name(Operators.formattedValue).l.size shouldBe 8
+    }
+
+    "Contains calls to templateOutRaw" in {
+      cpg.call.name(RubyOperators.templateOutRaw).l.size shouldBe 3
+    }
+
+    "Contains calls to templateOutEscape" in {
+      cpg.call.name(RubyOperators.templateOutEscape).l
+    }
+  }
+
+  "ERB With If-Elsif contains IF-Struct" in {
+    val cpg = code(
+      """
+        |app_name: <%= ENV['APP_NAME'] %>
+        |version: <%== ENV['APP_VERSION'] %>
+        |
+        |database:
+        |  host: <%= ENV['DB_HOST'] %>
+        |  port: <%== ENV['DB_PORT'] %>
+        |
+        |<% if ENV['USE_REDIS'] == 'true' %>
+        |redis:
+        |  host: <%= ENV['REDIS_HOST'] %>
+        |  port: <%= ENV['REDIS_PORT'] %>
+        |<% elsif ENV['USE_RABBITMQ'] == 'true' %>
+        |rabbitmq:
+        |  host: <%= ENV['RABBITMQ_HOST'] %>
+        |  port: <%== ENV['RABBITMQ_PORT'] %>
+        |<% end %>
+        |""".stripMargin,
+      "test.erb"
+    )
+
+    inside(cpg.controlStructure.controlStructureType(ControlStructureTypes.IF).l) {
+      case ifStruct :: _ :: Nil =>
+        inside(ifStruct.condition.l) {
+          case (cond: Call) :: Nil =>
+            cond.methodFullName shouldBe Operators.equals
+            val List(lhs: Call, rhs: Literal) = cond.argument.l: @unchecked
+            lhs.methodFullName shouldBe Operators.indexAccess
+            rhs.code shouldBe "'true'"
+          case xs => fail(s"Expected one condition, got [${xs.code.mkString(",")}]")
+        }
+
+        inside(ifStruct.whenTrue.isBlock.astChildren.isCall.l) {
+          case fmtStringCall :: Nil =>
+            fmtStringCall.methodFullName shouldBe Operators.formatString
+            fmtStringCall.argument.l.size shouldBe 4
+            val List(fmtVal1: Call, templateOutEscapeOne: Call, fmtVal3: Call, templateOutEscapeTwo: Call) =
+              fmtStringCall.argument.l: @unchecked
+
+            fmtVal1.methodFullName shouldBe Operators.formattedValue
+            fmtVal1.code shouldBe
+              """redis:
+                |  host: """.stripMargin
+
+            templateOutEscapeOne.methodFullName shouldBe RubyOperators.templateOutEscape
+            templateOutEscapeOne.code shouldBe "<%= ENV['REDIS_HOST'] %>"
+
+            fmtVal3.methodFullName shouldBe Operators.formattedValue
+            fmtVal3.code shouldBe """  port: """
+
+            templateOutEscapeTwo.methodFullName shouldBe RubyOperators.templateOutEscape
+            templateOutEscapeTwo.code shouldBe "<%= ENV['REDIS_PORT'] %>"
+
+          case xs => fail(s"Expected one body for true branch, got [${xs.code.mkString(",")}]")
+        }
+
+        inside(ifStruct.whenFalse.isBlock.astChildren.isControlStructure.l) {
+          case elsifStruct :: Nil =>
+            inside(elsifStruct.condition.l) {
+              case (cond: Call) :: Nil =>
+                cond.methodFullName shouldBe Operators.equals
+                val List(lhs: Call, rhs: Literal) = cond.argument.l: @unchecked
+                lhs.methodFullName shouldBe Operators.indexAccess
+                rhs.code shouldBe "'true'"
+              case xs => fail(s"Expected one condition, got [${xs.code.mkString(",")}]")
+            }
+
+            inside(elsifStruct.whenTrue.isBlock.astChildren.isCall.l) {
+              case fmtStringCall :: Nil =>
+                fmtStringCall.methodFullName shouldBe Operators.formatString
+                fmtStringCall.argument.l.size shouldBe 4
+                val List(fmtVal1: Call, templateOutEscape: Call, fmtVal3: Call, templateOutRaw: Call) =
+                  fmtStringCall.argument.l: @unchecked
+
+                fmtVal1.methodFullName shouldBe Operators.formattedValue
+                fmtVal1.code shouldBe
+                  """rabbitmq:
+                    |  host: """.stripMargin
+
+                templateOutEscape.methodFullName shouldBe RubyOperators.templateOutEscape
+                templateOutEscape.code shouldBe "<%= ENV['RABBITMQ_HOST'] %>"
+
+                fmtVal3.methodFullName shouldBe Operators.formattedValue
+                fmtVal3.code shouldBe "  port: "
+
+                templateOutRaw.methodFullName shouldBe RubyOperators.templateOutRaw
+                templateOutRaw.code shouldBe "<%== ENV['RABBITMQ_PORT'] %>"
+              case xs => fail(s"Expected one body with call for true branch, got [${xs.code.mkString(",")}]")
+            }
+          case _ => fail(s"Expected one IF struct in false branch")
+        }
+
+      case xs => fail(s"Expected two IF Structures, got [${xs.code.mkString(",")}]")
+    }
+  }
+
+  "Invalid ERB processing" in {
+    val cpg = code(
+      """
+        |app_name: <%= ENV['APP_NAME'] %>
+        |version: <%= ENV['APP_VERSION'] %>
+        |
+        |database:
+        |  host: <%= ENV['DB_HOST'] %>
+        |  port: <%= ENV['DB_PORT'] %>
+        |
+        |<% if ENV['USE_REDIS'] == 'true' %>
+        |redis:
+        |  host: <%= ENV['REDIS_HOST'] %>
+        |  port: <%= ENV['REDIS_PORT'] %>
+        |""".stripMargin,
+      "test.erb"
+    )
+
+    inside(cpg.method.name(Constants.Main).body.astChildren.isCall.l) {
+      case fmtString :: Nil =>
+        fmtString.methodFullName shouldBe Operators.formatString
+      case xs => fail(s"Expected one call to fmtString, got [${xs.code.mkString(",")}]")
+    }
+  }
+}

joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/LiteralTests.scala

@@ -189,9 +189,7 @@ class LiteralTests extends RubyCode2CpgFixture {
         |>
         |""".stripMargin)
 
-    val List(firstLine, xyz, one23) = cpg.literal.l
-    firstLine.code.trim shouldBe ""
-    firstLine.lineNumber shouldBe Some(2)
+    val List(xyz, one23) = cpg.literal.l
     xyz.code.trim shouldBe "xyz"
     xyz.lineNumber shouldBe Some(3)
     xyz.typeFullName shouldBe RubyDefines.prefixAsCoreType("String")