Run community build with -indent -rewrite after #17522

community-build/src/scala/dotty/communitybuild/projects.scala

@@ -111,11 +111,14 @@ final case class SbtCommunityProject(
     sbtDocCommand: String = null,
     scalacOptions: List[String] = SbtCommunityProject.scalacOptions,
     requiresExperimental: Boolean = false,
+    rewriteToIndent: Boolean = true,
   ) extends CommunityProject:
   override val binaryName: String = "sbt"
 
   private def scalacOptionsString: String =
-    scalacOptions.map("\"" + _ + "\"").mkString("List(", ",", ")")
+    ((if rewriteToIndent then Seq("-indent", "-rewrite") else Seq.empty) ++ scalacOptions)
+      .map("\"" + _ + "\"")
+      .mkString("List(", ",", ")")
 
   private val baseCommand =
     "clean; set Global/logLevel := Level.Error; set Global/updateOptions ~= (_.withLatestSnapshots(false)); "

compiler/src/dotty/tools/dotc/Run.scala

@@ -394,7 +394,12 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
   private var myCtx: Context | Null = rootContext(using ictx)
 
   /** The context created for this run */
-  given runContext[Dummy_so_its_a_def]: Context = myCtx.nn
+  given runContext[Dummy_so_its_a_def]: Context =
+    // if indent enabled, remove no-indent
+    if myCtx.nn.settings.indent.value(using myCtx.nn) then
+      myCtx.nn.fresh.setSetting(myCtx.nn.settings.noindent, false)
+    else myCtx.nn
+
   assert(runContext.runId <= Periods.MaxPossibleRunId)
 }
 

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -60,16 +60,19 @@ object Scanners {
     /** the base of a number */
     var base: Int = 0
 
-    def copyFrom(td: TokenData): Unit = {
+    def copyFrom(td: TokenData): this.type = {
       this.token = td.token
       this.offset = td.offset
       this.lastOffset = td.lastOffset
       this.lineOffset = td.lineOffset
       this.name = td.name
       this.strVal = td.strVal
       this.base = td.base
+      this
     }
 
+    def saveCopy: TokenData = newTokenData.copyFrom(this)
+
     def isNewLine = token == NEWLINE || token == NEWLINES
     def isStatSep = isNewLine || token == SEMI
     def isIdent = token == IDENTIFIER || token == BACKQUOTED_IDENT
@@ -86,12 +89,14 @@ object Scanners {
 
     def isOperator =
       token == BACKQUOTED_IDENT
-      || token == IDENTIFIER && isOperatorPart(name(name.length - 1))
+      || token == IDENTIFIER && isOperatorPart(name.last)
 
     def isArrow =
       token == ARROW || token == CTXARROW
   }
 
+  def newTokenData: TokenData = new TokenData {}
+
   abstract class ScannerCommon(source: SourceFile)(using Context) extends CharArrayReader with TokenData {
     val buf: Array[Char] = source.content
     def nextToken(): Unit
@@ -170,7 +175,7 @@ object Scanners {
         errorButContinue(em"trailing separator is not allowed", offset + litBuf.length - 1)
   }
 
-  class Scanner(source: SourceFile, override val startFrom: Offset = 0, profile: Profile = NoProfile, allowIndent: Boolean = true)(using Context) extends ScannerCommon(source) {
+  class Scanner(source: SourceFile, override val startFrom: Offset = 0, profile: Profile = NoProfile, allowRewrite: Boolean = true, allowIndent: Boolean = true)(using Context) extends ScannerCommon(source) {
     val keepComments = !ctx.settings.YdropComments.value
 
     /** A switch whether operators at the start of lines can be infix operators */
@@ -179,7 +184,7 @@ object Scanners {
     var debugTokenStream = false
     val showLookAheadOnDebug = false
 
-    val rewrite = ctx.settings.rewrite.value.isDefined
+    val rewrite = allowRewrite && ctx.settings.rewrite.value.isDefined
     val oldSyntax = ctx.settings.oldSyntax.value
     val newSyntax = ctx.settings.newSyntax.value
 
@@ -264,11 +269,10 @@ object Scanners {
       if (idx >= 0 && idx <= lastKeywordStart) handleMigration(kwArray(idx))
       else IDENTIFIER
 
-    def newTokenData: TokenData = new TokenData {}
-
     /** We need one token lookahead and one token history
      */
     val next = newTokenData
+    val last = newTokenData
     private val prev = newTokenData
 
     /** The current region. This is initially an Indented region with zero indentation width. */
@@ -385,6 +389,7 @@ object Scanners {
     /** Produce next token, filling TokenData fields of Scanner.
      */
     def nextToken(): Unit =
+      last.copyFrom(this)
       val lastToken = token
       val lastName = name
       adjustSepRegions(lastToken)
@@ -433,7 +438,7 @@ object Scanners {
         // in backticks and is a binary operator. Hence, `x` is not classified as a
         // leading infix operator.
         def assumeStartsExpr(lexeme: TokenData) =
-          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONeol)
+          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONfollow)
           && (!lexeme.isOperator || nme.raw.isUnary(lexeme.name))
         val lookahead = LookaheadScanner()
         lookahead.allowLeadingInfixOperators = false
@@ -483,7 +488,7 @@ object Scanners {
             if (nextChar == ch)
               recur(idx - 1, ch, n + 1, k)
             else {
-              val k1: IndentWidth => IndentWidth = if (n == 0) k else Conc(_, Run(ch, n))
+              val k1: IndentWidth => IndentWidth = if (n == 0) k else iw => k(Conc(iw, Run(ch, n)))
               recur(idx - 1, nextChar, 1, k1)
             }
           else recur(idx - 1, ' ', 0, identity)
@@ -523,7 +528,7 @@ object Scanners {
      *
      *      The following tokens can start an indentation region:
      *
-     *         :  =  =>  <-  if  then  else  while  do  try  catch  
+     *         :  =  =>  <-  if  then  else  while  do  try  catch
      *         finally  for  yield  match  throw  return  with
      *
      *      Inserting an INDENT starts a new indentation region with the indentation of the current
@@ -638,7 +643,8 @@ object Scanners {
             currentRegion.knownWidth = nextWidth
         else if (lastWidth != nextWidth)
           val lw = lastWidth
-          errorButContinue(spaceTabMismatchMsg(lw, nextWidth))
+          val msg = spaceTabMismatchMsg(lw, nextWidth)
+          if rewriteToIndent then report.warning(msg) else errorButContinue(msg)
       if token != OUTDENT then
         handleNewIndentWidth(currentRegion, _.otherIndentWidths += nextWidth)
       if next.token == EMPTY then
@@ -758,6 +764,8 @@ object Scanners {
         if endMarkerTokens.contains(lookahead.token)
           && source.offsetToLine(lookahead.offset) == endLine
         then
+          if rewriteToIndent && lookahead.token == MATCH then
+            patch(Span(offset, offset + 3), "`end`")
           lookahead.nextToken()
           if lookahead.token == EOF
           || source.offsetToLine(lookahead.offset) > endLine
@@ -1266,6 +1274,7 @@ object Scanners {
               putChar(ch) ; nextRawChar()
               loopRest()
             else
+              next.lineOffset = if next.lastOffset < lineStartOffset then lineStartOffset else -1
               finishNamedToken(IDENTIFIER, target = next)
           end loopRest
           setStrVal()
@@ -1312,10 +1321,10 @@ object Scanners {
       }
     end getStringPart
 
-    private def fetchStringPart(multiLine: Boolean) = {
+    private def fetchStringPart(multiLine: Boolean) =
       offset = charOffset - 1
+      lineOffset = if lastOffset < lineStartOffset then lineStartOffset else -1
       getStringPart(multiLine)
-    }
 
     private def isTripleQuote(): Boolean =
       if (ch == '"') {
@@ -1657,21 +1666,31 @@ object Scanners {
     case Run(ch: Char, n: Int)
     case Conc(l: IndentWidth, r: Run)
 
-    def <= (that: IndentWidth): Boolean = this match {
-      case Run(ch1, n1) =>
-        that match {
-          case Run(ch2, n2) => n1 <= n2 && (ch1 == ch2 || n1 == 0)
-          case Conc(l, r) => this <= l
-        }
-      case Conc(l1, r1) =>
-        that match {
-          case Conc(l2, r2) => l1 == l2 && r1 <= r2
-          case _ => false
-        }
-    }
+    def <= (that: IndentWidth): Boolean = (this, that) match
+      case (Run(ch1, n1), Run(ch2, n2)) => n1 <= n2 && (ch1 == ch2 || n1 == 0)
+      case (Conc(l1, r1), Conc(l2, r2)) => (l1 == l2 && r1 <= r2) || this <= l2
+      case (_, Conc(l2, _)) => this <= l2
+      case _ => false
 
     def < (that: IndentWidth): Boolean = this <= that && !(that <= this)
 
+    def >= (that: IndentWidth): Boolean = that <= this
+
+    def >(that: IndentWidth): Boolean = that < this
+
+    def size: Int = this match
+      case Run(_, n) => n
+      case Conc(l, r) => l.size + r.n
+
+    /** Add one level of indentation (one tab or two spaces depending on the last char) */
+    def increment: IndentWidth =
+      def incRun(ch: Char, n: Int): Run = ch match
+        case ' ' => IndentWidth.Run(' ', n + 2)
+        case ch => IndentWidth.Run(ch, n + 1)
+      this match
+        case Run(ch, n) => incRun(ch, n)
+        case Conc(l, Run(ch, n)) => Conc(l, incRun(ch, n))
+
     /** Does `this` differ from `that` by not more than a single space? */
     def isClose(that: IndentWidth): Boolean = this match
       case Run(ch1, n1) =>

compiler/src/dotty/tools/dotc/parsing/xml/MarkupParsers.scala

@@ -324,8 +324,7 @@ object MarkupParsers {
     /** Some try/catch/finally logic used by xLiteral and xLiteralPattern.  */
     inline private def xLiteralCommon(f: () => Tree, ifTruncated: String => Unit): Tree = {
       assert(parser.in.token == Tokens.XMLSTART)
-      val saved = parser.in.newTokenData
-      saved.copyFrom(parser.in)
+      val saved = parser.in.saveCopy
       var output: Tree = null.asInstanceOf[Tree]
       try output = f()
       catch {
@@ -404,7 +403,7 @@ object MarkupParsers {
     def escapeToScala[A](op: => A, kind: String): A = {
       xEmbeddedBlock = false
       val res = saving(parser.in.currentRegion, parser.in.currentRegion = _) {
-        val lbrace = parser.in.newTokenData
+        val lbrace = Scanners.newTokenData
         lbrace.token = LBRACE
         lbrace.offset = parser.in.charOffset - 1
         lbrace.lastOffset = parser.in.lastOffset

compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala

@@ -25,9 +25,13 @@ object Rewrites {
     def addPatch(span: Span, replacement: String): Unit =
       pbuf += Patch(span, replacement)
 
+    def patchOver(span: Span, replacement: String): Unit =
+      pbuf.indices.reverse.find(i => span.contains(pbuf(i).span)).foreach(pbuf.remove)
+      pbuf += Patch(span, replacement)
+
     def apply(cs: Array[Char]): Array[Char] = {
       val delta = pbuf.map(_.delta).sum
-      val patches = pbuf.toList.sortBy(_.span.start)
+      val patches = pbuf.toList.sortBy(p => (p.span.start, p.span.end))
       if (patches.nonEmpty)
         patches.reduceLeft {(p1, p2) =>
           assert(p1.span.end <= p2.span.start, s"overlapping patches in $source: $p1 and $p2")
@@ -71,6 +75,14 @@ object Rewrites {
          .addPatch(span, replacement)
     )
 
+  /** Record a patch that replaces the first patch that it contains */
+  def patchOver(source: SourceFile, span: Span, replacement: String)(using Context): Unit =
+    if ctx.reporter != Reporter.NoReporter // NoReporter is used for syntax highlighting
+    then ctx.settings.rewrite.value.foreach(_.patched
+         .getOrElseUpdate(source, new Patches(source))
+         .patchOver(span, replacement)
+    )
+
   /** Patch position in `ctx.compilationUnit.source`. */
   def patch(span: Span, replacement: String)(using Context): Unit =
     patch(ctx.compilationUnit.source, span, replacement)

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -82,9 +82,14 @@ class CompilationTests {
       compileFile("tests/rewrites/rewrites3x.scala", defaultOptions.and("-rewrite", "-source", "future-migration")),
       compileFile("tests/rewrites/filtering-fors.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")),
       compileFile("tests/rewrites/refutable-pattern-bindings.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")),
-      compileFile("tests/rewrites/i8982.scala", defaultOptions.and("-indent", "-rewrite")),
-      compileFile("tests/rewrites/i9632.scala", defaultOptions.and("-indent", "-rewrite")),
-      compileFile("tests/rewrites/i11895.scala", defaultOptions.and("-indent", "-rewrite")),
+      compileFile("tests/rewrites/i8982.scala", indentRewrite),
+      compileFile("tests/rewrites/i9632.scala", indentRewrite),
+      compileFile("tests/rewrites/i11895.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-rewrite.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-comments.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-mix-tab-space.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-3-spaces.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-mix-brace.scala", indentRewrite),
       compileFile("tests/rewrites/i12340.scala", unindentOptions.and("-rewrite")),
       compileFile("tests/rewrites/i17187.scala", unindentOptions.and("-rewrite")),
       compileFile("tests/rewrites/i17399.scala", unindentOptions.and("-rewrite")),

compiler/test/dotty/tools/vulpix/TestConfiguration.scala

@@ -65,6 +65,7 @@ object TestConfiguration {
 
   val commonOptions = Array("-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions
   val defaultOptions = TestFlags(basicClasspath, commonOptions)
+  val indentRewrite = defaultOptions.and("-rewrite")
   val unindentOptions = TestFlags(basicClasspath, Array("-no-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions)
   val withCompilerOptions =
     defaultOptions.withClasspath(withCompilerClasspath).withRunClasspath(withCompilerClasspath)

tests/pos/indent-colons.scala

@@ -152,6 +152,11 @@ object Test23:
   val _ = 1 `+`:  // ok
     x
 
+  // leading infix op
+  val _ = 1
+  `+` :
+    x
+
   val r = 1 to:
     100
 

tests/rewrites/indent-3-spaces.check

@@ -0,0 +1,21 @@
+// Rewrite to indent, keeping 3 spaces as indentation
+
+def m1 =
+   def m2 =
+      "" +
+         "" +
+         ""
+   m2
+
+def m4 =
+   def m5 =
+      def m6 =
+         val x = ""
+         x
+            .apply(0)
+            .toString
+      m6
+         .toString
+   m5 +
+      m5
+         .toString

tests/rewrites/indent-3-spaces.scala

@@ -0,0 +1,26 @@
+// Rewrite to indent, keeping 3 spaces as indentation
+
+def m1 = {
+   def m2 = {
+      "" +
+         "" +
+         ""
+   }
+   m2
+}
+
+def m4 = {
+   def m5 = {
+      def m6 = {
+         val x = ""
+         x
+            .apply(0)
+            .toString
+      }
+      m6
+         .toString
+   }
+   m5 +
+      m5
+         .toString
+}

tests/rewrites/indent-comments.check

@@ -0,0 +1,25 @@
+// Rewriting to indent should preserve comments
+class A: /* 1 */ /* 2 */
+  def m1(b: Boolean) = /* 3 */ /* 4 */
+    val x = if (b)
+    /* 5 */
+      "true"
+    /* 6 */
+    else
+    /* 7 */
+      "false"
+    /* 8 */
+    /* 9 */ x.toBoolean
+  /* 10 */ /* 11 */
+  /* 12 */def m2 = // 12
+    m1:// 14
+    /* 15 */// 16
+      true
+    /* 17 */// 18
+// because of the missing indent before {
+// the scanner inserts a new line between || and {
+// cannot rewrite to indentation without messing the comments up
+    true ||// 19
+    /* 20 */{
+      false
+    }// 21