[llvm-dwarfdump][LineCov 2/3] Add coverage baseline comparison and line table coverage in isolation (llvm#183790)

Patch 2 of 3 to add to llvm-dwarfdump the ability to measure DWARF
coverage of local variables in terms of source lines, as discussed in
[this
RFC](https://discourse.llvm.org/t/rfc-debug-info-coverage-tool-v2/83266).

This patch adds the ability to compare a variable’s coverage against a
baseline, e.g. an unoptimised compilation of the same code. This is
provided using the optional `--coverage-baseline` argument.

When a baseline is provided, the output also includes a per-variable
measure of the line table’s coverage (`LT`, `LTRatio`), distinct from
the variable’s coverage proper. See section 2.2 of the RFC for details
on this metric.

Author: unexpectedlydefined

llvm/docs/CommandGuide/llvm-dwarfdump.rst

@@ -198,6 +198,11 @@ OPTIONS
             Show per-variable coverage metrics. The output format is described
             in the section below (:ref:`variable-coverage-format`).
 
+.. option:: --coverage-baseline
+
+            File to use as the baseline for variable coverage statistics
+            (implies :option:`--show-variable-coverage`)
+
 .. option:: --combine-inline-variable-instances
 
             Use with :option:`--show-variable-coverage` to average variable
@@ -280,7 +285,19 @@ a tab-separated table containing the following columns:
         declaration
       - `LinesCovered` ==> Number of source lines covered by the variable's
         debug information in the input file
-
+      - `Baseline` (empty if :option:`--coverage-baseline` is not specified)
+        ==> Number of source lines covered by the variable's debug information
+        in the baseline
+      - `CoveredRatio` (empty if :option:`--coverage-baseline` is not
+        specified) ==> Ratio of the coverage compared to the baseline
+        (calculated as `LinesCovered/Baseline`)
+      - `LinesPresent` (empty if :option:`--coverage-baseline` is not
+        specified) ==> Number of source lines covered in the variable's
+        baseline debug information that are also present in the input file's
+        line table
+      - `LinesPresentRatio` (empty if :option:`--coverage-baseline` is not
+        specified) ==> Ratio of the line table coverage compared to the
+        baseline (calculated as `LinesPresent/Baseline`)
 
 EXIT STATUS
 -----------

llvm/test/tools/llvm-dwarfdump/X86/coverage.test

@@ -24,3 +24,15 @@ COMBINE-NEXT: fn1 1 u test.c:4 1
 COMBINE-NEXT: fn1 1 v test.c:13 0
 COMBINE-NEXT: fn1 1 x test.c:3 7
 COMBINE-NEXT: fn1 1 y test.c:3 7
+
+RUN: llvm-dwarfdump --show-variable-coverage --coverage-baseline=%t.o %t-opt.o | FileCheck %s --check-prefix=BASELINE
+
+BASELINE:      Variable coverage statistics:
+BASELINE-NEXT: Function InlChain Variable Decl LinesCovered Baseline CoveredRatio LinesPresent LinesPresentRatio
+BASELINE-NEXT: f k test.c:20 5 5 1 5 1
+BASELINE-NEXT: f l test.c:20 5 5 1 5 1
+BASELINE-NEXT: fn1 a test.c:11 5 14 0.357 7 0.5
+BASELINE-NEXT: fn1 u test.c:4 1 14 0.0714 7 0.5
+BASELINE-NEXT: fn1 v test.c:13 0 14 0 7 0.5
+BASELINE-NEXT: fn1 x test.c:3 7 14 0.5 7 0.5
+BASELINE-NEXT: fn1 y test.c:3 7 14 0.5 7 0.5

llvm/tools/llvm-dwarfdump/Coverage.cpp

@@ -30,32 +30,70 @@ using namespace llvm::object;
 /// Pair of file index and line number representing a source location.
 typedef std::pair<uint16_t, size_t> SourceLocation;
 
-/// Returns the set of source lines covered by a variable's debug information,
-/// computed by intersecting the variable's location ranges and the containing
-/// scope's address ranges.
-static DenseSet<SourceLocation>
-computeVariableCoverage(DWARFContext &DICtx, DWARFDie VariableDIE,
-                        const DWARFDebugLine::LineTable *const LineTable) {
-  /// Adds source locations to the set that correspond to an address range.
-  auto addLines = [](const DWARFDebugLine::LineTable *LineTable,
+/// Adds source locations to the line set that correspond to an address range.
+static void addLines(const DWARFDebugLine::LineTable *LineTable,
                      DenseSet<SourceLocation> &Lines, DWARFAddressRange Range) {
-    std::vector<uint32_t> Rows;
-    if (LineTable->lookupAddressRange({Range.LowPC, Range.SectionIndex},
-                                      Range.HighPC - Range.LowPC, Rows)) {
-      for (const auto &RowI : Rows) {
-        const auto Row = LineTable->Rows[RowI];
-        // Lookup can return addresses below the LowPC - filter these out.
-        if (Row.Address.Address < Range.LowPC)
-          continue;
-        const auto FileIndex = Row.File;
+  std::vector<uint32_t> Rows;
+  if (LineTable->lookupAddressRange({Range.LowPC, Range.SectionIndex},
+                                    Range.HighPC - Range.LowPC, Rows)) {
+    for (const auto &RowI : Rows) {
+      const auto Row = LineTable->Rows[RowI];
+      // Lookup can return addresses below the LowPC - filter these out.
+      if (Row.Address.Address < Range.LowPC)
+        continue;
+
+      if (Row.Line) // Ignore zero lines.
+        Lines.insert({Row.File, Row.Line});
+    }
+  }
+}
 
-        const auto Line = Row.Line;
-        if (Line) // Ignore zero lines.
-          Lines.insert({FileIndex, Line});
+// Converts the file index of each line in the set to use our own internal
+// file index. This is required for a reliable comparison as the DWARF index may
+// differ across compilations.
+static DenseSet<SourceLocation>
+convertFileIndices(DenseSet<SourceLocation> Lines,
+                   const DWARFDebugLine::LineTable *const LineTable,
+                   DenseMap<uint16_t, uint16_t> &FileIndexMap,
+                   StringMap<uint16_t> &FileNameMap) {
+  DenseSet<SourceLocation> ResultLines;
+  for (const auto &L : Lines) {
+    uint16_t Index;
+    const auto IndexIt = FileIndexMap.find(L.first);
+    if (IndexIt != FileIndexMap.end()) {
+      Index = IndexIt->second;
+    } else {
+      std::string Name;
+      [[maybe_unused]] bool ValidIndex = LineTable->getFileNameByIndex(
+          L.first, "", DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath,
+          Name);
+      assert(ValidIndex && "File index was not valid for its own line table");
+
+      auto NameIt = FileNameMap.find(Name);
+      if (NameIt != FileNameMap.end()) {
+        Index = NameIt->second;
+      } else {
+        Index = FileNameMap.size();
+        FileNameMap.insert({Name, Index});
       }
+
+      FileIndexMap.insert({L.first, Index});
     }
-  };
 
+    ResultLines.insert({Index, L.second});
+  }
+
+  return ResultLines;
+}
+
+/// Returns the set of source lines covered by a variable's debug information,
+/// computed by intersecting the variable's location ranges and the containing
+/// scope's address ranges.
+static DenseSet<SourceLocation>
+computeVariableCoverage(DWARFDie VariableDIE,
+                        const DWARFDebugLine::LineTable *const LineTable,
+                        DenseMap<uint16_t, uint16_t> &FileIndexMap,
+                        StringMap<uint16_t> &FileNameMap) {
   // The optionals below will be empty if no address ranges were found, and
   // present (but containing an empty set) if ranges were found but contained no
   // source locations, in order to distinguish the two cases.
@@ -92,9 +130,55 @@ computeVariableCoverage(DWARFContext &DICtx, DWARFDie VariableDIE,
   if (!Lines && ParentLines)
     Lines = std::move(ParentLines);
   else if (ParentLines)
-    llvm::set_intersect(*Lines, *ParentLines);
+    set_intersect(*Lines, *ParentLines);
 
-  return Lines.value_or(DenseSet<SourceLocation>());
+  if (!Lines)
+    return {};
+
+  return convertFileIndices(Lines.value_or(DenseSet<SourceLocation>()),
+                            LineTable, FileIndexMap, FileNameMap);
+}
+
+/// Adds source locations to the line set that are within an inlined subroutine.
+static void getInlinedLines(DWARFDie SubroutineDIE,
+                            DenseSet<SourceLocation> &Lines,
+                            const DWARFDebugLine::LineTable *const LineTable) {
+  for (const auto &ChildDIE : SubroutineDIE.children()) {
+    if (ChildDIE.getTag() == DW_TAG_inlined_subroutine) {
+      auto Ranges = ChildDIE.getAddressRanges();
+      if (Ranges) {
+        for (const auto &R : Ranges.get())
+          addLines(LineTable, Lines, R);
+      } else {
+        consumeError(Ranges.takeError());
+      }
+    } else {
+      getInlinedLines(ChildDIE, Lines, LineTable);
+    }
+  }
+}
+
+/// Returns the set of source lines present in the line table for a subroutine.
+static DenseSet<SourceLocation>
+computeSubroutineCoverage(DWARFDie SubroutineDIE,
+                          const DWARFDebugLine::LineTable *const LineTable,
+                          DenseMap<uint16_t, uint16_t> &FileIndexMap,
+                          StringMap<uint16_t> &FileNameMap) {
+  auto Ranges = SubroutineDIE.getAddressRanges();
+  DenseSet<SourceLocation> Lines;
+  if (Ranges) {
+    for (const auto &R : Ranges.get())
+      addLines(LineTable, Lines, R);
+  } else {
+    consumeError(Ranges.takeError());
+  }
+
+  // Exclude lines from any subroutines inlined into this one.
+  DenseSet<SourceLocation> InlinedLines;
+  getInlinedLines(SubroutineDIE, InlinedLines, LineTable);
+  set_subtract(Lines, InlinedLines);
+
+  return convertFileIndices(Lines, LineTable, FileIndexMap, FileNameMap);
 }
 
 static const SmallVector<DWARFDie> getParentSubroutines(DWARFDie DIE) {
@@ -140,19 +224,27 @@ struct VarKey {
 struct VarCoverage {
   SmallVector<DWARFDie> Parents;
   size_t Cov;
+  size_t BaselineCov;
+  size_t LTCov;
+  size_t Missing;
   size_t Instances;
+  bool MissingBaseline;
 };
 
 typedef std::multimap<VarKey, VarCoverage, std::less<>> VarMap;
+typedef std::map<VarKey, DenseSet<SourceLocation>, std::less<>> BaselineVarMap;
 
-static std::optional<const VarKey> getVarKey(DWARFDie Die, DWARFDie Parent) {
-  const auto *const DieName = Die.getName(DINameKind::LinkageName);
-  const auto DieFile =
-      Die.getDeclFile(DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
-  const auto *const ParentName = Parent.getName(DINameKind::LinkageName);
-  if (!DieName || !ParentName)
+static std::optional<const VarKey> getVarKey(DWARFDie VariableDIE,
+                                             DWARFDie SubroutineDIE) {
+  const auto *const VariableName = VariableDIE.getName(DINameKind::LinkageName);
+  const auto DeclFile = VariableDIE.getDeclFile(
+      DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
+  const auto *const SubroutineName =
+      SubroutineDIE.getName(DINameKind::LinkageName);
+  if (!VariableName || !SubroutineName)
     return std::nullopt;
-  return VarKey{ParentName, DieName, DieFile, Die.getDeclLine()};
+  return VarKey{SubroutineName, VariableName, DeclFile,
+                VariableDIE.getDeclLine()};
 }
 
 static void displayParents(SmallVector<DWARFDie> Parents, raw_ostream &OS) {
@@ -182,34 +274,105 @@ static void displayVariableCoverage(const VarKey &Key, const VarCoverage &Var,
     displayParents(Var.Parents, OS);
   OS << "\t";
   WithColor(OS, HighlightColor::String) << Key.Name;
-  OS << "\t" << Key.DeclFile << ":" << Key.DeclLine;
+  OS << "\t";
+  if (!Key.DeclFile.empty())
+    OS << Key.DeclFile << ":" << Key.DeclLine;
   OS << "\t" << format("%.3g", ((float)Var.Cov / Var.Instances));
+  if (Var.BaselineCov)
+    OS << "\t" << format("%.3g", ((float)Var.BaselineCov / Var.Instances))
+       << "\t" << format("%.3g", ((float)Var.Cov / Var.BaselineCov)) << "\t"
+       << format("%.3g", ((float)Var.LTCov / Var.Instances)) << "\t"
+       << format("%.3g", ((float)Var.LTCov / Var.BaselineCov));
   OS << "\n";
+  if (Var.MissingBaseline)
+    WithColor(errs(), HighlightColor::Warning).warning()
+        << "DIE not found in baseline\n";
+  if (Var.Missing)
+    WithColor(errs(), HighlightColor::Warning).warning()
+        << Var.Missing << " lines not found in baseline\n";
 }
 
 bool dwarfdump::showVariableCoverage(ObjectFile &Obj, DWARFContext &DICtx,
+                                     ObjectFile *BaselineObj,
+                                     DWARFContext *BaselineCtx,
                                      bool CombineInstances, raw_ostream &OS) {
+  BaselineVarMap BaselineVars;
+  StringMap<uint16_t> FileNameMap;
+
+  if (BaselineCtx) {
+    for (const auto &U : BaselineCtx->info_section_units()) {
+      const auto *const LT = BaselineCtx->getLineTableForUnit(U.get());
+      DenseMap<uint16_t, uint16_t> FileIndexMap;
+      for (const auto &Entry : U->dies()) {
+        DWARFDie VariableDIE = {U.get(), &Entry};
+        if (VariableDIE.getTag() != DW_TAG_variable &&
+            VariableDIE.getTag() != DW_TAG_formal_parameter)
+          continue;
+
+        const auto Parents = getParentSubroutines(VariableDIE);
+        if (!Parents.size())
+          continue;
+        const auto SubroutineDIE = Parents.front();
+        auto Key = getVarKey(VariableDIE, SubroutineDIE);
+        if (!Key)
+          continue;
+
+        auto Cov =
+            computeVariableCoverage(VariableDIE, LT, FileIndexMap, FileNameMap);
+        const auto SubroutineCov = computeSubroutineCoverage(
+            SubroutineDIE, LT, FileIndexMap, FileNameMap);
+        set_intersect(Cov, SubroutineCov);
+
+        auto Result = BaselineVars.insert({*Key, Cov});
+        if (!Result.second)
+          Result.first->second.insert_range(Cov);
+      }
+    }
+  }
+
   VarMap Vars;
 
   for (const auto &U : DICtx.info_section_units()) {
     const auto *const LT = DICtx.getLineTableForUnit(U.get());
+    DenseMap<uint16_t, uint16_t> FileIndexMap;
     for (const auto &Entry : U->dies()) {
-      DWARFDie Die = {U.get(), &Entry};
-      if (Die.getTag() != DW_TAG_variable &&
-          Die.getTag() != DW_TAG_formal_parameter)
+      DWARFDie VariableDIE = {U.get(), &Entry};
+      if (VariableDIE.getTag() != DW_TAG_variable &&
+          VariableDIE.getTag() != DW_TAG_formal_parameter)
         continue;
 
-      const auto Parents = getParentSubroutines(Die);
+      const auto Parents = getParentSubroutines(VariableDIE);
       if (!Parents.size())
         continue;
-      const auto Parent = Parents.front();
-      auto Key = getVarKey(Die, Parent);
+      const auto SubroutineDIE = Parents.front();
+      auto Key = getVarKey(VariableDIE, SubroutineDIE);
       if (!Key)
         continue;
 
-      const auto Cov = computeVariableCoverage(DICtx, Die, LT);
+      auto Cov =
+          computeVariableCoverage(VariableDIE, LT, FileIndexMap, FileNameMap);
+      const auto SubroutineCov = computeSubroutineCoverage(
+          SubroutineDIE, LT, FileIndexMap, FileNameMap);
+      set_intersect(Cov, SubroutineCov);
 
-      VarCoverage VarCov = {Parents, Cov.size(), 1};
+      VarCoverage VarCov = {Parents, Cov.size(), 0, 0, 0, 1, false};
+
+      if (BaselineCtx) {
+        BaselineVarMap::iterator Var = BaselineVars.find(*Key);
+
+        if (Var != BaselineVars.end()) {
+          const auto BCov = Var->second;
+          VarCov.BaselineCov = BCov.size();
+
+          for (const auto &L : Cov)
+            VarCov.Missing += (1 - BCov.count(L));
+
+          for (const auto &L : BCov)
+            VarCov.LTCov += SubroutineCov.count(L);
+        } else {
+          VarCov.MissingBaseline = true;
+        }
+      }
 
       Vars.insert({*Key, VarCov});
     }
@@ -219,16 +382,23 @@ bool dwarfdump::showVariableCoverage(ObjectFile &Obj, DWARFContext &DICtx,
 
   OS << "\nVariable coverage statistics:\nFunction\t"
      << (CombineInstances ? "InstanceCount" : "InlChain")
-     << "\tVariable\tDecl\tLinesCovered\n";
+     << "\tVariable\tDecl\tLinesCovered";
+  if (BaselineCtx)
+    OS << "\tBaseline\tCoveredRatio\tLinesPresent\tLinesPresentRatio";
+  OS << "\n";
 
   if (CombineInstances) {
     for (auto FirstVar = Vars.begin(); FirstVar != Vars.end();
          FirstVar = Range.second) {
       Range = Vars.equal_range(FirstVar->first);
-      VarCoverage CombinedCov = {{}, 0, 0};
+      VarCoverage CombinedCov = {{}, 0, 0, 0, 0, 0, false};
       for (auto Var = Range.first; Var != Range.second; ++Var) {
         ++CombinedCov.Instances;
         CombinedCov.Cov += Var->second.Cov;
+        CombinedCov.BaselineCov += Var->second.BaselineCov;
+        CombinedCov.LTCov += Var->second.LTCov;
+        CombinedCov.Missing += Var->second.Missing;
+        CombinedCov.MissingBaseline |= Var->second.MissingBaseline;
       }
       displayVariableCoverage(FirstVar->first, CombinedCov, true, OS);
     }