Add format string handling

clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp

@@ -619,10 +619,10 @@ bool FormatStringConverter::convertArgument(const PrintfSpecifier &FS,
 }
 
 /// Called for each format specifier by ParsePrintfString.
-bool FormatStringConverter::HandlePrintfSpecifier(const PrintfSpecifier &FS,
-                                                  const char *StartSpecifier,
-                                                  unsigned SpecifierLen,
-                                                  const TargetInfo &Target) {
+bool FormatStringConverter::HandlePrintfSpecifier(
+    const PrintfSpecifier &FS, const char *StartSpecifier,
+    unsigned SpecifierLen, const TargetInfo &Target,
+    const llvm::TextEncodingConverter &) {
   const size_t StartSpecifierPos = StartSpecifier - PrintfFormatString.data();
   assert(StartSpecifierPos + SpecifierLen <= PrintfFormatString.size());
 

clang-tools-extra/clang-tidy/utils/FormatStringConverter.h

@@ -106,7 +106,8 @@ class FormatStringConverter
 
   bool HandlePrintfSpecifier(const PrintfSpecifier &FS,
                              const char *StartSpecifier, unsigned SpecifierLen,
-                             const TargetInfo &Target) override;
+                             const TargetInfo &Target,
+                             const llvm::TextEncodingConverter &Conv) override;
 
   void appendFormatText(StringRef Text);
   void finalizeFormatText();

clang/include/clang/AST/FormatString.h

@@ -20,6 +20,7 @@
 
 #include "clang/AST/CanonicalType.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/TextEncoding.h"
 #include <optional>
 
 namespace clang {
@@ -764,14 +765,15 @@ class FormatStringHandler {
 
   virtual bool HandleInvalidPrintfConversionSpecifier(
       const analyze_printf::PrintfSpecifier &FS, const char *startSpecifier,
-      unsigned specifierLen) {
+      unsigned specifierLen, const llvm::TextEncodingConverter &Conv) {
     return true;
   }
 
   virtual bool HandlePrintfSpecifier(const analyze_printf::PrintfSpecifier &FS,
                                      const char *startSpecifier,
                                      unsigned specifierLen,
-                                     const TargetInfo &Target) {
+                                     const TargetInfo &Target,
+                                     const llvm::TextEncodingConverter &Conv) {
     return true;
   }
 
@@ -780,10 +782,9 @@ class FormatStringHandler {
 
   // Scanf-specific handlers.
 
-  virtual bool
-  HandleInvalidScanfConversionSpecifier(const analyze_scanf::ScanfSpecifier &FS,
-                                        const char *startSpecifier,
-                                        unsigned specifierLen) {
+  virtual bool HandleInvalidScanfConversionSpecifier(
+      const analyze_scanf::ScanfSpecifier &FS, const char *startSpecifier,
+      unsigned specifierLen, const llvm::TextEncodingConverter &Conv) {
     return true;
   }
 

clang/include/clang/Lex/TextEncoding.h

@@ -30,9 +30,6 @@ class TextEncoding {
   std::unique_ptr<llvm::TextEncodingConverter> FromLiteralEncodingConverter;
 
   // Only non-null on z/OS, where the system default encoding is IBM-1047.
-  // This converts UTF-8 to IBM-1047 for asm string literals so that
-  // octal/hex escape sequences are interpreted as IBM-1047 code points,
-  // regardless of -fexec-charset.
   std::unique_ptr<llvm::TextEncodingConverter> ToIBM1047Converter;
   std::unique_ptr<llvm::TextEncodingConverter> FromIBM1047Converter;
 

clang/lib/AST/FormatString.cpp

@@ -34,19 +34,20 @@ FormatStringHandler::~FormatStringHandler() {}
 // scanf format strings.
 //===----------------------------------------------------------------------===//
 
-OptionalAmount clang::analyze_format_string::ParseAmount(const char *&Beg,
-                                                         const char *E) {
+OptionalAmount clang::analyze_format_string::ParseAmount(
+    const char *&Beg, const char *E,
+    const llvm::TextEncodingConverter &Conv) {
   const char *I = Beg;
   UpdateOnReturn<const char *> UpdateBeg(Beg, I);
 
   unsigned accumulator = 0;
   bool hasDigits = false;
 
   for (; I != E; ++I) {
-    char c = *I;
-    if (c >= '0' && c <= '9') {
+    char c = Conv.convertBasicChar(*I);
+    if (c >= u8'0' && c <= u8'9') {
       hasDigits = true;
-      accumulator = (accumulator * 10) + (c - '0');
+      accumulator = (accumulator * 10) + (c - u8'0');
       continue;
     }
 
@@ -76,21 +77,23 @@ static bool ParseWidthModifier(const char *&I, const char *E,
 }
 
 OptionalAmount clang::analyze_format_string::ParseNonPositionAmount(
-    const char *&Beg, const char *E, unsigned &argIndex) {
-  if (*Beg == '*') {
+    const char *&Beg, const char *E, unsigned &argIndex,
+    const llvm::TextEncodingConverter &Conv) {
+  if (Conv.convertBasicChar(*Beg) == u8'*') {
     ++Beg;
     return OptionalAmount(OptionalAmount::Arg, argIndex++, Beg, 0, false);
   }
 
-  return ParseAmount(Beg, E);
+  return ParseAmount(Beg, E, Conv);
 }
 
 OptionalAmount clang::analyze_format_string::ParsePositionAmount(
     FormatStringHandler &H, const char *Start, const char *&Beg, const char *E,
-    PositionContext p) {
-  if (*Beg == '*') {
+    PositionContext p,
+    const llvm::TextEncodingConverter &Conv) {
+  if (Conv.convertBasicChar(*Beg) == u8'*') {
     const char *I = Beg + 1;
-    const OptionalAmount &Amt = ParseAmount(I, E);
+    const OptionalAmount &Amt = ParseAmount(I, E, Conv);
 
     if (Amt.getHowSpecified() == OptionalAmount::NotSpecified) {
       H.HandleInvalidPosition(Beg, I - Beg, p);
@@ -105,7 +108,7 @@ OptionalAmount clang::analyze_format_string::ParsePositionAmount(
 
     assert(Amt.getHowSpecified() == OptionalAmount::Constant);
 
-    if (*I == '$') {
+    if (Conv.convertBasicChar(*I) == u8'$') {
       // Handle positional arguments
 
       // Special case: '*0$', since this is an easy mistake.
@@ -125,18 +128,21 @@ OptionalAmount clang::analyze_format_string::ParsePositionAmount(
     return OptionalAmount(false);
   }
 
-  return ParseAmount(Beg, E);
+  return ParseAmount(Beg, E, Conv);
 }
 
 bool clang::analyze_format_string::ParseFieldWidth(
     FormatStringHandler &H, FormatSpecifier &CS, const char *Start,
-    const char *&Beg, const char *E, unsigned *argIndex) {
+    const char *&Beg, const char *E, unsigned *argIndex,
+    const llvm::TextEncodingConverter &Conv) {
   // FIXME: Support negative field widths.
   if (argIndex) {
-    CS.setFieldWidth(ParseNonPositionAmount(Beg, E, *argIndex));
+    CS.setFieldWidth(
+        ParseNonPositionAmount(Beg, E, *argIndex, Conv));
   } else {
     const OptionalAmount Amt = ParsePositionAmount(
-        H, Start, Beg, E, analyze_format_string::FieldWidthPos);
+        H, Start, Beg, E, analyze_format_string::FieldWidthPos,
+        Conv);
 
     if (Amt.isInvalid())
       return true;
@@ -145,22 +151,22 @@ bool clang::analyze_format_string::ParseFieldWidth(
   return false;
 }
 
-bool clang::analyze_format_string::ParseArgPosition(FormatStringHandler &H,
-                                                    FormatSpecifier &FS,
-                                                    const char *Start,
-                                                    const char *&Beg,
-                                                    const char *E) {
+bool clang::analyze_format_string::ParseArgPosition(
+    FormatStringHandler &H, FormatSpecifier &FS, const char *Start,
+    const char *&Beg, const char *E,
+    const llvm::TextEncodingConverter &Conv) {
   const char *I = Beg;
 
-  const OptionalAmount &Amt = ParseAmount(I, E);
+  const OptionalAmount &Amt = ParseAmount(I, E, Conv);
 
   if (I == E) {
     // No more characters left?
     H.HandleIncompleteSpecifier(Start, E - Start);
     return true;
   }
 
-  if (Amt.getHowSpecified() == OptionalAmount::Constant && *(I++) == '$') {
+  if (Amt.getHowSpecified() == OptionalAmount::Constant &&
+      Conv.convertBasicChar(*(I++)) == u8'$') {
     // Warn that positional arguments are non-standard.
     H.HandlePosition(Start, I - Start);
 
@@ -181,24 +187,23 @@ bool clang::analyze_format_string::ParseArgPosition(FormatStringHandler &H,
   return false;
 }
 
-bool clang::analyze_format_string::ParseVectorModifier(FormatStringHandler &H,
-                                                       FormatSpecifier &FS,
-                                                       const char *&I,
-                                                       const char *E,
-                                                       const LangOptions &LO) {
+bool clang::analyze_format_string::ParseVectorModifier(
+    FormatStringHandler &H, FormatSpecifier &FS, const char *&I, const char *E,
+    const LangOptions &LO,
+    const llvm::TextEncodingConverter &Conv) {
   if (!LO.OpenCL)
     return false;
 
   const char *Start = I;
-  if (*I == 'v') {
+  if (Conv.convertBasicChar(*I) == u8'v') {
     ++I;
 
     if (I == E) {
       H.HandleIncompleteSpecifier(Start, E - Start);
       return true;
     }
 
-    OptionalAmount NumElts = ParseAmount(I, E);
+    OptionalAmount NumElts = ParseAmount(I, E, Conv);
     if (NumElts.getHowSpecified() != OptionalAmount::Constant) {
       H.HandleIncompleteSpecifier(Start, E - Start);
       return true;
@@ -210,71 +215,75 @@ bool clang::analyze_format_string::ParseVectorModifier(FormatStringHandler &H,
   return false;
 }
 
-bool clang::analyze_format_string::ParseLengthModifier(FormatSpecifier &FS,
-                                                       const char *&I,
-                                                       const char *E,
-                                                       const LangOptions &LO,
-                                                       bool IsScanf) {
+bool clang::analyze_format_string::ParseLengthModifier(
+    FormatSpecifier &FS, const char *&I, const char *E, const LangOptions &LO,
+    const llvm::TextEncodingConverter &Conv,
+    bool IsScanf) {
   LengthModifier::Kind lmKind = LengthModifier::None;
   const char *lmPosition = I;
-  switch (*I) {
+  switch (Conv.convertBasicChar(*I)) {
   default:
     return false;
-  case 'h':
+  case u8'h':
     ++I;
-    if (I != E && *I == 'h') {
+    if (I != E && Conv.convertBasicChar(*I) == u8'h') {
       ++I;
       lmKind = LengthModifier::AsChar;
-    } else if (I != E && *I == 'l' && LO.OpenCL) {
+    } else if (I != E &&
+               Conv.convertBasicChar(*I) == u8'l' &&
+               LO.OpenCL) {
       ++I;
       lmKind = LengthModifier::AsShortLong;
     } else {
       lmKind = LengthModifier::AsShort;
     }
     break;
-  case 'l':
+  case u8'l':
     ++I;
-    if (I != E && *I == 'l') {
+    if (I != E && Conv.convertBasicChar(*I) == u8'l') {
       ++I;
       lmKind = LengthModifier::AsLongLong;
     } else {
       lmKind = LengthModifier::AsLong;
     }
     break;
-  case 'j':
+  case u8'j':
     lmKind = LengthModifier::AsIntMax;
     ++I;
     break;
-  case 'z':
+  case u8'z':
     lmKind = LengthModifier::AsSizeT;
     ++I;
     break;
-  case 't':
+  case u8't':
     lmKind = LengthModifier::AsPtrDiff;
     ++I;
     break;
-  case 'L':
+  case u8'L':
     lmKind = LengthModifier::AsLongDouble;
     ++I;
     break;
-  case 'q':
+  case u8'q':
     lmKind = LengthModifier::AsQuad;
     ++I;
     break;
-  case 'a':
+  case u8'a':
     if (IsScanf && !LO.C99 && !LO.CPlusPlus11) {
       // For scanf in C90, look at the next character to see if this should
       // be parsed as the GNU extension 'a' length modifier. If not, this
       // will be parsed as a conversion specifier.
       ++I;
-      if (I != E && (*I == 's' || *I == 'S' || *I == '[')) {
+      if (I != E &&
+          (Conv.convertBasicChar(*I) == u8's' ||
+           Conv.convertBasicChar(*I) == u8'S' ||
+           Conv.convertBasicChar(*I) == u8'[')) {
         lmKind = LengthModifier::AsAllocate;
         break;
       }
       --I;
     }
     return false;
-  case 'm':
+  case u8'm':
     if (IsScanf) {
       lmKind = LengthModifier::AsMAllocate;
       ++I;
@@ -283,17 +292,19 @@ bool clang::analyze_format_string::ParseLengthModifier(FormatSpecifier &FS,
     return false;
   // printf: AsInt64, AsInt32, AsInt3264
   // scanf:  AsInt64
-  case 'I':
+  case u8'I':
     if (I + 1 != E && I + 2 != E) {
-      if (I[1] == '6' && I[2] == '4') {
+      if (Conv.convertBasicChar(I[1]) == u8'6' &&
+          Conv.convertBasicChar(I[2]) == u8'4') {
         I += 3;
         lmKind = LengthModifier::AsInt64;
         break;
       }
       if (IsScanf)
         return false;
 
-      if (I[1] == '3' && I[2] == '2') {
+      if (Conv.convertBasicChar(I[1]) == u8'3' &&
+          Conv.convertBasicChar(I[2]) == u8'2') {
         I += 3;
         lmKind = LengthModifier::AsInt32;
         break;