remove ParserConversionAction, add paramter to ParseStringLiteralExpression instead, add conversion to SystemCharset then back to UTF8 to handle escape sequences in system encoding

Author: abhina-sree

clang/include/clang/Basic/TargetInfo.h

@@ -38,6 +38,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/TextEncoding.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cassert>
@@ -323,6 +324,8 @@ class TargetInfo : public TransferrableTargetInfo,
 
   virtual ~TargetInfo();
 
+  std::unique_ptr<llvm::TextEncodingConverter> TargetStrConverter;
+
   /// Retrieve the target options.
   TargetOptions &getTargetOpts() const {
     assert(TargetOpts && "Missing target options");

clang/include/clang/Lex/TextEncoding.h

@@ -10,23 +10,30 @@
 #define LLVM_CLANG_LEX_TEXTENCODING_H
 
 #include "clang/Basic/LangOptions.h"
+#include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace llvm {
 class TextEncodingConverter;
 } // namespace llvm
 
 namespace clang {
-enum ConversionAction { CA_NoConversion, CA_ToLiteralEncoding };
+enum ConversionAction {
+  CA_NoConversion,
+  CA_ToSystemEncoding,
+  CA_ToLiteralEncoding
+};
 
 class TextEncoding {
   llvm::StringRef LiteralEncoding;
   std::unique_ptr<llvm::TextEncodingConverter> ToLiteralEncodingConverter;
+  std::unique_ptr<llvm::TextEncodingConverter> ToSystemEncodingConverter;
 
 public:
   llvm::TextEncodingConverter *getConverter(ConversionAction Action) const;
   static std::error_code
-  setConvertersFromOptions(TextEncoding &TE, const clang::LangOptions &Opts);
+  setConvertersFromOptions(TextEncoding &TE, const clang::LangOptions &Opts,
+                           clang::TargetInfo &TInfo);
 
   llvm::StringRef getLiteralEncoding() { return LiteralEncoding; }
 };

clang/include/clang/Parse/Parser.h

@@ -3866,8 +3866,9 @@ class Parser : public CodeCompletionHandler {
   /// associated with the class's definition.
   void PopParsingClass(Sema::ParsingClassState);
 
-  ExprResult ParseStringLiteralExpression(bool AllowUserDefinedLiteral,
-                                          bool Unevaluated);
+  ExprResult
+  ParseStringLiteralExpression(bool AllowUserDefinedLiteral, bool Unevaluated,
+                               ConversionAction CA = CA_ToLiteralEncoding);
 
   /// This routine is called when the '@' is seen and consumed.
   /// Current token is an Identifier and is not a 'try'. This
@@ -5706,7 +5707,6 @@ class Parser : public CodeCompletionHandler {
     bool Finished;
   };
   ObjCImplParsingDataRAII *CurParsedObjCImpl;
-  ConversionAction ParserConversionAction;
 
   /// StashAwayMethodOrFunctionBodyTokens -  Consume the tokens and store them
   /// for later parsing.

clang/lib/Basic/TargetInfo.cpp

@@ -194,6 +194,8 @@ TargetInfo::TargetInfo(const llvm::Triple &T) : Triple(T) {
   MaxOpenCLWorkGroupSize = 1024;
 
   MaxBitIntWidth.reset();
+
+  TargetStrConverter = nullptr;
 }
 
 // Out of line virtual dtor for TargetInfo.

clang/lib/Frontend/CompilerInstance.cpp

@@ -561,8 +561,8 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
   if (GetDependencyDirectives)
     PP->setDependencyDirectivesGetter(*GetDependencyDirectives);
 
-  if (auto EC = TextEncoding::setConvertersFromOptions(PP->getTextEncoding(),
-                                                       getLangOpts()))
+  if (auto EC = TextEncoding::setConvertersFromOptions(
+          PP->getTextEncoding(), getLangOpts(), getTarget()))
     PP->getDiagnostics().Report(clang::diag::err_fe_text_encoding_config)
         << PP->getTextEncoding().getLiteralEncoding();
 }

clang/lib/Lex/TextEncoding.cpp

@@ -17,30 +17,55 @@ TextEncoding::getConverter(ConversionAction Action) const {
   switch (Action) {
   case CA_ToLiteralEncoding:
     return ToLiteralEncodingConverter.get();
+  case CA_ToSystemEncoding:
+    return ToSystemEncodingConverter.get();
   default:
     return nullptr;
   }
 }
 
 std::error_code
 TextEncoding::setConvertersFromOptions(TextEncoding &TE,
-                                       const clang::LangOptions &Opts) {
+                                       const clang::LangOptions &Opts,
+                                       clang::TargetInfo &TInfo) {
   using namespace llvm;
 
   const char *UTF8 = "UTF-8";
   TE.LiteralEncoding =
       Opts.LiteralEncoding.empty() ? UTF8 : Opts.LiteralEncoding.c_str();
 
-  // Create converter between internal and literal encoding specified
-  // in fexec-charset option.
-  if (TE.LiteralEncoding == UTF8)
+  if (TE.LiteralEncoding != UTF8) {
+    ErrorOr<TextEncodingConverter> ErrorOrLiteralConverter =
+        llvm::TextEncodingConverter::create(UTF8, TE.LiteralEncoding);
+    if (ErrorOrLiteralConverter)
+      TE.ToLiteralEncodingConverter = std::make_unique<TextEncodingConverter>(
+          std::move(*ErrorOrLiteralConverter));
+    else
+      return ErrorOrLiteralConverter.getError();
+  }
+
+  if (TInfo.getDefaultOrdinaryLiteralEncoding() == UTF8)
     return std::error_code();
+
+  // Create converter between internal and default ordinary encoding for the
+  // target
   ErrorOr<TextEncodingConverter> ErrorOrConverter =
-      llvm::TextEncodingConverter::create(UTF8, TE.LiteralEncoding);
+      llvm::TextEncodingConverter::create(
+          UTF8, TInfo.getDefaultOrdinaryLiteralEncoding());
+  if (ErrorOrConverter)
+    TE.ToSystemEncodingConverter =
+        std::make_unique<TextEncodingConverter>(std::move(*ErrorOrConverter));
+  else
+    return ErrorOrConverter.getError();
+
+  ErrorOrConverter = llvm::TextEncodingConverter::create(
+      TInfo.getDefaultOrdinaryLiteralEncoding(), UTF8);
+
   if (ErrorOrConverter)
-    TE.ToLiteralEncodingConverter =
+    TInfo.TargetStrConverter =
         std::make_unique<TextEncodingConverter>(std::move(*ErrorOrConverter));
   else
     return ErrorOrConverter.getError();
+
   return std::error_code();
 }

clang/lib/Parse/ParseExpr.cpp

@@ -3021,7 +3021,8 @@ ExprResult Parser::ParseUnevaluatedStringLiteralExpression() {
 }
 
 ExprResult Parser::ParseStringLiteralExpression(bool AllowUserDefinedLiteral,
-                                                bool Unevaluated) {
+                                                bool Unevaluated,
+                                                ConversionAction CA) {
   assert(tokenIsLikeStringLiteral(Tok, getLangOpts()) &&
          "Not a string-literal-like token!");
 
@@ -3042,8 +3043,7 @@ ExprResult Parser::ParseStringLiteralExpression(bool AllowUserDefinedLiteral,
 
   // Pass the set of string tokens, ready for concatenation, to the actions.
   return Actions.ActOnStringLiteral(
-      StringToks, AllowUserDefinedLiteral ? getCurScope() : nullptr,
-      ParserConversionAction);
+      StringToks, AllowUserDefinedLiteral ? getCurScope() : nullptr, CA);
 }
 
 ExprResult Parser::ParseGenericSelectionExpression() {

clang/lib/Parse/Parser.cpp

@@ -70,8 +70,6 @@ Parser::Parser(Preprocessor &pp, Sema &actions, bool skipFunctionBodies)
   NumCachedScopes = 0;
   CurParsedObjCImpl = nullptr;
 
-  ParserConversionAction = CA_ToLiteralEncoding;
-
   // Add #pragma handlers. These are removed and destroyed in the
   // destructor.
   initializePragmaHandlers();
@@ -1553,12 +1551,12 @@ void Parser::ParseKNRParamDeclarations(Declarator &D) {
 }
 
 ExprResult Parser::ParseAsmStringLiteral(bool ForAsmLabel) {
-  SaveAndRestore<ConversionAction> SavedTranslationState(ParserConversionAction,
-                                                         CA_NoConversion);
 
   ExprResult AsmString;
   if (isTokenStringLiteral()) {
-    AsmString = ParseStringLiteralExpression();
+    AsmString = ParseStringLiteralExpression(/*AllowUserDefinedLiteral=*/false,
+                                             /*Unevaluated=*/false,
+                                             CA_ToSystemEncoding);
     if (AsmString.isInvalid())
       return AsmString;
 

clang/lib/Sema/SemaStmtAsm.cpp

@@ -245,6 +245,14 @@ ExprResult Sema::ActOnGCCAsmStmtString(Expr *Expr, bool ForAsmLabel) {
       Diag(Expr->getBeginLoc(), diag::err_asm_operand_empty_string)
           << SL->getSourceRange();
     }
+    if (Context.getTargetInfo().TargetStrConverter) {
+      SmallString<16> ConvertedAsm;
+      Context.getTargetInfo().TargetStrConverter->convert(SL->getString(),
+                                                          ConvertedAsm);
+      return StringLiteral::Create(Context, ConvertedAsm,
+                                   StringLiteralKind::Ordinary,
+                                   /*Pascal*/ false, {}, SL->getBeginLoc());
+    }
     return SL;
   }
   if (DiagnoseUnexpandedParameterPack(Expr))

clang/test/CodeGen/systemz-charset.c

@@ -67,7 +67,11 @@ const char* preprocessor_concatenation = HELLO_WORLD;
 //CHECK: c"\C8\85\93\93\96@\E6\96\99\93\84Z\00"
 //CHECK-UTF8: c"Hello World!\00"
 
-void asm_labeled_fn(void) __asm__("asm_sym");
+void asm_labeled_fn(void) __asm__("\174asm_sym");
 void asm_labeled_fn(void) {}
-//CHECK: define{{.*}} void @asm_sym(
-//CHECK-UTF8: define{{.*}} void @asm_sym(
+//CHECK: define{{.*}}void @"@asm_sym"(
+//CHECK-UTF8: define{{.*}}void @"@asm_sym"(
+
+int RC = 25;
+asm(" AHI %0,%1\n" : "+r"(RC) : "g"(30) );
+`