[features/run] Align with main

include/cudaq/Optimizer/Builder/Factory.h

@@ -186,6 +186,8 @@ std::optional<double> maybeValueOfFloatConstant(mlir::Value v);
 /// \em{not} control dependent (other than on function entry).
 mlir::Value createLLVMTemporary(mlir::Location loc, mlir::OpBuilder &builder,
                                 mlir::Type type, std::size_t size = 1);
+mlir::Value createTemporary(mlir::Location loc, mlir::OpBuilder &builder,
+                            mlir::Type type, std::size_t size = 1);
 
 //===----------------------------------------------------------------------===//
 

include/cudaq/Optimizer/Builder/Intrinsics.h

@@ -148,6 +148,11 @@ class IRBuilder : public mlir::OpBuilder {
   mlir::LogicalResult loadIntrinsic(mlir::ModuleOp module,
                                     llvm::StringRef name);
 
+  llvm::StringRef getIntrinsicText(llvm::StringRef name);
+  mlir::LogicalResult loadIntrinsicWithAliases(mlir::ModuleOp module,
+                                               llvm::StringRef name,
+                                               llvm::StringRef prefix);
+
   std::string hashStringByContent(llvm::StringRef sref);
 
   /// Generates code that yields the size of any type that can be reified in

include/cudaq/Optimizer/CodeGen/CodeGenOps.td

@@ -47,4 +47,22 @@ def cgq_RAIIOp : CGQOp<"qmem_raii", [MemoryEffects<[MemAlloc, MemWrite]>]> {
   }];
 }
 
+def cgq_MaterializeConstantArrayOp :
+    CGQOp<"materialize_constant_array", [Pure]> {
+  let summary = "Macro to materialize a cc.const_array into memory.";
+  let description = [{
+    This operation is the equivalent of creating some space in memory (such as
+    on the stack) and storing a cc.const_array value to that memory address.
+    This operation is needed for local rewrites, but is likely to be eliminated
+    if the constant array can be completely folded away.
+  }];
+
+  let arguments = (ins cc_ArrayType:$constArray);
+  let results = (outs cc_PointerType);
+
+  let assemblyFormat = [{
+    $constArray `:` functional-type(operands, results) attr-dict
+  }];
+}
+
 #endif

include/cudaq/Optimizer/CodeGen/Passes.h

@@ -32,17 +32,17 @@ namespace cudaq::opt {
 /// @param pm Pass Manager to add QIR passes to
 /// @param convertTo Expected to be `qir-base` or `qir-adaptive` (comes from the
 /// cudaq-translate command line `--convert-to` parameter)
-/// @param performPrep Whether or not to perform the initial prep pass (normally
-/// true, but false for the WireSet QIR path)
-void addQIRProfilePipeline(mlir::OpPassManager &pm, llvm::StringRef convertTo,
-                           bool performPrep = true);
+/// \deprecated Replaced by the convert to QIR API pipeline.
+void addQIRProfilePipeline(mlir::OpPassManager &pm, llvm::StringRef convertTo);
+
+void addQIRProfileVerify(mlir::OpPassManager &pm, llvm::StringRef convertTo);
 
 void addLowerToCCPipeline(mlir::OpPassManager &pm);
 void addWiresetToProfileQIRPipeline(mlir::OpPassManager &pm,
                                     llvm::StringRef profile);
 
-/// @brief Verify that all `CallOp` targets are QIR- or NVQIR-defined functions
-/// or in the provided allowed list.
+/// Verify that all `CallOp` targets are QIR- or NVQIR-defined functions or in
+/// the provided allowed list.
 std::unique_ptr<mlir::Pass>
 createVerifyNVQIRCallOpsPass(const std::vector<llvm::StringRef> &allowedFuncs);
 
@@ -61,7 +61,18 @@ void registerCodeGenDialect(mlir::DialectRegistry &registry);
 
 mlir::LLVM::LLVMStructType lambdaAsPairOfPointers(mlir::MLIRContext *context);
 
+/// The pipeline for lowering Quake code to the QIR API. There will be three
+/// distinct flavors of QIR that can be generated with this pipeline. These
+/// are `"qir"`, `"qir-base"`, and `"qir-adaptive"`. This pipeline should be run
+/// before conversion to the LLVM-IR dialect.
+void registerToQIRAPIPipeline();
+void addConvertToQIRAPIPipeline(mlir::OpPassManager &pm, mlir::StringRef api,
+                                bool opaquePtr = false);
+
+/// The pipeline for lowering Quake code to the execution manager API. This
+/// pipeline should be run before conversion to the LLVM-IR dialect.
 void registerToExecutionManagerCCPipeline();
+
 void registerWireSetToProfileQIRPipeline();
 void populateCCTypeConversions(mlir::LLVMTypeConverter *converter);
 

include/cudaq/Optimizer/CodeGen/Passes.td

@@ -65,7 +65,10 @@ def ConvertToQIR : Pass<"quake-to-qir", "mlir::ModuleOp"> {
 def LowerToCG : Pass<"lower-to-cg", "mlir::ModuleOp"> {
   let summary = "Lower Quake to CG dialect.";
   let description = [{
-    For testing purposes only.
+    Lower the Quake IR to the codegen dialect. The codegen dialect is used to
+    fuse small DAGs of IR into larger macro operations just prior to the final
+    codegen. This allows conversions to take place on the macro operations and
+    avoid some of the limitations of an MLIR conversion pass.
   }];
   let dependentDialects = [ "cudaq::codegen::CodeGenDialect" ];
 }
@@ -212,4 +215,61 @@ def WireSetToProfileQIRPrep :
   let dependentDialects = ["cudaq::cc::CCDialect", "mlir::func::FuncDialect"];
 }
 
+def QuakeToQIRAPI : Pass<"quake-to-qir-api"> {
+  let summary = "Convert the Quake dialect to the QIR API.";
+  let description = [{
+    This pass converts Quake operations to the QIR API as expressed in terms
+    of function calls to QIR functions.
+
+    Which QIR functions are to be used is parameterized on the `api` option.
+
+    This pass can lower to either use the obsolete opaque structure types (per
+    the QIR spec) or to use LLVM's currently supported opaque pointers. In the
+    latter case, type information is fully understood from the function names
+    themselves.
+  }];
+
+  let dependentDialects = ["cudaq::cc::CCDialect", "mlir::arith::ArithDialect",
+      "mlir::cf::ControlFlowDialect", "mlir::func::FuncDialect",
+      "mlir::LLVM::LLVMDialect"];
+
+  let options = [
+    Option<"api", "api", "std::string", /*default=*/"\"full\"",
+           "Select the QIR API to use.">,
+    Option<"opaquePtr", "opaque-pointer", "bool", /*default=*/"false",
+           "Use opaque pointers.">
+  ];
+}
+
+def QuakeToQIRAPIFinal : Pass<"quake-to-qir-api-final", "mlir::ModuleOp"> {
+  let summary = "Convert the Quake dialect to the QIR API finalization.";
+  let description = [{
+  }];
+
+  let dependentDialects = ["cudaq::cc::CCDialect", "mlir::arith::ArithDialect",
+      "mlir::cf::ControlFlowDialect", "mlir::func::FuncDialect",
+      "mlir::LLVM::LLVMDialect"];
+
+  let options = [
+    Option<"api", "api", "std::string", /*default=*/"\"full\"",
+           "Select the QIR API to use.">
+  ];
+}
+
+def QuakeToQIRAPIPrep : Pass<"quake-to-qir-api-prep", "mlir::ModuleOp"> {
+  let summary = "Convert the Quake dialect to the QIR API preparation.";
+  let description = [{
+  }];
+  let dependentDialects = ["cudaq::cc::CCDialect", "mlir::arith::ArithDialect",
+      "mlir::cf::ControlFlowDialect", "mlir::func::FuncDialect",
+      "mlir::LLVM::LLVMDialect", "cudaq::codegen::CodeGenDialect"];
+  let options = [
+    Option<"api", "api", "std::string", /*default=*/"\"full\"",
+           "Select the QIR API to use.">,
+    Option<"opaquePtr", "opaque-pointer", "bool", /*default=*/"false",
+           "Use opaque pointers.">
+  ];
+}
+
+
 #endif // CUDAQ_OPT_OPTIMIZER_CODEGEN_PASSES

include/cudaq/Optimizer/CodeGen/Peephole.h

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "cudaq/Optimizer/CodeGen/QIRFunctionNames.h"
+#include "cudaq/Optimizer/CodeGen/QIROpaqueStructTypes.h"
 #include "cudaq/Optimizer/Dialect/Quake/QuakeOps.h"
 #include "llvm/ADT/StringRef.h"
 #include "mlir/IR/ValueRange.h"

include/cudaq/Optimizer/CodeGen/Pipelines.h

@@ -21,25 +21,41 @@
 
 namespace cudaq::opt {
 
-/// The common pipeline.
-/// Adds the common pipeline (with or without a profile specifier) but without
-/// the final QIR profile lowering passes.
-void commonPipelineConvertToQIR(
+/// Adds the common pipeline. \p codeGenFor specifies which variant of QIR is to
+/// be generated: full, base-profile, adaptive-profile, etc. \p passConfigAs
+/// specifies which variant of QIR to use with \e other passes, and not the
+/// final `codegen`, in the pipeline. Typically, \p codeGenFor and \p
+/// passConfigAs will have identical values.
+void commonPipelineConvertToQIR(mlir::PassManager &pm,
+                                mlir::StringRef codeGenFor = "qir",
+                                mlir::StringRef passConfigAs = "qir");
+
+/// \deprecated{Only for Python, since it can't use the new QIR codegen.}
+void commonPipelineConvertToQIR_PythonWorkaround(
     mlir::PassManager &pm, const std::optional<mlir::StringRef> &convertTo);
 
 /// \brief Pipeline builder to convert Quake to QIR.
 /// Does not specify a particular QIR profile.
 inline void addPipelineConvertToQIR(mlir::PassManager &pm) {
-  commonPipelineConvertToQIR(pm, std::nullopt);
+  commonPipelineConvertToQIR(pm);
+}
+
+/// \deprecated{Only for Python, since it can't use the new QIR codegen.}
+inline void addPipelineConvertToQIR_PythonWorkaround(mlir::PassManager &pm) {
+  commonPipelineConvertToQIR_PythonWorkaround(pm, std::nullopt);
 }
 
 /// \brief Pipeline builder to convert Quake to QIR.
 /// Specifies a particular QIR profile in \p convertTo.
 /// \p pm Pass manager to append passes to
 /// \p convertTo name of QIR profile (e.g., `qir-base`, `qir-adaptive`, ...)
-inline void addPipelineConvertToQIR(mlir::PassManager &pm,
-                                    mlir::StringRef convertTo) {
-  commonPipelineConvertToQIR(pm, convertTo);
+void addPipelineConvertToQIR(mlir::PassManager &pm, mlir::StringRef convertTo);
+
+/// \deprecated{Only for Python, since it can't use the new QIR codegen.}
+inline void
+addPipelineConvertToQIR_PythonWorkaround(mlir::PassManager &pm,
+                                         mlir::StringRef convertTo) {
+  commonPipelineConvertToQIR_PythonWorkaround(pm, convertTo);
   addQIRProfilePipeline(pm, convertTo);
 }
 

include/cudaq/Optimizer/CodeGen/QIRAttributeNames.h

@@ -23,5 +23,7 @@ static constexpr const char QIRRequiredResultsAttrName[] = "requiredResults";
 static constexpr const char QIRIrreversibleFlagName[] = "irreversible";
 
 static constexpr const char StartingOffsetAttrName[] = "StartingOffset";
+static constexpr const char ResultIndexAttrName[] = "ResultIndex";
+static constexpr const char MzAssignedNameAttrName[] = "MzAssignedName";
 
 } // namespace cudaq::opt

include/cudaq/Optimizer/CodeGen/QIRFunctionNames.h

@@ -11,8 +11,6 @@
 /// This file provides some common QIR function names for use throughout our
 /// MLIR lowering infrastructure.
 
-#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
-
 namespace cudaq::opt {
 
 /// QIS Function name strings
@@ -21,14 +19,19 @@ static constexpr const char QIRMeasureBody[] = "__quantum__qis__mz__body";
 static constexpr const char QIRMeasure[] = "__quantum__qis__mz";
 static constexpr const char QIRMeasureToRegister[] =
     "__quantum__qis__mz__to__register";
+static constexpr const char QIRResetBody[] = "__quantum__qis__reset__body";
+static constexpr const char QIRReset[] = "__quantum__qis__reset";
 
-static constexpr const char QIRCnot[] = "__quantum__qis__cnot";
+static constexpr const char QIRCnot[] = "__quantum__qis__cnot__body";
 static constexpr const char QIRCphase[] = "__quantum__qis__cphase";
-static constexpr const char QIRCZ[] = "__quantum__qis__cz";
+static constexpr const char QIRCZ[] = "__quantum__qis__cz__body";
 static constexpr const char QIRReadResultBody[] =
     "__quantum__qis__read_result__body";
 
 static constexpr const char QIRCustomOp[] = "__quantum__qis__custom_unitary";
+static constexpr const char QIRCustomAdjOp[] =
+    "__quantum__qis__custom_unitary__adj";
+static constexpr const char QIRExpPauli[] = "__quantum__qis__exp_pauli";
 
 static constexpr const char NVQIRInvokeWithControlBits[] =
     "invokeWithControlQubits";
@@ -38,6 +41,8 @@ static constexpr const char NVQIRInvokeU3RotationWithControlBits[] =
     "invokeU3RotationWithControlQubits";
 static constexpr const char NVQIRInvokeWithControlRegisterOrBits[] =
     "invokeWithControlRegisterOrQubits";
+static constexpr const char NVQIRGeneralizedInvokeAny[] =
+    "generalizedInvokeWithRotationsControlsTargets";
 static constexpr const char NVQIRPackSingleQubitInArray[] =
     "packSingleQubitInArray";
 static constexpr const char NVQIRReleasePackedQubitArray[] =
@@ -89,30 +94,4 @@ static constexpr const char QIRRecordOutput[] =
 static constexpr const char QIRClearResultMaps[] =
     "__quantum__rt__clear_result_maps";
 
-inline mlir::Type getQuantumTypeByName(mlir::StringRef type,
-                                       mlir::MLIRContext *context) {
-  return mlir::LLVM::LLVMStructType::getOpaque(type, context);
-}
-
-inline mlir::Type getQubitType(mlir::MLIRContext *context) {
-  return mlir::LLVM::LLVMPointerType::get(
-      getQuantumTypeByName("Qubit", context));
-}
-
-inline mlir::Type getArrayType(mlir::MLIRContext *context) {
-  return mlir::LLVM::LLVMPointerType::get(
-      getQuantumTypeByName("Array", context));
-}
-
-inline mlir::Type getResultType(mlir::MLIRContext *context) {
-  return mlir::LLVM::LLVMPointerType::get(
-      getQuantumTypeByName("Result", context));
-}
-
-inline mlir::Type getCharPointerType(mlir::MLIRContext *context) {
-  return mlir::LLVM::LLVMPointerType::get(mlir::IntegerType::get(context, 8));
-}
-
-void initializeTypeConversions(mlir::LLVMTypeConverter &typeConverter);
-
 } // namespace cudaq::opt

include/cudaq/Optimizer/CodeGen/QIROpaqueStructTypes.h

@@ -0,0 +1,94 @@
+/****************************************************************-*- C++ -*-****
+ * Copyright (c) 2022 - 2024 NVIDIA Corporation & Affiliates.                  *
+ * All rights reserved.                                                        *
+ *                                                                             *
+ * This source code and the accompanying materials are made available under    *
+ * the terms of the Apache License 2.0 which accompanies this distribution.    *
+ ******************************************************************************/
+
+#pragma once
+
+/// This file provides the opaque struct types to be used with the obsolete LLVM
+/// typed pointer type.
+
+#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+
+namespace cudaq {
+inline mlir::Type getQuantumTypeByName(mlir::StringRef type,
+                                       mlir::MLIRContext *context) {
+  return mlir::LLVM::LLVMStructType::getOpaque(type, context);
+}
+
+namespace opt {
+
+// The following type creators are deprecated and should only be used in the
+// older codegen passes. Use the creators in the cg namespace immediately below
+// instead.
+inline mlir::Type getOpaquePointerType(mlir::MLIRContext *context) {
+  return mlir::LLVM::LLVMPointerType::get(context);
+}
+
+inline mlir::Type getQubitType(mlir::MLIRContext *context) {
+  return mlir::LLVM::LLVMPointerType::get(
+      getQuantumTypeByName("Qubit", context));
+}
+
+inline mlir::Type getArrayType(mlir::MLIRContext *context) {
+  return mlir::LLVM::LLVMPointerType::get(
+      getQuantumTypeByName("Array", context));
+}
+
+inline mlir::Type getResultType(mlir::MLIRContext *context) {
+  return mlir::LLVM::LLVMPointerType::get(
+      getQuantumTypeByName("Result", context));
+}
+
+inline mlir::Type getCharPointerType(mlir::MLIRContext *context) {
+  return mlir::LLVM::LLVMPointerType::get(mlir::IntegerType::get(context, 8));
+}
+
+void initializeTypeConversions(mlir::LLVMTypeConverter &typeConverter);
+
+} // namespace opt
+
+namespace cg {
+
+// The following type creators replace the ones above. They are configurable on
+// the fly to either use opaque structs or opaque pointers. The default is to
+// use pointers to opaque structs, which is no longer supported in modern LLVM.
+
+inline mlir::Type getOpaquePointerType(mlir::MLIRContext *context) {
+  return cc::PointerType::get(mlir::NoneType::get(context));
+}
+
+inline mlir::Type getQubitType(mlir::MLIRContext *context,
+                               bool useOpaquePtr = false) {
+  if (useOpaquePtr)
+    return getOpaquePointerType(context);
+  return cc::PointerType::get(getQuantumTypeByName("Qubit", context));
+}
+
+inline mlir::Type getArrayType(mlir::MLIRContext *context,
+                               bool useOpaquePtr = false) {
+  if (useOpaquePtr)
+    return getOpaquePointerType(context);
+  return cc::PointerType::get(getQuantumTypeByName("Array", context));
+}
+
+inline mlir::Type getResultType(mlir::MLIRContext *context,
+                                bool useOpaquePtr = false) {
+  if (useOpaquePtr)
+    return getOpaquePointerType(context);
+  return cc::PointerType::get(getQuantumTypeByName("Result", context));
+}
+
+inline mlir::Type getCharPointerType(mlir::MLIRContext *context,
+                                     bool useOpaquePtr = false) {
+  if (useOpaquePtr)
+    return getOpaquePointerType(context);
+  return cc::PointerType::get(mlir::IntegerType::get(context, 8));
+}
+
+} // namespace cg
+} // namespace cudaq