ConvertToQIRAPI.cpp

/*******************************************************************************
 * Copyright (c) 2022 - 2024 NVIDIA Corporation & Affiliates.                  *
 * All rights reserved.                                                        *
 *                                                                             *
 * This source code and the accompanying materials are made available under    *
 * the terms of the Apache License 2.0 which accompanies this distribution.    *
 ******************************************************************************/

#include "CodeGenOps.h"
#include "cudaq/Optimizer/Builder/Intrinsics.h"
#include "cudaq/Optimizer/Builder/Runtime.h"
#include "cudaq/Optimizer/CodeGen/CodeGenDialect.h"
#include "cudaq/Optimizer/CodeGen/Passes.h"
#include "cudaq/Optimizer/CodeGen/Pipelines.h"
#include "cudaq/Optimizer/CodeGen/QIRAttributeNames.h"
#include "cudaq/Optimizer/CodeGen/QIRFunctionNames.h"
#include "cudaq/Optimizer/CodeGen/QIROpaqueStructTypes.h"
#include "cudaq/Optimizer/CodeGen/QuakeToExecMgr.h"
#include "cudaq/Optimizer/Dialect/CC/CCDialect.h"
#include "cudaq/Optimizer/Dialect/CC/CCOps.h"
#include "cudaq/Optimizer/Dialect/Quake/QuakeDialect.h"
#include "cudaq/Optimizer/Dialect/Quake/QuakeOps.h"
#include "nlohmann/json.hpp"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Pass/PassOptions.h"
#include "mlir/Transforms/DialectConversion.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

#define DEBUG_TYPE "convert-to-qir-api"

namespace cudaq::opt {
#define GEN_PASS_DEF_QUAKETOQIRAPI
#define GEN_PASS_DEF_QUAKETOQIRAPIPREP
#define GEN_PASS_DEF_QUAKETOQIRAPIFINAL
#include "cudaq/Optimizer/CodeGen/Passes.h.inc"
} // namespace cudaq::opt

using namespace mlir;

//===----------------------------------------------------------------------===//

static std::string getGateName(Operation *op) {
  return op->getName().stripDialect().str();
}

static std::string getGateFunctionPrefix(Operation *op) {
  return cudaq::opt::QIRQISPrefix + getGateName(op);
}

constexpr std::array<std::string_view, 2> filterAdjointNames = {"s", "t"};

template <typename OP>
std::pair<std::string, bool> generateGateFunctionName(OP op) {
  auto prefix = getGateFunctionPrefix(op.getOperation());
  auto gateName = getGateName(op.getOperation());
  if (op.isAdj()) {
    if (std::find(filterAdjointNames.begin(), filterAdjointNames.end(),
                  gateName) != filterAdjointNames.end())
      prefix += "dg";
  }
  if (!op.getControls().empty())
    return {prefix + "__ctl", false};
  return {prefix, true};
}

static Value createGlobalCString(Operation *op, Location loc,
                                 ConversionPatternRewriter &rewriter,
                                 StringRef regName) {
  cudaq::IRBuilder irb(rewriter.getContext());
  auto mod = op->getParentOfType<ModuleOp>();
  auto nameObj = irb.genCStringLiteralAppendNul(loc, mod, regName);
  Value nameVal = rewriter.create<cudaq::cc::AddressOfOp>(
      loc, cudaq::cc::PointerType::get(nameObj.getType()), nameObj.getName());
  auto cstrTy = cudaq::cc::PointerType::get(rewriter.getI8Type());
  return rewriter.create<cudaq::cc::CastOp>(loc, cstrTy, nameVal);
}

/// Use modifier class classes to specialize the QIR API to a particular flavor
/// of QIR. For example, the names of the actual functions in "full QIR" are
/// different than the names used by the other API flavors.
namespace {

//===----------------------------------------------------------------------===//
// Type converter
//===----------------------------------------------------------------------===//

/// Type converter for converting quake dialect to one of the QIR APIs. This
/// class is used for conversions as well as instantiating QIR types in
/// conversion patterns.

struct QIRAPITypeConverter : public TypeConverter {
  using TypeConverter::convertType;

  QIRAPITypeConverter(bool useOpaque) : useOpaque(useOpaque) {
    addConversion([&](Type ty) { return ty; });
    addConversion([&](FunctionType ft) { return convertFunctionType(ft); });
    addConversion([&](cudaq::cc::PointerType ty) {
      return cudaq::cc::PointerType::get(convertType(ty.getElementType()));
    });
    addConversion([&](cudaq::cc::CallableType ty) {
      auto newSig = cast<FunctionType>(convertType(ty.getSignature()));
      return cudaq::cc::CallableType::get(newSig);
    });
    addConversion([&](cudaq::cc::IndirectCallableType ty) {
      auto newSig = cast<FunctionType>(convertType(ty.getSignature()));
      return cudaq::cc::IndirectCallableType::get(newSig);
    });
    addConversion(
        [&](quake::VeqType ty) { return getArrayType(ty.getContext()); });
    addConversion(
        [&](quake::RefType ty) { return getQubitType(ty.getContext()); });
    addConversion(
        [&](quake::WireType ty) { return getQubitType(ty.getContext()); });
    addConversion(
        [&](quake::ControlType ty) { return getQubitType(ty.getContext()); });
    addConversion(
        [&](quake::MeasureType ty) { return getResultType(ty.getContext()); });
    addConversion([&](quake::StruqType ty) { return convertStruqType(ty); });
  }

  Type convertFunctionType(FunctionType ty) {
    SmallVector<Type> args;
    if (failed(convertTypes(ty.getInputs(), args)))
      return {};
    SmallVector<Type> res;
    if (failed(convertTypes(ty.getResults(), res)))
      return {};
    return FunctionType::get(ty.getContext(), args, res);
  }

  Type convertStruqType(quake::StruqType ty) {
    SmallVector<Type> mems;
    mems.reserve(ty.getNumMembers());
    if (failed(convertTypes(ty.getMembers(), mems)))
      return {};
    return cudaq::cc::StructType::get(ty.getContext(), mems);
  }

  Type getQubitType(MLIRContext *ctx) {
    return cudaq::cg::getQubitType(ctx, useOpaque);
  }
  Type getArrayType(MLIRContext *ctx) {
    return cudaq::cg::getArrayType(ctx, useOpaque);
  }
  Type getResultType(MLIRContext *ctx) {
    return cudaq::cg::getResultType(ctx, useOpaque);
  }

  bool useOpaque;
};
} // namespace

namespace {

//===----------------------------------------------------------------------===//
// Conversion patterns
//===----------------------------------------------------------------------===//

template <typename M>
struct AllocaOpToCallsRewrite : public OpConversionPattern<quake::AllocaOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::AllocaOp alloc, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    // If this alloc is just returning a qubit
    if (auto resultType =
            dyn_cast_if_present<quake::RefType>(alloc.getType())) {

      // StringRef qirQubitAllocate = cudaq::opt::QIRQubitAllocate;
      StringRef qirQubitAllocate = cudaq::opt::QIRQubitAllocate;
      Type qubitTy = M::getQubitType(rewriter.getContext());

      rewriter.replaceOpWithNewOp<func::CallOp>(alloc, TypeRange{qubitTy},
                                                qirQubitAllocate, ValueRange{});
      return success();
    }

    // Create a QIR call to allocate the qubits.
    StringRef qirQubitArrayAllocate = cudaq::opt::QIRArrayQubitAllocateArray;
    Type arrayQubitTy = M::getArrayType(rewriter.getContext());

    // AllocaOp could have a size operand, or the size could be compile time
    // known and encoded in the veq return type.
    Value sizeOperand;
    auto loc = alloc.getLoc();
    if (adaptor.getOperands().empty()) {
      auto type = alloc.getType().cast<quake::VeqType>();
      if (!type.hasSpecifiedSize())
        return failure();
      auto constantSize = type.getSize();
      sizeOperand =
          rewriter.create<arith::ConstantIntOp>(loc, constantSize, 64);
    } else {
      sizeOperand = adaptor.getOperands().front();
      auto sizeOpTy = cast<IntegerType>(sizeOperand.getType());
      if (sizeOpTy.getWidth() < 64)
        sizeOperand = rewriter.create<cudaq::cc::CastOp>(
            loc, rewriter.getI64Type(), sizeOperand,
            cudaq::cc::CastOpMode::Unsigned);
      else if (sizeOpTy.getWidth() > 64)
        sizeOperand = rewriter.create<cudaq::cc::CastOp>(
            loc, rewriter.getI64Type(), sizeOperand);
    }

    // Replace the AllocaOp with the QIR call.
    rewriter.replaceOpWithNewOp<func::CallOp>(alloc, TypeRange{arrayQubitTy},
                                              qirQubitArrayAllocate,
                                              ValueRange{sizeOperand});
    return success();
  }
};

template <typename M>
struct AllocaOpToIntRewrite : public OpConversionPattern<quake::AllocaOp> {
  using OpConversionPattern::OpConversionPattern;

  // Precondition: every allocation must have been annotated with a starting
  // index by the preparation phase.
  LogicalResult
  matchAndRewrite(quake::AllocaOp alloc, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    if (!alloc->hasAttr(cudaq::opt::StartingOffsetAttrName))
      return alloc.emitOpError("allocation must be annotated.");

    auto loc = alloc.getLoc();
    // If this alloc is just returning a qubit, so just replace it with the
    // attribute.
    Type ty = alloc.getType();
    if (!ty)
      return alloc.emitOpError("quake alloca is malformed");
    auto startingOffsetAttr =
        alloc->getAttr(cudaq::opt::StartingOffsetAttrName);
    auto startingOffset = cast<IntegerAttr>(startingOffsetAttr).getInt();

    // In the case this is allocating a single qubit, we can just substitute
    // the startingIndex as the qubit value. Voila!
    if (auto resultType = dyn_cast<quake::RefType>(ty)) {
      Value index =
          rewriter.create<arith::ConstantIntOp>(loc, startingOffset, 64);
      auto qubitTy = M::getQubitType(rewriter.getContext());
      rewriter.replaceOpWithNewOp<cudaq::cc::CastOp>(alloc, qubitTy, index);
      return success();
    }

    auto veqTy = dyn_cast<quake::VeqType>(ty);
    if (!veqTy)
      return alloc.emitOpError("quake alloca must be a veq");
    if (!veqTy.hasSpecifiedSize())
      return alloc.emitOpError("quake alloca must be a veq with constant size");

    // Otherwise, the allocation is of a sequence of qubits. Here, we allocate a
    // constant array value with the qubit integral values in an ascending
    // sequence. These will be accessed by extract_value or used collectively.
    auto *ctx = rewriter.getContext();
    const std::int64_t veqSize = veqTy.getSize();
    auto arrTy = cudaq::cc::ArrayType::get(ctx, rewriter.getI64Type(), veqSize);
    SmallVector<std::int64_t> data;
    for (std::int64_t i = 0; i < veqSize; ++i)
      data.emplace_back(startingOffset + i);
    auto arr = rewriter.create<cudaq::cc::ConstantArrayOp>(
        loc, arrTy, rewriter.getI64ArrayAttr(data));
    Type qirArrTy = M::getArrayType(rewriter.getContext());
    rewriter.replaceOpWithNewOp<cudaq::codegen::MaterializeConstantArrayOp>(
        alloc, qirArrTy, arr);
    return success();
  }
};

struct MaterializeConstantArrayOpRewrite
    : public OpConversionPattern<cudaq::codegen::MaterializeConstantArrayOp> {
  using OpConversionPattern::OpConversionPattern;

  // Rewrite this operation into a stack allocation and storing the array value
  // to that stack slot.
  // TODO: it is more efficient to use a global constant, which is done by the
  // pass `globalize-array-values`.
  LogicalResult
  matchAndRewrite(cudaq::codegen::MaterializeConstantArrayOp mca,
                  OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = mca.getLoc();
    auto arr = adaptor.getConstArray();
    auto veqSize = cast<cudaq::cc::ArrayType>(arr.getType()).getSize();
    Value stackObj = cudaq::opt::factory::createTemporary(
        loc, rewriter, rewriter.getI64Type(), veqSize);
    rewriter.create<cudaq::cc::StoreOp>(loc, arr, stackObj);
    auto ty = mca.getType();
    rewriter.replaceOpWithNewOp<cudaq::cc::CastOp>(mca, ty, stackObj);
    return success();
  }
};

template <typename M, typename OP>
struct QubitHelperConversionPattern : public OpConversionPattern<OP> {
  using Base = OpConversionPattern<OP>;
  using Base::Base;

  Value wrapQubitAsArray(Location loc, ConversionPatternRewriter &rewriter,
                         Value val) const {
    Type qubitTy = M::getQubitType(rewriter.getContext());
    if (val.getType() != qubitTy)
      return val;

    // Create a QIR array container of 1 element.
    auto ptrTy = cudaq::cc::PointerType::get(rewriter.getNoneType());
    Value sizeofPtrVal =
        rewriter.create<cudaq::cc::SizeOfOp>(loc, rewriter.getI32Type(), ptrTy);
    Value one = rewriter.create<arith::ConstantIntOp>(loc, 1, 64);
    Type arrayTy = M::getArrayType(rewriter.getContext());
    auto newArr = rewriter.create<func::CallOp>(
        loc, TypeRange{arrayTy}, cudaq::opt::QIRArrayCreateArray,
        ArrayRef<Value>{sizeofPtrVal, one});
    Value result = newArr.getResult(0);

    // Get a pointer to element 0.
    Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 64);
    auto ptrQubitTy = cudaq::cc::PointerType::get(qubitTy);
    auto elePtr = rewriter.create<func::CallOp>(
        loc, TypeRange{ptrQubitTy}, cudaq::opt::QIRArrayGetElementPtr1d,
        ArrayRef<Value>{result, zero});

    // Write the qubit into the array at position 0.
    auto castVal = rewriter.create<cudaq::cc::CastOp>(loc, qubitTy, val);
    Value addr = elePtr.getResult(0);
    rewriter.create<cudaq::cc::StoreOp>(loc, castVal, addr);

    return result;
  }
};

template <typename M>
struct ConcatOpRewrite
    : public QubitHelperConversionPattern<M, quake::ConcatOp> {
  using Base = QubitHelperConversionPattern<M, quake::ConcatOp>;
  using Base::Base;

  // For this rewrite, we walk the list of operands (if any) and for each
  // operand, $o$, we ensure $o$ is already of type QIR array or convert $o$ to
  // the array type using QIR functions. Then, we walk the list and pairwise
  // concatenate each operand. First, take $c$ to be $o_0$ and then update $c$
  // to be the concat of the previous $c$ and $o_i \forall i \in \{ 1..N \}$.
  // This algorithm will generate a linear number of concat calls for the number
  // of operands.
  LogicalResult
  matchAndRewrite(quake::ConcatOp concat, Base::OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    if (adaptor.getOperands().empty()) {
      rewriter.eraseOp(concat);
      return success();
    }

    auto loc = concat.getLoc();
    Type arrayTy = M::getArrayType(rewriter.getContext());
    Value firstOperand = adaptor.getOperands().front();
    Value resultArray = Base::wrapQubitAsArray(loc, rewriter, firstOperand);
    for (auto next : adaptor.getOperands().drop_front()) {
      Value wrapNext = Base::wrapQubitAsArray(loc, rewriter, next);
      auto appended = rewriter.create<func::CallOp>(
          loc, arrayTy, cudaq::opt::QIRArrayConcatArray,
          ArrayRef<Value>{resultArray, wrapNext});
      resultArray = appended.getResult(0);
    }
    rewriter.replaceOp(concat, resultArray);
    return success();
  }
};

struct DeallocOpRewrite : public OpConversionPattern<quake::DeallocOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::DeallocOp dealloc, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto ty = dealloc.getReference().getType();
    StringRef qirFuncName = isa<quake::VeqType>(ty)
                                ? cudaq::opt::QIRArrayQubitReleaseArray
                                : cudaq::opt::QIRArrayQubitReleaseQubit;
    rewriter.replaceOpWithNewOp<func::CallOp>(dealloc, TypeRange{}, qirFuncName,
                                              adaptor.getReference());
    return success();
  }
};

struct DeallocOpErase : public OpConversionPattern<quake::DeallocOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::DeallocOp dealloc, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    rewriter.eraseOp(dealloc);
    return success();
  }
};

struct DiscriminateOpRewrite
    : public OpConversionPattern<quake::DiscriminateOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::DiscriminateOp disc, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = disc.getLoc();
    Value m = adaptor.getMeasurement();
    auto i1PtrTy = cudaq::cc::PointerType::get(rewriter.getI1Type());
    auto cast = rewriter.create<cudaq::cc::CastOp>(loc, i1PtrTy, m);
    rewriter.replaceOpWithNewOp<cudaq::cc::LoadOp>(disc, cast);
    return success();
  }
};

template <typename M>
struct DiscriminateOpToCallRewrite
    : public OpConversionPattern<quake::DiscriminateOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::DiscriminateOp disc, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    if constexpr (M::discriminateToClassical) {
      rewriter.replaceOpWithNewOp<func::CallOp>(disc, rewriter.getI1Type(),
                                                cudaq::opt::QIRReadResultBody,
                                                adaptor.getOperands());
    } else {
      rewriter.replaceOpWithNewOp<cudaq::cc::PoisonOp>(disc,
                                                       rewriter.getI1Type());
    }
    return success();
  }
};

template <typename M>
struct ExtractRefOpRewrite : public OpConversionPattern<quake::ExtractRefOp> {
  using OpConversionPattern::OpConversionPattern;

  // There are two cases depending on which flavor of QIR is being generated.
  // For full QIR, we need to generate calls to QIR functions to select the
  // qubit from a QIR array.
  // For the profile QIRs, we replace this with a `cc.extract_value` operation,
  // which will be canonicalized into a constant.
  LogicalResult
  matchAndRewrite(quake::ExtractRefOp extract, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = extract.getLoc();
    auto veq = adaptor.getVeq();
    auto i64Ty = rewriter.getI64Type();

    Value index;
    if (!adaptor.getIndex()) {
      index = rewriter.create<arith::ConstantIntOp>(
          loc, extract.getConstantIndex(), 64);
    } else {
      index = adaptor.getIndex();
      if (index.getType().isIntOrFloat()) {
        if (cast<IntegerType>(index.getType()).getWidth() < 64)
          index = rewriter.create<cudaq::cc::CastOp>(
              loc, i64Ty, index, cudaq::cc::CastOpMode::Unsigned);
        else if (cast<IntegerType>(index.getType()).getWidth() > 64)
          index = rewriter.create<cudaq::cc::CastOp>(loc, i64Ty, index);
      }
    }
    auto qubitTy = M::getQubitType(rewriter.getContext());

    if (auto mca =
            veq.getDefiningOp<cudaq::codegen::MaterializeConstantArrayOp>()) {
      // This is the profile QIR case.
      auto ext = rewriter.create<cudaq::cc::ExtractValueOp>(
          loc, i64Ty, mca.getConstArray(), index);
      rewriter.replaceOpWithNewOp<cudaq::cc::CastOp>(extract, qubitTy, ext);
      return success();
    }

    // Otherwise, this must be full QIR.
    auto call = rewriter.create<func::CallOp>(
        loc, cudaq::cc::PointerType::get(qubitTy),
        cudaq::opt::QIRArrayGetElementPtr1d, ArrayRef<Value>{veq, index});
    rewriter.replaceOpWithNewOp<cudaq::cc::LoadOp>(extract, call.getResult(0));
    return success();
  }
};

struct GetMemberOpRewrite : public OpConversionPattern<quake::GetMemberOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::GetMemberOp member, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto toTy = getTypeConverter()->convertType(member.getType());
    std::int32_t position = adaptor.getIndex();
    rewriter.replaceOpWithNewOp<cudaq::cc::ExtractValueOp>(
        member, toTy, adaptor.getStruq(), position);
    return success();
  }
};

struct VeqSizeOpRewrite : public OpConversionPattern<quake::VeqSizeOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::VeqSizeOp veqsize, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    rewriter.replaceOpWithNewOp<func::CallOp>(
        veqsize, TypeRange{veqsize.getType()}, cudaq::opt::QIRArrayGetSize,
        adaptor.getOperands());
    return success();
  }
};

struct MakeStruqOpRewrite : public OpConversionPattern<quake::MakeStruqOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::MakeStruqOp mkstruq, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = mkstruq.getLoc();
    auto *ctx = rewriter.getContext();
    auto toTy = getTypeConverter()->convertType(mkstruq.getType());
    Value result = rewriter.create<cudaq::cc::UndefOp>(loc, toTy);
    std::int64_t count = 0;
    for (auto op : adaptor.getOperands()) {
      auto off = DenseI64ArrayAttr::get(ctx, ArrayRef<std::int64_t>{count});
      result =
          rewriter.create<cudaq::cc::InsertValueOp>(loc, toTy, result, op, off);
      count++;
    }
    rewriter.replaceOp(mkstruq, result);
    return success();
  }
};

template <typename M>
struct QmemRAIIOpRewrite : public OpConversionPattern<cudaq::codegen::RAIIOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(cudaq::codegen::RAIIOp raii, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = raii.getLoc();
    auto arrayTy = M::getArrayType(rewriter.getContext());

    // Get the CC Pointer for the state
    auto ccState = adaptor.getInitState();

    // Inspect the element type of the complex data, need to
    // know if its f32 or f64
    Type eleTy = raii.getInitElementType();
    if (auto elePtrTy = dyn_cast<cudaq::cc::PointerType>(eleTy))
      eleTy = elePtrTy.getElementType();
    if (auto arrayTy = dyn_cast<cudaq::cc::ArrayType>(eleTy))
      eleTy = arrayTy.getElementType();
    bool fromComplex = false;
    if (auto complexTy = dyn_cast<ComplexType>(eleTy)) {
      fromComplex = true;
      eleTy = complexTy.getElementType();
    }

    // Cascade to set functionName.
    StringRef functionName;
    Type ptrTy;
    if (isa<cudaq::cc::StateType>(eleTy)) {
      functionName = cudaq::opt::QIRArrayQubitAllocateArrayWithCudaqStatePtr;
      ptrTy = cudaq::cc::PointerType::get(
          cudaq::cc::StateType::get(rewriter.getContext()));
    } else if (eleTy == rewriter.getF64Type()) {
      if (fromComplex) {
        functionName = cudaq::opt::QIRArrayQubitAllocateArrayWithStateComplex64;
        ptrTy = cudaq::cc::PointerType::get(
            ComplexType::get(rewriter.getF64Type()));
      } else {
        functionName = cudaq::opt::QIRArrayQubitAllocateArrayWithStateFP64;
        ptrTy = cudaq::cc::PointerType::get(rewriter.getF64Type());
      }
    } else if (eleTy == rewriter.getF32Type()) {
      if (fromComplex) {
        functionName = cudaq::opt::QIRArrayQubitAllocateArrayWithStateComplex32;
        ptrTy = cudaq::cc::PointerType::get(
            ComplexType::get(rewriter.getF32Type()));
      } else {
        functionName = cudaq::opt::QIRArrayQubitAllocateArrayWithStateFP32;
        ptrTy = cudaq::cc::PointerType::get(rewriter.getF32Type());
      }
    }

    if (functionName.empty())
      return raii.emitOpError("initialize state has an invalid element type.");
    assert(ptrTy && "argument pointer type must be set");

    // Get the size of the qubit register
    Type allocTy = adaptor.getAllocType();
    auto i64Ty = rewriter.getI64Type();

    Value sizeOperand;
    if (!adaptor.getAllocSize()) {
      auto type = cast<quake::VeqType>(allocTy);
      auto constantSize = type.getSize();
      sizeOperand =
          rewriter.create<arith::ConstantIntOp>(loc, constantSize, 64);
    } else {
      sizeOperand = adaptor.getAllocSize();
      auto sizeTy = cast<IntegerType>(sizeOperand.getType());
      if (sizeTy.getWidth() < 64)
        sizeOperand = rewriter.create<cudaq::cc::CastOp>(
            loc, i64Ty, sizeOperand, cudaq::cc::CastOpMode::Unsigned);
      else if (sizeTy.getWidth() > 64)
        sizeOperand =
            rewriter.create<cudaq::cc::CastOp>(loc, i64Ty, sizeOperand);
    }

    // Call the allocation function
    Value casted = rewriter.create<cudaq::cc::CastOp>(loc, ptrTy, ccState);
    rewriter.replaceOpWithNewOp<func::CallOp>(
        raii, arrayTy, functionName, ArrayRef<Value>{sizeOperand, casted});
    return success();
  }
};

struct RelaxSizeOpErase : public OpConversionPattern<quake::RelaxSizeOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::RelaxSizeOp relax, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    rewriter.replaceOp(relax, relax.getInputVec());
    return success();
  }
};

template <typename M>
struct SubveqOpRewrite : public OpConversionPattern<quake::SubVeqOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::SubVeqOp subveq, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = subveq.getLoc();

    auto lowArg = [&]() -> Value {
      if (!adaptor.getLower())
        return rewriter.create<arith::ConstantIntOp>(loc, adaptor.getRawLower(),
                                                     64);
      return adaptor.getLower();
    }();
    auto highArg = [&]() -> Value {
      if (!adaptor.getUpper())
        return rewriter.create<arith::ConstantIntOp>(loc, adaptor.getRawUpper(),
                                                     64);
      return adaptor.getUpper();
    }();
    auto i64Ty = rewriter.getI64Type();
    auto extend = [&](Value &v) -> Value {
      if (auto intTy = dyn_cast<IntegerType>(v.getType())) {
        if (intTy.getWidth() < 64)
          return rewriter.create<cudaq::cc::CastOp>(
              loc, i64Ty, v, cudaq::cc::CastOpMode::Unsigned);
        if (intTy.getWidth() > 64)
          return rewriter.create<cudaq::cc::CastOp>(loc, i64Ty, v);
      }
      return v;
    };
    lowArg = extend(lowArg);
    highArg = extend(highArg);
    Value inArr = adaptor.getVeq();
    auto i32Ty = rewriter.getI32Type();
    Value one32 = rewriter.create<arith::ConstantIntOp>(loc, 1, i32Ty);
    Value one64 = rewriter.create<arith::ConstantIntOp>(loc, 1, i64Ty);
    auto arrayTy = M::getArrayType(rewriter.getContext());
    rewriter.replaceOpWithNewOp<func::CallOp>(
        subveq, arrayTy, cudaq::opt::QIRArraySlice,
        ArrayRef<Value>{inArr, one32, lowArg, one64, highArg});
    return success();
  }
};

//===----------------------------------------------------------------------===//
// Custom handing of irregular quantum gates.
//===----------------------------------------------------------------------===//

template <typename M>
struct CustomUnitaryOpPattern
    : public QubitHelperConversionPattern<M, quake::CustomUnitarySymbolOp> {
  using Base = QubitHelperConversionPattern<M, quake::CustomUnitarySymbolOp>;
  using Base::Base;

  LogicalResult
  matchAndRewrite(quake::CustomUnitarySymbolOp unitary, Base::OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    if (!unitary.getParameters().empty())
      return unitary.emitOpError(
          "Parameterized custom operations not yet supported.");

    auto loc = unitary.getLoc();
    auto arrayTy = M::getArrayType(rewriter.getContext());

    if (adaptor.getTargets().empty())
      return unitary.emitOpError("Custom operations must have targets.");

    // Concat all the targets into an array.
    auto targetArray =
        Base::wrapQubitAsArray(loc, rewriter, adaptor.getTargets().front());
    for (auto next : adaptor.getTargets().drop_front()) {
      auto wrapNext = Base::wrapQubitAsArray(loc, rewriter, next);
      auto result = rewriter.create<func::CallOp>(
          loc, arrayTy, cudaq::opt::QIRArrayConcatArray,
          ArrayRef<Value>{targetArray, wrapNext});
      targetArray = result.getResult(0);
    }

    // Concat all the controls (if any) into an array.
    Value controlArray;
    if (adaptor.getControls().empty()) {
      // Use a nullptr for when 0 control qubits are present.
      Value zero = rewriter.create<arith::ConstantIntOp>(loc, 0, 64);
      controlArray = rewriter.create<cudaq::cc::CastOp>(loc, arrayTy, zero);
    } else {
      controlArray =
          Base::wrapQubitAsArray(loc, rewriter, adaptor.getControls().front());
      for (auto next : adaptor.getControls().drop_front()) {
        auto wrapNext = Base::wrapQubitAsArray(loc, rewriter, next);
        auto result = rewriter.create<func::CallOp>(
            loc, arrayTy, cudaq::opt::QIRArrayConcatArray,
            ArrayRef<Value>{controlArray, wrapNext});
        controlArray = result.getResult(0);
      }
    }

    // Fetch the unitary matrix generator for this custom operation
    auto generatorSym = unitary.getGenerator();
    StringRef generatorName = generatorSym.getRootReference();
    const auto customOpName = extractCustomNamePart(generatorName);

    // Create a global string for the unitary name.
    auto nameOp = createGlobalCString(unitary, loc, rewriter, customOpName);

    auto complex64Ty = ComplexType::get(rewriter.getF64Type());
    auto complex64PtrTy = cudaq::cc::PointerType::get(complex64Ty);
    auto globalObj = cast<cudaq::cc::GlobalOp>(
        unitary->getParentOfType<ModuleOp>().lookupSymbol(generatorName));
    auto addrOp = rewriter.create<cudaq::cc::AddressOfOp>(
        loc, globalObj.getType(), generatorName);
    auto unitaryData =
        rewriter.create<cudaq::cc::CastOp>(loc, complex64PtrTy, addrOp);

    StringRef functionName =
        unitary.isAdj() ? cudaq::opt::QIRCustomAdjOp : cudaq::opt::QIRCustomOp;

    rewriter.replaceOpWithNewOp<func::CallOp>(
        unitary, TypeRange{}, functionName,
        ArrayRef<Value>{unitaryData, controlArray, targetArray, nameOp});

    return success();
  }

  // IMPORTANT: this must match the logic to generate global data globalName =
  // f'{nvqppPrefix}{opName}_generator_{numTargets}.rodata'
  std::string extractCustomNamePart(StringRef generatorName) const {
    auto globalName = generatorName.str();
    if (globalName.starts_with(cudaq::runtime::cudaqGenPrefixName)) {
      globalName = globalName.substr(cudaq::runtime::cudaqGenPrefixLength);
      const size_t pos = globalName.find("_generator");
      if (pos != std::string::npos)
        return globalName.substr(0, pos);
    }
    return {};
  }
};

struct ExpPauliOpPattern : public OpConversionPattern<quake::ExpPauliOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::ExpPauliOp pauli, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = pauli.getLoc();
    SmallVector<Value> operands = adaptor.getOperands();

    // First need to check the type of the Pauli word. We expect
    // a pauli_word directly `{i8*,i64}` or a string literal
    // `ptr<i8>`. If it is a string literal, we need to map it to
    // a pauli word.
    auto pauliWord = operands.back();
    auto i8PtrTy = cudaq::cc::PointerType::get(rewriter.getI8Type());
    if (auto ptrTy = dyn_cast<cudaq::cc::PointerType>(pauliWord.getType())) {
      // Make sure we have the right types to extract the
      // length of the string literal
      auto arrayTy = dyn_cast<cudaq::cc::ArrayType>(ptrTy.getElementType());
      if (!arrayTy)
        return pauli.emitOpError(
            "exp_pauli string literal must have ptr<array<i8 x N> type.");
      if (!arrayTy.getSize())
        return pauli.emitOpError("string literal may not be empty.");

      // We must create the {i8*, i64} struct from the string literal
      SmallVector<Type> structTys{i8PtrTy, rewriter.getI64Type()};
      auto structTy =
          cudaq::cc::StructType::get(rewriter.getContext(), structTys);

      // Allocate the char span struct
      Value alloca =
          cudaq::opt::factory::createTemporary(loc, rewriter, structTy);

      // Convert the number of elements to a constant op.
      auto size =
          rewriter.create<arith::ConstantIntOp>(loc, arrayTy.getSize() - 1, 64);

      // Set the string literal data
      auto castedPauli =
          rewriter.create<cudaq::cc::CastOp>(loc, i8PtrTy, pauliWord);
      auto strPtr = rewriter.create<cudaq::cc::ComputePtrOp>(
          loc, cudaq::cc::PointerType::get(i8PtrTy), alloca,
          ArrayRef<cudaq::cc::ComputePtrArg>{0, 0});
      rewriter.create<cudaq::cc::StoreOp>(loc, castedPauli, strPtr);

      // Set the integer length
      auto intPtr = rewriter.create<cudaq::cc::ComputePtrOp>(
          loc, cudaq::cc::PointerType::get(rewriter.getI64Type()), alloca,
          ArrayRef<cudaq::cc::ComputePtrArg>{0, 1});
      rewriter.create<cudaq::cc::StoreOp>(loc, size, intPtr);

      // Cast to raw opaque pointer
      auto castedStore =
          rewriter.create<cudaq::cc::CastOp>(loc, i8PtrTy, alloca);
      operands.back() = castedStore;
      rewriter.replaceOpWithNewOp<func::CallOp>(
          pauli, TypeRange{}, cudaq::opt::QIRExpPauli, operands);
      return success();
    }

    // Here we know we have a pauli word expressed as `{i8*, i64}`.
    // Allocate a stack slot for it and store what we have to that pointer,
    // pass the pointer to NVQIR
    auto newPauliWord = adaptor.getOperands().back();
    auto newPauliWordTy = newPauliWord.getType();
    Value alloca =
        cudaq::opt::factory::createTemporary(loc, rewriter, newPauliWordTy);
    auto castedVar = rewriter.create<cudaq::cc::CastOp>(
        loc, cudaq::cc::PointerType::get(newPauliWordTy), alloca);
    rewriter.create<cudaq::cc::StoreOp>(loc, newPauliWord, castedVar);
    auto castedPauli = rewriter.create<cudaq::cc::CastOp>(loc, i8PtrTy, alloca);
    operands.back() = castedPauli;
    rewriter.replaceOpWithNewOp<func::CallOp>(
        pauli, TypeRange{}, cudaq::opt::QIRExpPauli, operands);
    return success();
  }
};

template <typename M>
struct MeasurementOpPattern : public OpConversionPattern<quake::MzOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::MzOp mz, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto loc = mz.getLoc();
    auto regNameAttr = dyn_cast<StringAttr>(mz.getRegisterNameAttr());
    if (!regNameAttr)
      return mz.emitOpError("mz operation must have a name.");
    if (regNameAttr.getValue().empty())
      return mz.emitOpError("mz name may not be an empty string.");
    SmallVector<Value> args;
    args.append(adaptor.getTargets().begin(), adaptor.getTargets().end());
    auto functionName = M::getQIRMeasure();

    // Are we using the measurement that returns a result?
    if constexpr (M::mzReturnsResultType) {
      // Yes, the measurement results the result, so we can use a
      // straightforward codegen pattern. Use either the mz or the
      // mz_to_register call (with the name as an extra argument) and forward
      // the result of the call as the result.

      if (mz->getAttr(cudaq::opt::MzAssignedNameAttrName)) {
        functionName = cudaq::opt::QIRMeasureToRegister;
        auto cstringGlobal =
            createGlobalCString(mz, loc, rewriter, regNameAttr.getValue());
        args.push_back(cstringGlobal);
      }
      auto resultTy = M::getResultType(rewriter.getContext());
      auto call = rewriter.replaceOpWithNewOp<func::CallOp>(mz, resultTy,
                                                            functionName, args);
      call->setAttr(cudaq::opt::QIRRegisterNameAttr, regNameAttr);
    } else {
      // No, the measurement doesn't return any result so use a much more
      // convoluted pattern.
      // 1. Cast an integer to the result and append it to the mz call. This
      // will be the token to identify the result. The value will have been
      // attached to the MzOp in preprocessing.
      // 2. Call the mz function.
      // 3. Call the result_record_output to bind the name, which is not folded
      // into the mz call. There is always a name in this case.

      auto resultAttr = mz->getAttr(cudaq::opt::ResultIndexAttrName);
      std::int64_t annInt = cast<IntegerAttr>(resultAttr).getInt();
      Value intVal = rewriter.create<arith::ConstantIntOp>(loc, annInt, 64);
      auto resultTy = M::getResultType(rewriter.getContext());
      Value res = rewriter.create<cudaq::cc::CastOp>(loc, resultTy, intVal);
      args.push_back(res);
      auto call =
          rewriter.create<func::CallOp>(loc, TypeRange{}, functionName, args);
      call->setAttr(cudaq::opt::QIRRegisterNameAttr, regNameAttr);
      auto cstringGlobal =
          createGlobalCString(mz, loc, rewriter, regNameAttr.getValue());
      if constexpr (!M::discriminateToClassical) {
        // These QIR profile variants force all record output calls to appear at
        // the end. In these variants, control-flow isn't allowed in the final
        // LLVM. Therefore, a single basic block is assumed but unchecked here
        // as the verifier will raise an error.
        rewriter.setInsertionPoint(rewriter.getBlock()->getTerminator());
      }
      auto recOut = rewriter.create<func::CallOp>(
          loc, TypeRange{}, cudaq::opt::QIRRecordOutput,
          ArrayRef<Value>{res, cstringGlobal});
      recOut->setAttr(cudaq::opt::ResultIndexAttrName, resultAttr);
      recOut->setAttr(cudaq::opt::QIRRegisterNameAttr, regNameAttr);
      rewriter.replaceOp(mz, res);
    }
    return success();
  }
};

template <typename M>
struct ResetOpPattern : public OpConversionPattern<quake::ResetOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(quake::ResetOp reset, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    // Get the reset QIR function name
    auto qirFunctionName = M::getQIRReset();

    // Replace the quake op with the new call op.
    rewriter.replaceOpWithNewOp<func::CallOp>(
        reset, TypeRange{}, qirFunctionName, adaptor.getOperands());
    return success();
  }
};

struct AnnotateKernelsWithMeasurementStringsPattern
    : public OpConversionPattern<func::FuncOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(func::FuncOp func, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    constexpr const char PassthroughAttr[] = "passthrough";
    if (!func->hasAttr(cudaq::kernelAttrName))
      return failure();
    if (!func->hasAttr(PassthroughAttr))
      return failure();
    auto passthru = cast<ArrayAttr>(func->getAttr(PassthroughAttr));
    for (auto a : passthru) {
      if (auto strArrAttr = dyn_cast<ArrayAttr>(a)) {
        auto strAttr = dyn_cast<StringAttr>(strArrAttr[0]);
        if (!strAttr)
          continue;
        if (strAttr.getValue() == cudaq::opt::QIROutputNamesAttrName)
          return failure();
      }
    }

    // Lambda to help recover an integer value (the QIR qubit or result as an
    // integer).
    auto recoverIntValue = [&](Value v) -> std::optional<std::size_t> {
      auto cast = v.getDefiningOp<cudaq::cc::CastOp>();
      if (!cast)
        return {};
      return cudaq::opt::factory::maybeValueOfIntConstant(cast.getValue());
    };

    // If we're here, then `func` is a kernel, it has a passthrough attribute,
    // and the passthrough attribute does *not* have an output names entry.
    //
    // OUTPUT-NAME-MAP: At this point, we will try to heroically generate the
    // output names attribute for the QIR consumer. The content of the attribute
    // is a map from results back to pairs of qubits and names. The map is
    // encoded in a JSON string. The map is appended to the passthrough
    // attribute array.

    std::map<std::size_t, std::size_t> measMap;
    std::map<std::size_t, std::pair<std::size_t, std::string>> nameMap;
    func.walk([&](func::CallOp call) {
      auto calleeName = call.getCallee();
      if (calleeName == cudaq::opt::QIRMeasureBody) {
        auto qubit = recoverIntValue(call.getOperand(0));
        auto meas = recoverIntValue(call.getOperand(1));
        if (qubit && meas)
          measMap[*meas] = *qubit;
      } else if (calleeName == cudaq::opt::QIRRecordOutput) {
        auto resAttr = call->getAttr(cudaq::opt::ResultIndexAttrName);
        std::size_t res = cast<IntegerAttr>(resAttr).getInt();
        auto regNameAttr = call->getAttr(cudaq::opt::QIRRegisterNameAttr);
        std::string regName = cast<StringAttr>(regNameAttr).getValue().str();
        if (measMap.count(res)) {
          std::size_t qubit = measMap[res];
          nameMap[res] = std::pair{qubit, regName};
        }
      }
    });

    // If there were no measurements, then nothing to see here.
    if (nameMap.empty())
      return failure();

    // Append the name map. Use a `const T&` to introduce another layer of
    // brackets here to maintain backwards compatibility.
    const auto &outputNameMapRef = nameMap;
    nlohmann::json outputNames{outputNameMapRef};
    std::string outputNamesStr = outputNames.dump();
    SmallVector<Attribute> funcAttrs(passthru.begin(), passthru.end());
    funcAttrs.push_back(
        rewriter.getStrArrayAttr({cudaq::opt::QIROutputNamesAttrName,
                                  rewriter.getStringAttr(outputNamesStr)}));
    func->setAttr(PassthroughAttr, rewriter.getArrayAttr(funcAttrs));
    return success();
  }
};

//===----------------------------------------------------------------------===//
// Generic handling of regular quantum gates.
//===----------------------------------------------------------------------===//

template <typename M, typename OP>
struct QuantumGatePattern : public OpConversionPattern<OP> {
  using Base = OpConversionPattern<OP>;
  using Base::Base;

  LogicalResult
  matchAndRewrite(OP op, typename Base::OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto forwardOrEraseOp = [&]() {
      if (op.getResults().empty())
        rewriter.eraseOp(op);
      else
        rewriter.replaceOp(op, adaptor.getTargets());
      return success();
    };
    auto qirFunctionName = M::quakeToFuncName(op);

    // Make sure that apply-control-negations pass was run.
    if (adaptor.getNegatedQubitControls())
      return op.emitOpError("negated control qubits not allowed.");

    // Prepare any floating-point parameters.
    auto loc = op.getLoc();
    SmallVector<Value> opParams = adaptor.getParameters();
    if (!opParams.empty()) {
      // If this is adjoint, each parameter is negated.
      if (op.getIsAdj())
        for (std::size_t i = 0; i < opParams.size(); ++i)
          opParams[i] = rewriter.create<arith::NegFOp>(loc, opParams[i]);

      // Each parameter must be converted to double-precision.
      auto f64Ty = rewriter.getF64Type();
      for (std::size_t i = 0; i < opParams.size(); ++i) {
        if (opParams[i].getType().getIntOrFloatBitWidth() < 64)
          opParams[i] = rewriter.create<arith::ExtFOp>(loc, f64Ty, opParams[i]);
        else if (opParams[i].getType().getIntOrFloatBitWidth() > 64)
          opParams[i] =
              rewriter.create<arith::TruncFOp>(loc, f64Ty, opParams[i]);
      }
    }

    // If no control qubits or if there is 1 control and it is already a veq,
    // just add a call and forward the target qubits as needed.
    auto numControls = adaptor.getControls().size();
    if (op.getControls().empty() ||
        conformsToIntendedCall(numControls, op.getControls().front(), op,
                               qirFunctionName)) {
      SmallVector<Value> args{opParams.begin(), opParams.end()};
      args.append(adaptor.getControls().begin(), adaptor.getControls().end());
      args.append(adaptor.getTargets().begin(), adaptor.getTargets().end());
      qirFunctionName =
          specializeFunctionName(op, qirFunctionName, numControls);
      rewriter.create<func::CallOp>(loc, TypeRange{}, qirFunctionName, args);
      return forwardOrEraseOp();
    }

    // Otherwise, we'll use the generalized invoke helper function. This
    // function takes 4 size_t values, which delimit the different argument
    // types, a pointer to the function to be invoked, and varargs of all the
    // arguments being used. This function's signature is tuned so as to reduce
    // or eliminate the creation of auxiliary temporaries needs to make the
    // call to the helper.
    std::size_t numArrayCtrls = 0;
    SmallVector<Value> opArrCtrls;
    std::size_t numQubitCtrls = 0;
    SmallVector<Value> opQubitCtrls;
    Type i64Ty = rewriter.getI64Type();
    auto ptrNoneTy = M::getLLVMPointerType(rewriter.getContext());

    // Process the controls, sorting them by type.
    for (auto pr : llvm::zip(op.getControls(), adaptor.getControls())) {
      if (isa<quake::VeqType>(std::get<0>(pr).getType())) {
        numArrayCtrls++;
        auto sizeCall = rewriter.create<func::CallOp>(
            loc, i64Ty, cudaq::opt::QIRArrayGetSize,
            ValueRange{std::get<1>(pr)});
        // Arrays are encoded as pairs of arguments: length and Array*
        opArrCtrls.push_back(sizeCall.getResult(0));
        opArrCtrls.push_back(rewriter.create<cudaq::cc::CastOp>(
            loc, ptrNoneTy, std::get<1>(pr)));
      } else {
        numQubitCtrls++;
        // Qubits are simply the Qubit**
        opQubitCtrls.emplace_back(rewriter.create<cudaq::cc::CastOp>(
            loc, ptrNoneTy, std::get<1>(pr)));
      }
    }

    // Lookup and process the gate operation we're invoking.
    auto module = op->template getParentOfType<ModuleOp>();
    auto symOp = module.lookupSymbol(qirFunctionName);
    if (!symOp)
      return op.emitError("cannot find QIR function");
    auto funOp = dyn_cast<func::FuncOp>(symOp);
    if (!funOp)
      return op.emitError("cannot find " + qirFunctionName);
    FunctionType qirFunctionTy = funOp.getFunctionType();
    auto funCon =
        rewriter.create<func::ConstantOp>(loc, qirFunctionTy, qirFunctionName);
    auto funPtr =
        rewriter.create<cudaq::cc::FuncToPtrOp>(loc, ptrNoneTy, funCon);

    // Process the target qubits.
    auto numTargets = adaptor.getTargets().size();
    if (numTargets == 0)
      return op.emitOpError("quake op must have at least 1 target.");
    SmallVector<Value> opTargs;
    for (auto t : adaptor.getTargets())
      opTargs.push_back(rewriter.create<cudaq::cc::CastOp>(loc, ptrNoneTy, t));

    // Build the declared arguments for the helper call (5 total).
    SmallVector<Value> args;
    args.emplace_back(
        rewriter.create<arith::ConstantIntOp>(loc, opParams.size(), 64));
    args.emplace_back(
        rewriter.create<arith::ConstantIntOp>(loc, numArrayCtrls, 64));
    args.emplace_back(
        rewriter.create<arith::ConstantIntOp>(loc, numQubitCtrls, 64));
    args.emplace_back(
        rewriter.create<arith::ConstantIntOp>(loc, numTargets, 64));
    args.emplace_back(funPtr);

    // Finally, append the varargs to the end of the argument list.
    args.append(opParams.begin(), opParams.end());
    args.append(opArrCtrls.begin(), opArrCtrls.end());
    args.append(opQubitCtrls.begin(), opQubitCtrls.end());
    args.append(opTargs.begin(), opTargs.end());

    // Call the generalized version of the gate invocation.
    rewriter.create<LLVM::CallOp>(loc, TypeRange{},
                                  cudaq::opt::NVQIRGeneralizedInvokeAny, args);
    return forwardOrEraseOp();
  }

  static bool conformsToIntendedCall(std::size_t numControls, Value ctrl, OP op,
                                     StringRef qirFunctionName) {
    if (numControls != 1)
      return false;
    auto ctrlTy = ctrl.getType();
    auto trivialName = specializeFunctionName(op, qirFunctionName, numControls);
    const bool nameChanged = trivialName != qirFunctionName;
    if (nameChanged && !isa<quake::VeqType>(ctrlTy))
      return true;
    return !nameChanged && isa<quake::VeqType>(ctrlTy);
  }

  static StringRef specializeFunctionName(OP op, StringRef funcName,
                                          std::size_t numCtrls) {
    // Last resort to change the names of particular functions from the general
    // scheme to specialized names under the right conditions.
    if constexpr (std::is_same_v<OP, quake::XOp> && M::convertToCNot) {
      if (numCtrls == 1)
        return cudaq::opt::QIRCnot;
    }
    if constexpr (std::is_same_v<OP, quake::ZOp> && M::convertToCZ) {
      if (numCtrls == 1)
        return cudaq::opt::QIRCZ;
    }
    return funcName;
  }
};

//===----------------------------------------------------------------------===//
// Handling of functions, calls, and classic memory ops on callables.
//===----------------------------------------------------------------------===//

struct AllocaOpPattern : public OpConversionPattern<cudaq::cc::AllocaOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(cudaq::cc::AllocaOp alloc, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto eleTy = alloc.getElementType();
    auto newEleTy = getTypeConverter()->convertType(eleTy);
    if (eleTy == newEleTy)
      return failure();
    Value ss = alloc.getSeqSize();
    if (ss)
      rewriter.replaceOpWithNewOp<cudaq::cc::AllocaOp>(alloc, newEleTy, ss);
    else
      rewriter.replaceOpWithNewOp<cudaq::cc::AllocaOp>(alloc, newEleTy);
    return success();
  }
};

/// Convert the quake types in `func::FuncOp` signatures.
struct FuncSignaturePattern : public OpConversionPattern<func::FuncOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(func::FuncOp func, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto funcTy = func.getFunctionType();
    auto newFuncTy =
        cast<FunctionType>(getTypeConverter()->convertType(funcTy));
    if (funcTy == newFuncTy)
      return failure();
    if (funcTy.getNumInputs() && !func.getBody().empty()) {
      // Replace the block argument types.
      for (auto [blockArg, argTy] : llvm::zip(
               func.getBody().front().getArguments(), newFuncTy.getInputs()))
        blockArg.setType(argTy);
    }
    // Replace the signature.
    rewriter.updateRootInPlace(func,
                               [&]() { func.setFunctionType(newFuncTy); });
    return success();
  }
};

struct CreateLambdaPattern
    : public OpConversionPattern<cudaq::cc::CreateLambdaOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(cudaq::cc::CreateLambdaOp op, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto sigTy = cast<cudaq::cc::CallableType>(op.getSignature().getType());
    auto newSigTy =
        cast<cudaq::cc::CallableType>(getTypeConverter()->convertType(sigTy));
    if (sigTy == newSigTy)
      return failure();
    if (sigTy.getSignature().getNumInputs() && !op.getInitRegion().empty()) {
      // Replace the block argument types.
      for (auto [blockArg, argTy] :
           llvm::zip(op.getInitRegion().front().getArguments(),
                     newSigTy.getSignature().getInputs()))
        blockArg.setType(argTy);
    }
    // Replace the signature.
    rewriter.updateRootInPlace(op,
                               [&]() { op.getSignature().setType(newSigTy); });
    return success();
  }
};

struct CallableFuncPattern
    : public OpConversionPattern<cudaq::cc::CallableFuncOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(cudaq::cc::CallableFuncOp op, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto funcTy = op.getFunction().getType();
    auto newFuncTy =
        cast<FunctionType>(getTypeConverter()->convertType(funcTy));
    rewriter.replaceOpWithNewOp<cudaq::cc::CallableFuncOp>(op, newFuncTy,
                                                           op.getCallable());
    return success();
  }
};

template <typename OP>
struct OpInterfacePattern : public OpConversionPattern<OP> {
  using Base = OpConversionPattern<OP>;
  using Base::Base;
  using Base::getTypeConverter;

  LogicalResult
  matchAndRewrite(OP op, typename Base::OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto newResultTy = getTypeConverter()->convertType(op.getType());
    rewriter.replaceOpWithNewOp<OP>(op, newResultTy, adaptor.getOperands(),
                                    op->getAttrs());
    return success();
  }
};

using FuncConstantPattern = OpInterfacePattern<func::ConstantOp>;
using FuncToPtrPattern = OpInterfacePattern<cudaq::cc::FuncToPtrOp>;
using LoadOpPattern = OpInterfacePattern<cudaq::cc::LoadOp>;
using UndefOpPattern = OpInterfacePattern<cudaq::cc::UndefOp>;
using PoisonOpPattern = OpInterfacePattern<cudaq::cc::PoisonOp>;
using CastOpPattern = OpInterfacePattern<cudaq::cc::CastOp>;

struct InstantiateCallablePattern
    : public OpConversionPattern<cudaq::cc::InstantiateCallableOp> {
  using OpConversionPattern::OpConversionPattern;

  LogicalResult
  matchAndRewrite(cudaq::cc::InstantiateCallableOp op, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto sigTy = cast<cudaq::cc::CallableType>(op.getSignature().getType());
    auto newSigTy =
        cast<cudaq::cc::CallableType>(getTypeConverter()->convertType(sigTy));
    rewriter.replaceOpWithNewOp<cudaq::cc::InstantiateCallableOp>(
        op, newSigTy, op.getCallee(), op.getClosureData(),
        op.getNoCaptureAttr());
    return success();
  }
};

struct StoreOpPattern : public OpConversionPattern<cudaq::cc::StoreOp> {
  using Base = OpConversionPattern;
  using Base::Base;
  using Base::getTypeConverter;

  LogicalResult
  matchAndRewrite(cudaq::cc::StoreOp op, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    rewriter.replaceOpWithNewOp<cudaq::cc::StoreOp>(
        op, TypeRange{}, adaptor.getOperands(), op->getAttrs());
    return success();
  }
};

template <typename CALLOP>
struct CallOpInterfacePattern : public OpConversionPattern<CALLOP> {
  using Base = OpConversionPattern<CALLOP>;
  using Base::Base;
  using Base::getTypeConverter;

  LogicalResult
  matchAndRewrite(CALLOP op, typename Base::OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    SmallVector<Type> newResultTys;
    for (auto ty : op.getResultTypes())
      newResultTys.emplace_back(getTypeConverter()->convertType(ty));
    rewriter.replaceOpWithNewOp<CALLOP>(op, newResultTys, adaptor.getOperands(),
                                        op->getAttrs());
    return success();
  }
};

using CallOpPattern = CallOpInterfacePattern<func::CallOp>;
using CallIndirectOpPattern = CallOpInterfacePattern<func::CallIndirectOp>;
using CallCallableOpPattern = CallOpInterfacePattern<cudaq::cc::CallCallableOp>;
using CallIndirectCallableOpPattern =
    CallOpInterfacePattern<cudaq::cc::CallIndirectCallableOp>;

//===----------------------------------------------------------------------===//
// Patterns that are common to all QIR conversions.
//===----------------------------------------------------------------------===//

static void commonClassicalHandlingPatterns(RewritePatternSet &patterns,
                                            TypeConverter &typeConverter,
                                            MLIRContext *ctx) {
  patterns.insert<AllocaOpPattern, CallableFuncPattern, CallCallableOpPattern,
                  CallIndirectCallableOpPattern, CallIndirectOpPattern,
                  CallOpPattern, CastOpPattern, CreateLambdaPattern,
                  FuncConstantPattern, FuncSignaturePattern, FuncToPtrPattern,
                  InstantiateCallablePattern, LoadOpPattern, PoisonOpPattern,
                  StoreOpPattern, UndefOpPattern>(typeConverter, ctx);
}

static void commonQuakeHandlingPatterns(RewritePatternSet &patterns,
                                        TypeConverter &typeConverter,
                                        MLIRContext *ctx) {
  patterns.insert<GetMemberOpRewrite, MakeStruqOpRewrite, RelaxSizeOpErase,
                  VeqSizeOpRewrite>(typeConverter, ctx);
}

//===----------------------------------------------------------------------===//
// Modifier classes
//===----------------------------------------------------------------------===//

template <bool opaquePtr>
Type GetLLVMPointerType(MLIRContext *ctx) {
  if constexpr (opaquePtr) {
    return LLVM::LLVMPointerType::get(ctx);
  } else {
    return LLVM::LLVMPointerType::get(IntegerType::get(ctx, 8));
  }
}

/// The modifier class for the "full QIR" API.
template <bool opaquePtr>
struct FullQIR {
  using Self = FullQIR;

  template <typename QuakeOp>
  static std::string quakeToFuncName(QuakeOp op) {
    auto [prefix, _] = generateGateFunctionName(op);
    return prefix;
  }

  static void populateRewritePatterns(RewritePatternSet &patterns,
                                      TypeConverter &typeConverter) {
    auto *ctx = patterns.getContext();
    patterns.insert<
        /* Rewrites for qubit management and aggregation. */
        AllocaOpToCallsRewrite<Self>, ConcatOpRewrite<Self>, DeallocOpRewrite,
        DiscriminateOpRewrite, ExtractRefOpRewrite<Self>,
        QmemRAIIOpRewrite<Self>, SubveqOpRewrite<Self>,

        /* Irregular quantum operators. */
        CustomUnitaryOpPattern<Self>, ExpPauliOpPattern,
        MeasurementOpPattern<Self>, ResetOpPattern<Self>,

        /* Regular quantum operators. */
        QuantumGatePattern<Self, quake::HOp>,
        QuantumGatePattern<Self, quake::PhasedRxOp>,
        QuantumGatePattern<Self, quake::R1Op>,
        QuantumGatePattern<Self, quake::RxOp>,
        QuantumGatePattern<Self, quake::RyOp>,
        QuantumGatePattern<Self, quake::RzOp>,
        QuantumGatePattern<Self, quake::SOp>,
        QuantumGatePattern<Self, quake::SwapOp>,
        QuantumGatePattern<Self, quake::TOp>,
        QuantumGatePattern<Self, quake::U2Op>,
        QuantumGatePattern<Self, quake::U3Op>,
        QuantumGatePattern<Self, quake::XOp>,
        QuantumGatePattern<Self, quake::YOp>,
        QuantumGatePattern<Self, quake::ZOp>>(typeConverter, ctx);
    commonQuakeHandlingPatterns(patterns, typeConverter, ctx);
    commonClassicalHandlingPatterns(patterns, typeConverter, ctx);
  }

  static StringRef getQIRMeasure() { return cudaq::opt::QIRMeasure; }
  static StringRef getQIRReset() { return cudaq::opt::QIRReset; }

  static constexpr bool mzReturnsResultType = true;
  static constexpr bool convertToCNot = false;
  static constexpr bool convertToCZ = false;

  static Type getQubitType(MLIRContext *ctx) {
    return cudaq::cg::getQubitType(ctx, opaquePtr);
  }
  static Type getArrayType(MLIRContext *ctx) {
    return cudaq::cg::getArrayType(ctx, opaquePtr);
  }
  static Type getResultType(MLIRContext *ctx) {
    return cudaq::cg::getResultType(ctx, opaquePtr);
  }
  static Type getCharPointerType(MLIRContext *ctx) {
    return cudaq::cg::getCharPointerType(ctx, opaquePtr);
  }
  static Type getLLVMPointerType(MLIRContext *ctx) {
    return GetLLVMPointerType<opaquePtr>(ctx);
  }
};

/// The base modifier class for the "profile QIR" APIs.
template <bool opaquePtr>
struct AnyProfileQIR {
  using Self = AnyProfileQIR;

  template <typename QuakeOp>
  static std::string quakeToFuncName(QuakeOp op) {
    auto [prefix, isBarePrefix] = generateGateFunctionName(op);
    return isBarePrefix ? prefix + "__body" : prefix;
  }

  static void populateRewritePatterns(RewritePatternSet &patterns,
                                      TypeConverter &typeConverter) {
    auto *ctx = patterns.getContext();
    patterns.insert<
        /* Rewrites for qubit management and aggregation. */
        AllocaOpToIntRewrite<Self>, ConcatOpRewrite<Self>, DeallocOpErase,
        ExtractRefOpRewrite<Self>, QmemRAIIOpRewrite<Self>,
        SubveqOpRewrite<Self>,

        /* Irregular quantum operators. */
        CustomUnitaryOpPattern<Self>, ExpPauliOpPattern, ResetOpPattern<Self>,

        /* Regular quantum operators. */
        QuantumGatePattern<Self, quake::HOp>,
        QuantumGatePattern<Self, quake::PhasedRxOp>,
        QuantumGatePattern<Self, quake::R1Op>,
        QuantumGatePattern<Self, quake::RxOp>,
        QuantumGatePattern<Self, quake::RyOp>,
        QuantumGatePattern<Self, quake::RzOp>,
        QuantumGatePattern<Self, quake::SOp>,
        QuantumGatePattern<Self, quake::SwapOp>,
        QuantumGatePattern<Self, quake::TOp>,
        QuantumGatePattern<Self, quake::U2Op>,
        QuantumGatePattern<Self, quake::U3Op>,
        QuantumGatePattern<Self, quake::XOp>,
        QuantumGatePattern<Self, quake::YOp>,
        QuantumGatePattern<Self, quake::ZOp>>(typeConverter, ctx);
    commonQuakeHandlingPatterns(patterns, typeConverter, ctx);
    commonClassicalHandlingPatterns(patterns, typeConverter, ctx);
  }

  static StringRef getQIRMeasure() { return cudaq::opt::QIRMeasureBody; }
  static StringRef getQIRReset() { return cudaq::opt::QIRResetBody; }

  static constexpr bool mzReturnsResultType = false;
  static constexpr bool convertToCNot = true;
  static constexpr bool convertToCZ = true;

  static Type getQubitType(MLIRContext *ctx) {
    return cudaq::cg::getQubitType(ctx, opaquePtr);
  }
  static Type getArrayType(MLIRContext *ctx) {
    return cudaq::cg::getArrayType(ctx, opaquePtr);
  }
  static Type getResultType(MLIRContext *ctx) {
    return cudaq::cg::getResultType(ctx, opaquePtr);
  }
  static Type getCharPointerType(MLIRContext *ctx) {
    return cudaq::cg::getCharPointerType(ctx, opaquePtr);
  }
  static Type getLLVMPointerType(MLIRContext *ctx) {
    return GetLLVMPointerType<opaquePtr>(ctx);
  }
};

/// The QIR base profile modifier class.
template <bool opaquePtr>
struct BaseProfileQIR : public AnyProfileQIR<opaquePtr> {
  using Self = BaseProfileQIR;
  using Base = AnyProfileQIR<opaquePtr>;

  static void populateRewritePatterns(RewritePatternSet &patterns,
                                      TypeConverter &typeConverter) {
    Base::populateRewritePatterns(patterns, typeConverter);
    patterns
        .insert<DiscriminateOpToCallRewrite<Self>, MeasurementOpPattern<Self>>(
            typeConverter, patterns.getContext());
  }

  static constexpr bool discriminateToClassical = false;
};

/// The QIR adaptive profile modifier class.
template <bool opaquePtr>
struct AdaptiveProfileQIR : public AnyProfileQIR<opaquePtr> {
  using Self = AdaptiveProfileQIR;
  using Base = AnyProfileQIR<opaquePtr>;

  static void populateRewritePatterns(RewritePatternSet &patterns,
                                      TypeConverter &typeConverter) {
    Base::populateRewritePatterns(patterns, typeConverter);
    patterns
        .insert<DiscriminateOpToCallRewrite<Self>, MeasurementOpPattern<Self>>(
            typeConverter, patterns.getContext());
  }

  static constexpr bool discriminateToClassical = true;
};

//===----------------------------------------------------------------------===//
// Quake conversion to the QIR API driver pass.
//
// This is done in 3 phased: preparation, conversion, and finalization.
//===----------------------------------------------------------------------===//

/// Conversion of quake IR to QIR calls for the intended API.
struct QuakeToQIRAPIPass
    : public cudaq::opt::impl::QuakeToQIRAPIBase<QuakeToQIRAPIPass> {
  using QuakeToQIRAPIBase::QuakeToQIRAPIBase;

  template <typename A>
  void processOperation(QIRAPITypeConverter &typeConverter) {
    auto *op = getOperation();
    LLVM_DEBUG(llvm::dbgs() << "Before QIR API conversion:\n" << *op << '\n');
    auto *ctx = &getContext();
    RewritePatternSet patterns(ctx);
    A::populateRewritePatterns(patterns, typeConverter);
    ConversionTarget target(*ctx);
    target.addLegalDialect<arith::ArithDialect, cudaq::cc::CCDialect,
                           cf::ControlFlowDialect, func::FuncDialect,
                           LLVM::LLVMDialect>();
    target.addIllegalDialect<quake::QuakeDialect,
                             cudaq::codegen::CodeGenDialect>();
    target.addLegalOp<cudaq::codegen::MaterializeConstantArrayOp>();
    target.addDynamicallyLegalOp<func::FuncOp>(
        [&](func::FuncOp fn) { return !hasQuakeType(fn.getFunctionType()); });
    target.addDynamicallyLegalOp<func::ConstantOp>([&](func::ConstantOp fn) {
      return !hasQuakeType(fn.getResult().getType());
    });
    target.addDynamicallyLegalOp<cudaq::cc::UndefOp, cudaq::cc::PoisonOp>(
        [&](Operation *op) {
          return !hasQuakeType(op->getResult(0).getType());
        });
    target.addDynamicallyLegalOp<cudaq::cc::CallableFuncOp>(
        [&](cudaq::cc::CallableFuncOp op) {
          return !hasQuakeType(op.getFunction().getType());
        });
    target.addDynamicallyLegalOp<cudaq::cc::CreateLambdaOp>(
        [&](cudaq::cc::CreateLambdaOp op) {
          return !hasQuakeType(op.getSignature().getType());
        });
    target.addDynamicallyLegalOp<cudaq::cc::InstantiateCallableOp>(
        [&](cudaq::cc::InstantiateCallableOp op) {
          return !hasQuakeType(op.getSignature().getType());
        });
    target.addDynamicallyLegalOp<cudaq::cc::AllocaOp>(
        [&](cudaq::cc::AllocaOp op) {
          return !hasQuakeType(op.getElementType());
        });
    target.addDynamicallyLegalOp<
        func::CallOp, func::CallIndirectOp, cudaq::cc::CallCallableOp,
        cudaq::cc::CallIndirectCallableOp, cudaq::cc::CastOp,
        cudaq::cc::FuncToPtrOp, cudaq::cc::StoreOp, cudaq::cc::LoadOp>(
        [&](Operation *op) {
          for (auto opnd : op->getOperands())
            if (hasQuakeType(opnd.getType()))
              return false;
          for (auto res : op->getResults())
            if (hasQuakeType(res.getType()))
              return false;
          return true;
        });
    target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
    if (failed(applyPartialConversion(op, target, std::move(patterns))))
      signalPassFailure();
    LLVM_DEBUG(llvm::dbgs() << "After QIR API conversion:\n" << *op << '\n');
  }

  static bool hasQuakeType(Type ty) {
    if (auto pty = dyn_cast<cudaq::cc::PointerType>(ty))
      return hasQuakeType(pty.getElementType());
    if (auto cty = dyn_cast<cudaq::cc::CallableType>(ty))
      return hasQuakeType(cty.getSignature());
    if (auto cty = dyn_cast<cudaq::cc::IndirectCallableType>(ty))
      return hasQuakeType(cty.getSignature());
    if (auto fty = dyn_cast<FunctionType>(ty)) {
      for (auto t : fty.getInputs())
        if (hasQuakeType(t))
          return true;
      for (auto t : fty.getResults())
        if (hasQuakeType(t))
          return true;
      return false;
    }
    return quake::isQuakeType(ty);
  }

  void runOnOperation() override {
    LLVM_DEBUG(llvm::dbgs() << "Begin converting to QIR\n");
    QIRAPITypeConverter typeConverter(opaquePtr);
    if (api == "full") {
      if (opaquePtr)
        processOperation<FullQIR</*opaquePtr=*/true>>(typeConverter);
      processOperation<FullQIR</*opaquePtr=*/false>>(typeConverter);
    } else if (api == "base-profile") {
      if (opaquePtr)
        processOperation<BaseProfileQIR</*opaquePtr=*/true>>(typeConverter);
      processOperation<BaseProfileQIR</*opaquePtr=*/false>>(typeConverter);
    } else if (api == "adaptive-profile") {
      if (opaquePtr)
        processOperation<AdaptiveProfileQIR</*opaquePtr=*/true>>(typeConverter);
      processOperation<AdaptiveProfileQIR</*opaquePtr=*/false>>(typeConverter);
    } else {
      getOperation()->emitOpError("The currently supported APIs are: 'full', "
                                  "'base-profile', 'adaptive-profile'.");
      signalPassFailure();
    }
  }
};

struct QuakeToQIRAPIPrepPass
    : public cudaq::opt::impl::QuakeToQIRAPIPrepBase<QuakeToQIRAPIPrepPass> {
  using QuakeToQIRAPIPrepBase::QuakeToQIRAPIPrepBase;

  void runOnOperation() override {
    ModuleOp module = getOperation();

    {
      auto *ctx = &getContext();
      RewritePatternSet patterns(ctx);
      QIRAPITypeConverter typeConverter(opaquePtr);
      cudaq::opt::populateQuakeToCCPrepPatterns(patterns);
      if (failed(applyPatternsAndFoldGreedily(module, std::move(patterns)))) {
        signalPassFailure();
        return;
      }
    }

    auto irBuilder = cudaq::IRBuilder::atBlockEnd(module.getBody());

    // Get the type aliases to use to dynamically configure the prototypes.
    StringRef typeAliases;
    if (opaquePtr) {
      LLVM_DEBUG(llvm::dbgs() << "Using opaque pointers\n");
      typeAliases = irBuilder.getIntrinsicText("qir_opaque_pointer");
    } else {
      LLVM_DEBUG(llvm::dbgs() << "Using pointers to opaque structs\n");
      typeAliases = irBuilder.getIntrinsicText("qir_opaque_struct");
    }

    bool usingFullQIR = api == "full";
    if (usingFullQIR) {
      if (failed(irBuilder.loadIntrinsicWithAliases(module, "qir_full",
                                                    typeAliases))) {
        module.emitError("could not load full QIR declarations.");
        signalPassFailure();
        return;
      }

    } else {
      if (failed(irBuilder.loadIntrinsicWithAliases(
              module, "qir_common_profile", typeAliases))) {
        module.emitError("could not load QIR profile declarations.");
        signalPassFailure();
        return;
      }
    }

    if (usingFullQIR) {
      OpBuilder builder(module);
      module.walk([&](func::FuncOp func) {
        int counter = 0;
        func.walk([&](quake::MzOp mz) {
          guaranteeMzIsLabeled(mz, counter, builder);
        });
      });
    } else {
      // If the API is one of the profile variants, we must perform allocation
      // and measurement analysis and stick attributes on the Ops as needed.
      OpBuilder builder(module);
      module.walk([&](func::FuncOp func) {
        std::size_t totalQubits = 0;
        std::size_t totalResults = 0;
        int counter = 0;

        // A map to keep track of wire set usage.
        DenseMap<StringRef, DenseSet<std::size_t>> borrowSets;

        // Recursive walk in func.
        func.walk([&](Operation *op) {
          // Annotate all qubit allocations with the starting qubit index value.
          // This ought to handle both reference and value semantics. If the
          // value semantics is using wire sets, no (redundant) annotation is
          // needed.
          if (auto alloc = dyn_cast<quake::AllocaOp>(op)) {
            auto allocTy = alloc.getType();
            if (isa<quake::RefType>(allocTy)) {
              alloc->setAttr(cudaq::opt::StartingOffsetAttrName,
                             builder.getI64IntegerAttr(totalQubits++));
              return;
            }
            if (!isa<quake::VeqType>(allocTy)) {
              alloc.emitOpError("must be veq type.");
              return;
            }
            auto veqTy = cast<quake::VeqType>(allocTy);
            if (!veqTy.hasSpecifiedSize()) {
              alloc.emitOpError("must have a constant size.");
              return;
            }
            alloc->setAttr(cudaq::opt::StartingOffsetAttrName,
                           builder.getI64IntegerAttr(totalQubits));
            totalQubits += veqTy.getSize();
            return;
          }
          if (auto nw = dyn_cast<quake::NullWireOp>(op)) {
            nw->setAttr(cudaq::opt::StartingOffsetAttrName,
                        builder.getI64IntegerAttr(totalQubits++));
            return;
          }
          if (auto bw = dyn_cast<quake::BorrowWireOp>(op)) {
            [[maybe_unused]] StringRef name = bw.getSetName();
            [[maybe_unused]] std::int32_t wire = bw.getIdentity();
            bw.emitOpError("not implemented.");
            return;
          }

          // For each mz, we want to assign it a result index.
          if (auto mz = dyn_cast<quake::MzOp>(op)) {
            // Verify there is exactly one qubit being measured.
            if (mz.getTargets().empty() ||
                std::distance(mz.getTargets().begin(), mz.getTargets().end()) !=
                    1) {
              mz.emitOpError("must measure exactly one qubit.");
              return;
            }
            mz->setAttr(cudaq::opt::ResultIndexAttrName,
                        builder.getI64IntegerAttr(totalResults++));
            guaranteeMzIsLabeled(mz, counter, builder);
          }
        });

        // If the API is one of the profile variants, the QIR consumer expects
        // some bonus information by way of attributes. Add most of them here.
        // (See also OUTPUT-NAME-MAP.)
        SmallVector<Attribute> funcAttrs;
        if (func->hasAttr(cudaq::kernelAttrName)) {
          if (func->getAttr(cudaq::entryPointAttrName))
            funcAttrs.push_back(
                builder.getStringAttr(cudaq::opt::QIREntryPointAttrName));
          if (api == "base-profile") {
            funcAttrs.push_back(builder.getStrArrayAttr(
                {cudaq::opt::QIRProfilesAttrName, "base_profile"}));
            funcAttrs.push_back(builder.getStrArrayAttr(
                {cudaq::opt::QIROutputLabelingSchemaAttrName, "schema_id"}));
          } else if (api == "adaptive-profile") {
            funcAttrs.push_back(builder.getStrArrayAttr(
                {cudaq::opt::QIRProfilesAttrName, "adaptive_profile"}));
          }
          if (totalQubits)
            funcAttrs.push_back(builder.getStrArrayAttr(
                {cudaq::opt::QIRRequiredQubitsAttrName,
                 builder.getStringAttr(std::to_string(totalQubits))}));
          if (totalResults)
            funcAttrs.push_back(builder.getStrArrayAttr(
                {cudaq::opt::QIRRequiredResultsAttrName,
                 builder.getStringAttr(std::to_string(totalResults))}));
        }
        if (!funcAttrs.empty())
          func->setAttr("passthrough", builder.getArrayAttr(funcAttrs));
      });
    }
  }

  void guaranteeMzIsLabeled(quake::MzOp mz, int &counter, OpBuilder &builder) {
    if (mz.getRegisterNameAttr() &&
        /* FIXME: issue 2538: the name should never be empty. */
        !mz.getRegisterNameAttr().getValue().empty()) {
      mz->setAttr(cudaq::opt::MzAssignedNameAttrName, builder.getUnitAttr());
      return;
    }
    // Manufacture a bogus name on demand here.
    std::string manuName = std::to_string(counter++);
    constexpr std::size_t padSize = 5;
    manuName =
        std::string(padSize - std::min(padSize, manuName.length()), '0') +
        manuName;
    mz.setRegisterName("r" + manuName);
  }
};

struct QuakeToQIRAPIFinalPass
    : public cudaq::opt::impl::QuakeToQIRAPIFinalBase<QuakeToQIRAPIFinalPass> {
  using QuakeToQIRAPIFinalBase::QuakeToQIRAPIFinalBase;

  void runOnOperation() override {
    auto *ctx = &getContext();
    ModuleOp module = getOperation();
    RewritePatternSet patterns(ctx);
    patterns.insert<MaterializeConstantArrayOpRewrite>(ctx);
    if (api == "base-profile")
      patterns.insert<AnnotateKernelsWithMeasurementStringsPattern>(ctx);
    if (failed(applyPatternsAndFoldGreedily(module, std::move(patterns))))
      signalPassFailure();
  }
};

} // namespace

void cudaq::opt::addConvertToQIRAPIPipeline(OpPassManager &pm, StringRef api,
                                            bool opaquePtr) {
  QuakeToQIRAPIPrepOptions prepApiOpt{.api = api.str(), .opaquePtr = opaquePtr};
  pm.addPass(cudaq::opt::createQuakeToQIRAPIPrep(prepApiOpt));
  pm.addPass(cudaq::opt::createLowerToCG());
  QuakeToQIRAPIOptions apiOpt{.api = api.str(), .opaquePtr = opaquePtr};
  pm.addPass(cudaq::opt::createQuakeToQIRAPI(apiOpt));
  pm.addPass(createCanonicalizerPass());
  QuakeToQIRAPIFinalOptions finalApiOpt{.api = api.str()};
  pm.addPass(cudaq::opt::createQuakeToQIRAPIFinal(finalApiOpt));
  pm.addPass(cudaq::opt::createGlobalizeArrayValues());
}

namespace {
struct QIRAPIPipelineOptions
    : public PassPipelineOptions<QIRAPIPipelineOptions> {
  PassOptions::Option<std::string> api{
      *this, "api",
      llvm::cl::desc("select the profile to convert to [full, base-profile, "
                     "adaptive-profile]"),
      llvm::cl::init("full")};
  PassOptions::Option<bool> opaquePtr{*this, "opaque-pointer",
                                      llvm::cl::desc("use opaque pointers"),
                                      llvm::cl::init(false)};
};
} // namespace

void cudaq::opt::registerToQIRAPIPipeline() {
  PassPipelineRegistration<QIRAPIPipelineOptions>(
      "convert-to-qir-api", "Convert quake to one of the QIR APIs.",
      [](OpPassManager &pm, const QIRAPIPipelineOptions &opt) {
        addConvertToQIRAPIPipeline(pm, opt.api, opt.opaquePtr);
      });
}