assembler_riscv.hpp

/*
 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */

#ifndef CPU_RISCV_ASSEMBLER_RISCV_HPP
#define CPU_RISCV_ASSEMBLER_RISCV_HPP

#include "asm/assembler.hpp"
#include "asm/register.hpp"
#include "code/codeCache.hpp"
#include "cppstdlib/type_traits.hpp"
#include "metaprogramming/enableIf.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"

#define XLEN 64

// definitions of various symbolic names for machine registers

// First intercalls between C and Java which use 8 general registers
// and 8 floating registers

class Argument {
 public:
  enum {
    // check more info at https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc
    n_int_register_parameters_c   = 8,   // x10, x11, ... x17 (c_rarg0, c_rarg1, ...)
    n_float_register_parameters_c = 8,   // f10, f11, ... f17 (c_farg0, c_farg1, ... )
    n_vector_register_parameters_c = 16,  // v8, v9, ... v23

    n_int_register_parameters_j   = 8, // x11, ... x17, x10 (j_rarg0, j_rarg1, ...)
    n_float_register_parameters_j = 8  // f10, f11, ... f17 (j_farg0, j_farg1, ...)
  };
};

// function argument(caller-save registers)
constexpr Register c_rarg0 = x10;
constexpr Register c_rarg1 = x11;
constexpr Register c_rarg2 = x12;
constexpr Register c_rarg3 = x13;
constexpr Register c_rarg4 = x14;
constexpr Register c_rarg5 = x15;
constexpr Register c_rarg6 = x16;
constexpr Register c_rarg7 = x17;

constexpr FloatRegister c_farg0 = f10;
constexpr FloatRegister c_farg1 = f11;
constexpr FloatRegister c_farg2 = f12;
constexpr FloatRegister c_farg3 = f13;
constexpr FloatRegister c_farg4 = f14;
constexpr FloatRegister c_farg5 = f15;
constexpr FloatRegister c_farg6 = f16;
constexpr FloatRegister c_farg7 = f17;

// Symbolically name the register arguments used by the Java calling convention.
// We have control over the convention for java so we can do what we please.
// What pleases us is to offset the java calling convention so that when
// we call a suitable jni method the arguments are lined up and we don't
// have to do much shuffling. A suitable jni method is non-static and a
// small number of arguments.
//
// |------------------------------------------------------------------------|
// | c_rarg0  c_rarg1  c_rarg2  c_rarg3  c_rarg4  c_rarg5  c_rarg6  c_rarg7 |
// |------------------------------------------------------------------------|
// | x10      x11      x12      x13      x14      x15      x16      x17     |
// |------------------------------------------------------------------------|
// | j_rarg7  j_rarg0  j_rarg1  j_rarg2  j_rarg3  j_rarg4  j_rarg5  j_rarg6 |
// |------------------------------------------------------------------------|

constexpr Register j_rarg0 = c_rarg1;
constexpr Register j_rarg1 = c_rarg2;
constexpr Register j_rarg2 = c_rarg3;
constexpr Register j_rarg3 = c_rarg4;
constexpr Register j_rarg4 = c_rarg5;
constexpr Register j_rarg5 = c_rarg6;
constexpr Register j_rarg6 = c_rarg7;
constexpr Register j_rarg7 = c_rarg0;

// Java floating args are passed as per C

constexpr FloatRegister j_farg0 = f10;
constexpr FloatRegister j_farg1 = f11;
constexpr FloatRegister j_farg2 = f12;
constexpr FloatRegister j_farg3 = f13;
constexpr FloatRegister j_farg4 = f14;
constexpr FloatRegister j_farg5 = f15;
constexpr FloatRegister j_farg6 = f16;
constexpr FloatRegister j_farg7 = f17;

// zero rigster
constexpr Register zr = x0;
// global pointer
constexpr Register gp = x3;
// thread pointer
constexpr Register tp = x4;

// registers used to hold VM data either temporarily within a method
// or across method calls

// volatile (caller-save) registers

// current method -- must be in a call-clobbered register
constexpr Register xmethod =  x31;
// return address
constexpr Register ra      =  x1;

// non-volatile (callee-save) registers

constexpr Register sp            = x2; // stack pointer
constexpr Register fp            = x8; // frame pointer
constexpr Register xheapbase     = x27; // base of heap
constexpr Register xcpool        = x26; // constant pool cache
constexpr Register xmonitors     = x25; // monitors allocated on stack
constexpr Register xlocals       = x24; // locals on stack
constexpr Register xthread       = x23; // java thread pointer
constexpr Register xbcp          = x22; // bytecode pointer
constexpr Register xdispatch     = x21; // Dispatch table base
constexpr Register esp           = x20; // Java expression stack pointer
constexpr Register x19_sender_sp = x19; // Sender's SP while in interpreter

// temporary register(caller-save registers)
constexpr Register t0 = x5;
constexpr Register t1 = x6;
constexpr Register t2 = x7;
constexpr Register t3 = x28;
constexpr Register t4 = x29;
constexpr Register t5 = x30;
constexpr Register t6 = x31;

const Register g_INTArgReg[Argument::n_int_register_parameters_c] = {
  c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7
};

const FloatRegister g_FPArgReg[Argument::n_float_register_parameters_c] = {
  c_farg0, c_farg1, c_farg2, c_farg3, c_farg4, c_farg5, c_farg6, c_farg7
};

#define assert_cond(ARG1) assert(ARG1, #ARG1)

// Addressing modes
class Address {
 public:

  enum mode { no_mode, base_plus_offset, literal };

 private:
  struct Nonliteral {
    Nonliteral(Register base, Register index, int64_t offset)
      : _base(base), _index(index), _offset(offset) {}
    Register _base;
    Register _index;
    int64_t _offset;
  };

  struct Literal {
    Literal(address target, const RelocationHolder& rspec)
      : _target(target), _rspec(rspec) {}
    // If the target is far we'll need to load the ea of this to a
    // register to reach it. Otherwise if near we can do PC-relative
    // addressing.
    address _target;

    RelocationHolder _rspec;
  };

  void assert_is_nonliteral() const NOT_DEBUG_RETURN;
  void assert_is_literal() const NOT_DEBUG_RETURN;

  // Discriminated union, based on _mode.
  // - no_mode: uses dummy _nonliteral, for ease of copying.
  // - literal: only _literal is used.
  // - others: only _nonliteral is used.
  enum mode _mode;
  union {
    Nonliteral _nonliteral;
    Literal _literal;
  };

  // Helper for copy constructor and assignment operator.
  // Copy mode-relevant part of a into this.
  void copy_data(const Address& a) {
    assert(_mode == a._mode, "precondition");
    if (_mode == literal) {
      new (&_literal) Literal(a._literal);
    } else {
      // non-literal mode or no_mode.
      new (&_nonliteral) Nonliteral(a._nonliteral);
    }
  }

 public:
  // no_mode initializes _nonliteral for ease of copying.
  Address() :
    _mode(no_mode),
    _nonliteral(noreg, noreg, 0)
  {}

  Address(Register r) :
    _mode(base_plus_offset),
    _nonliteral(r, noreg, 0)
  {}

  template<typename T, ENABLE_IF(std::is_integral<T>::value)>
  Address(Register r, T o) :
    _mode(base_plus_offset),
    _nonliteral(r, noreg, o)
  {}

  Address(Register r, ByteSize disp) : Address(r, in_bytes(disp)) {}

  Address(address target, const RelocationHolder& rspec) :
    _mode(literal),
    _literal(target, rspec)
  {}

  Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type);

  Address(const Address& a) : _mode(a._mode) { copy_data(a); }

  // Verify the value is trivially destructible regardless of mode, so our
  // destructor can also be trivial, and so our assignment operator doesn't
  // need to destruct the old value before copying over it.
  static_assert(std::is_trivially_destructible<Literal>::value, "must be");
  static_assert(std::is_trivially_destructible<Nonliteral>::value, "must be");

  Address& operator=(const Address& a) {
    _mode = a._mode;
    copy_data(a);
    return *this;
  }

  ~Address() = default;

  const Register base() const {
    assert_is_nonliteral();
    return _nonliteral._base;
  }

  long offset() const {
    assert_is_nonliteral();
    return _nonliteral._offset;
  }

  Register index() const {
    assert_is_nonliteral();
    return _nonliteral._index;
  }

  mode getMode() const {
    return _mode;
  }

  bool uses(Register reg) const {
    return _mode != literal && base() == reg;
  }

  address target() const {
    assert_is_literal();
    return _literal._target;
  }

  const RelocationHolder& rspec() const {
    assert_is_literal();
    return _literal._rspec;
  }
};

// Convenience classes
class RuntimeAddress: public Address {

  public:

  RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
  ~RuntimeAddress() {}
};

class OopAddress: public Address {

  public:

  OopAddress(address target) : Address(target, relocInfo::oop_type) {}
  ~OopAddress() {}
};

class ExternalAddress: public Address {
 private:
  static relocInfo::relocType reloc_for_target(address target) {
    // Sometimes ExternalAddress is used for values which aren't
    // exactly addresses, like the card table base.
    // external_word_type can't be used for values in the first page
    // so just skip the reloc in that case.
    return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none;
  }

 public:

  ExternalAddress(address target) : Address(target, reloc_for_target(target)) {}
  ~ExternalAddress() {}
};

class InternalAddress: public Address {

  public:

  InternalAddress(address target) : Address(target, relocInfo::internal_word_type) {}
  ~InternalAddress() {}
};

class Assembler : public AbstractAssembler {
protected:

  static int zfa_zli_lookup_double(uint64_t value) {
    switch(value) {
      case 0xbff0000000000000 : return   0;
      case 0x0010000000000000 : return   1;
      case 0x3ef0000000000000 : return   2;
      case 0x3f00000000000000 : return   3;
      case 0x3f70000000000000 : return   4;
      case 0x3f80000000000000 : return   5;
      case 0x3fb0000000000000 : return   6;
      case 0x3fc0000000000000 : return   7;
      case 0x3fd0000000000000 : return   8;
      case 0x3fd4000000000000 : return   9;
      case 0x3fd8000000000000 : return  10;
      case 0x3fdc000000000000 : return  11;
      case 0x3fe0000000000000 : return  12;
      case 0x3fe4000000000000 : return  13;
      case 0x3fe8000000000000 : return  14;
      case 0x3fec000000000000 : return  15;
      case 0x3ff0000000000000 : return  16;
      case 0x3ff4000000000000 : return  17;
      case 0x3ff8000000000000 : return  18;
      case 0x3ffc000000000000 : return  19;
      case 0x4000000000000000 : return  20;
      case 0x4004000000000000 : return  21;
      case 0x4008000000000000 : return  22;
      case 0x4010000000000000 : return  23;
      case 0x4020000000000000 : return  24;
      case 0x4030000000000000 : return  25;
      case 0x4060000000000000 : return  26;
      case 0x4070000000000000 : return  27;
      case 0x40e0000000000000 : return  28;
      case 0x40f0000000000000 : return  29;
      case 0x7ff0000000000000 : return  30;
      case 0x7ff8000000000000 : return  31;
      default: break;
    }
    return -1;
  }

  static int zfa_zli_lookup_float(uint32_t value) {
    switch(value) {
      case 0xbf800000 : return  0;
      case 0x00800000 : return  1;
      case 0x37800000 : return  2;
      case 0x38000000 : return  3;
      case 0x3b800000 : return  4;
      case 0x3c000000 : return  5;
      case 0x3d800000 : return  6;
      case 0x3e000000 : return  7;
      case 0x3e800000 : return  8;
      case 0x3ea00000 : return  9;
      case 0x3ec00000 : return 10;
      case 0x3ee00000 : return 11;
      case 0x3f000000 : return 12;
      case 0x3f200000 : return 13;
      case 0x3f400000 : return 14;
      case 0x3f600000 : return 15;
      case 0x3f800000 : return 16;
      case 0x3fa00000 : return 17;
      case 0x3fc00000 : return 18;
      case 0x3fe00000 : return 19;
      case 0x40000000 : return 20;
      case 0x40200000 : return 21;
      case 0x40400000 : return 22;
      case 0x40800000 : return 23;
      case 0x41000000 : return 24;
      case 0x41800000 : return 25;
      case 0x43000000 : return 26;
      case 0x43800000 : return 27;
      case 0x47000000 : return 28;
      case 0x47800000 : return 29;
      case 0x7f800000 : return 30;
      case 0x7fc00000 : return 31;
      default: break;
    }
    return -1;
  }

  static int zfa_zli_lookup_half_float(uint16_t value) {
    switch(value) {
      case 0xbc00 : return  0;
      case 0x0400 : return  1;
      case 0x0100 : return  2;
      case 0x0200 : return  3;
      case 0x1c00 : return  4;
      case 0x2000 : return  5;
      case 0x2c00 : return  6;
      case 0x3000 : return  7;
      case 0x3400 : return  8;
      case 0x3500 : return  9;
      case 0x3600 : return 10;
      case 0x3700 : return 11;
      case 0x3800 : return 12;
      case 0x3900 : return 13;
      case 0x3a00 : return 14;
      case 0x3b00 : return 15;
      case 0x3c00 : return 16;
      case 0x3d00 : return 17;
      case 0x3e00 : return 18;
      case 0x3f00 : return 19;
      case 0x4000 : return 20;
      case 0x4100 : return 21;
      case 0x4200 : return 22;
      case 0x4400 : return 23;
      case 0x4800 : return 24;
      case 0x4c00 : return 25;
      case 0x5800 : return 26;
      case 0x5c00 : return 27;
      case 0x7800 : return 28;
      case 0x7c00 : return 29;
      // case 0x7c00 : return 30; // redundant with 29
      case 0x7e00 : return 31;
      default: break;
    }
    return -1;
  }

 public:

  static bool can_zfa_zli_half_float(jshort hf) {
    if (!UseZfa || !UseZfh) {
      return false;
    }
    uint16_t hf_bits = (uint16_t)hf;
    return zfa_zli_lookup_half_float(hf_bits) != -1;
  }

  static bool can_zfa_zli_float(jfloat f) {
    if (!UseZfa) {
      return false;
    }
    uint32_t f_bits = (uint32_t)jint_cast(f);
    return zfa_zli_lookup_float(f_bits) != -1;
  }

  static bool can_zfa_zli_double(jdouble d) {
    if (!UseZfa) {
      return false;
    }
    uint64_t d_bits = (uint64_t)julong_cast(d);
    return zfa_zli_lookup_double(d_bits) != -1;
  }

  enum {
    instruction_size = 4,
    compressed_instruction_size = 2,
  };

  // instruction must start at passed address
  static bool is_compressed_instr(address instr) {
    // The RISC-V ISA Manual, Section 'Base Instruction-Length Encoding':
    // Instructions are stored in memory as a sequence of 16-bit little-endian parcels, regardless of
    // memory system endianness. Parcels forming one instruction are stored at increasing halfword
    // addresses, with the lowest-addressed parcel holding the lowest-numbered bits in the instruction
    // specification.
    if (UseRVC && (((uint16_t *)instr)[0] & 0b11) != 0b11) {
      // 16-bit instructions have their lowest two bits equal to 0b00, 0b01, or 0b10
      return true;
    }
    // 32-bit instructions have their lowest two bits set to 0b11
    return false;
  }

  //---<  calculate length of instruction  >---
  // We just use the values set above.
  // instruction must start at passed address
  static unsigned int instr_len(address instr) {
    return is_compressed_instr(instr) ? compressed_instruction_size : instruction_size;
  }

  //---<  longest instructions  >---
  static unsigned int instr_maxlen() { return instruction_size; }

  enum RoundingMode {
    rne = 0b000,     // round to Nearest, ties to Even
    rtz = 0b001,     // round towards Zero
    rdn = 0b010,     // round Down (towards eegative infinity)
    rup = 0b011,     // round Up (towards infinity)
    rmm = 0b100,     // round to Nearest, ties to Max Magnitude
    rdy = 0b111,     // in instruction's rm field, selects dynamic rounding mode.In Rounding Mode register, Invalid.
  };

  // handle unaligned access
  static inline uint16_t ld_c_instr(address addr) {
    return Bytes::get_native_u2(addr);
  }
  static inline void sd_c_instr(address addr, uint16_t c_instr) {
    Bytes::put_native_u2(addr, c_instr);
  }

  // handle unaligned access
  static inline uint32_t ld_instr(address addr) {
    return Bytes::get_native_u4(addr);
  }
  static inline void sd_instr(address addr, uint32_t instr) {
    Bytes::put_native_u4(addr, instr);
  }

  static inline uint32_t extract(uint32_t val, unsigned msb, unsigned lsb) {
    assert_cond(msb >= lsb && msb <= 31);
    unsigned nbits = msb - lsb + 1;
    uint32_t mask = (1U << nbits) - 1;
    uint32_t result = val >> lsb;
    result &= mask;
    return result;
  }

  static inline int32_t sextract(uint32_t val, unsigned msb, unsigned lsb) {
    assert_cond(msb >= lsb && msb <= 31);
    int32_t result = val << (31 - msb);
    result >>= (31 - msb + lsb);
    return result;
  }

  static void patch(address a, unsigned msb, unsigned lsb, unsigned val) {
    assert_cond(a != nullptr);
    assert_cond(msb >= lsb && msb <= 31);
    unsigned nbits = msb - lsb + 1;
    guarantee(val < (1U << nbits), "Field too big for insn");
    unsigned mask = (1U << nbits) - 1;
    val <<= lsb;
    mask <<= lsb;
    unsigned target = ld_instr(a);
    target &= ~mask;
    target |= val;
    sd_instr(a, target);
  }

  static void patch(address a, unsigned bit, unsigned val) {
    patch(a, bit, bit, val);
  }

  static void patch_reg(address a, unsigned lsb, Register reg) {
    patch(a, lsb + 4, lsb, reg->raw_encoding());
  }

  static void patch_reg(address a, unsigned lsb, FloatRegister reg) {
    patch(a, lsb + 4, lsb, reg->raw_encoding());
  }

  static void patch_reg(address a, unsigned lsb, VectorRegister reg) {
    patch(a, lsb + 4, lsb, reg->raw_encoding());
  }

  void emit(unsigned insn) {
    emit_int32((jint)insn);
  }

  enum csr {
    cycle = 0xc00,
    time,
    instret,
    hpmcounter3,
    hpmcounter4,
    hpmcounter5,
    hpmcounter6,
    hpmcounter7,
    hpmcounter8,
    hpmcounter9,
    hpmcounter10,
    hpmcounter11,
    hpmcounter12,
    hpmcounter13,
    hpmcounter14,
    hpmcounter15,
    hpmcounter16,
    hpmcounter17,
    hpmcounter18,
    hpmcounter19,
    hpmcounter20,
    hpmcounter21,
    hpmcounter22,
    hpmcounter23,
    hpmcounter24,
    hpmcounter25,
    hpmcounter26,
    hpmcounter27,
    hpmcounter28,
    hpmcounter29,
    hpmcounter30,
    hpmcounter31 = 0xc1f
  };

  // Emit an illegal instruction that's known to trap, with 32 read-only CSR
  // to choose as the input operand.
  // According to the RISC-V Assembly Programmer's Manual, a de facto implementation
  // of this instruction is the UNIMP pseduo-instruction, 'CSRRW x0, cycle, x0',
  // attempting to write zero to a read-only CSR 'cycle' (0xC00).
  // RISC-V ISAs provide a set of up to 32 read-only CSR registers 0xC00-0xC1F,
  // and an attempt to write into any read-only CSR (whether it exists or not)
  // will generate an illegal instruction exception.
  void illegal_instruction(csr csr_reg) {
    csrrw(x0, (unsigned)csr_reg, x0);
  }

// Register Instruction
#define INSN(NAME, op, funct3, funct7)                          \
  void NAME(Register Rd, Register Rs1, Register Rs2) {          \
    unsigned insn = 0;                                          \
    patch((address)&insn, 6,  0, op);                           \
    patch((address)&insn, 14, 12, funct3);                      \
    patch((address)&insn, 31, 25, funct7);                      \
    patch_reg((address)&insn, 7, Rd);                           \
    patch_reg((address)&insn, 15, Rs1);                         \
    patch_reg((address)&insn, 20, Rs2);                         \
    emit(insn);                                                 \
  }

  INSN(_add,  0b0110011, 0b000, 0b0000000);
  INSN(_sub,  0b0110011, 0b000, 0b0100000);
  INSN(_andr, 0b0110011, 0b111, 0b0000000);
  INSN(_orr,  0b0110011, 0b110, 0b0000000);
  INSN(_xorr, 0b0110011, 0b100, 0b0000000);
  INSN(sll,   0b0110011, 0b001, 0b0000000);
  INSN(sra,   0b0110011, 0b101, 0b0100000);
  INSN(srl,   0b0110011, 0b101, 0b0000000);
  INSN(slt,   0b0110011, 0b010, 0b0000000);
  INSN(sltu,  0b0110011, 0b011, 0b0000000);
  INSN(_addw, 0b0111011, 0b000, 0b0000000);
  INSN(_subw, 0b0111011, 0b000, 0b0100000);
  INSN(sllw,  0b0111011, 0b001, 0b0000000);
  INSN(sraw,  0b0111011, 0b101, 0b0100000);
  INSN(srlw,  0b0111011, 0b101, 0b0000000);
  INSN(_mul,  0b0110011, 0b000, 0b0000001);
  INSN(mulh,  0b0110011, 0b001, 0b0000001);
  INSN(mulhsu,0b0110011, 0b010, 0b0000001);
  INSN(mulhu, 0b0110011, 0b011, 0b0000001);
  INSN(mulw,  0b0111011, 0b000, 0b0000001);
  INSN(div,   0b0110011, 0b100, 0b0000001);
  INSN(divu,  0b0110011, 0b101, 0b0000001);
  INSN(divw,  0b0111011, 0b100, 0b0000001);
  INSN(divuw, 0b0111011, 0b101, 0b0000001);
  INSN(rem,   0b0110011, 0b110, 0b0000001);
  INSN(remu,  0b0110011, 0b111, 0b0000001);
  INSN(remw,  0b0111011, 0b110, 0b0000001);
  INSN(remuw, 0b0111011, 0b111, 0b0000001);

#undef INSN

 private:
  // Load
  enum LoadWidthFunct3 : uint8_t {
    LOAD_WIDTH_BYTE              = 0b000,
    LOAD_WIDTH_HALFWORD          = 0b001,
    LOAD_WIDTH_WORD              = 0b010,
    LOAD_WIDTH_DOUBLEWORD        = 0b011,
    LOAD_WIDTH_BYTE_UNSIGNED     = 0b100,
    LOAD_WIDTH_HALFWORD_UNSIGNED = 0b101,
    LOAD_WIDTH_WORD_UNSIGNED     = 0b110,
    // 0b111 is reserved
  };

  static constexpr uint8_t OP_LOAD_MAJOR    = 0b0000011;
  static constexpr uint8_t OP_FP_LOAD_MAJOR = 0b0000111;

  template <uint8_t op_major, LoadWidthFunct3 width>
  void load_base(uint8_t Rd, Register Rs, const int32_t offset) {
    guarantee(is_simm12(offset), "offset is invalid.");
    unsigned insn = 0;
    int32_t val = offset & 0xfff;
    patch((address)&insn,  6,  0, op_major);
    patch((address)&insn, 11,  7, Rd);
    patch((address)&insn, 14, 12, width);
    patch_reg((address)&insn, 15, Rs);
    patch((address)&insn, 31, 20, val);
    emit(insn);
  }

  template <LoadWidthFunct3 width>
  void load_base(Register Rd, Register Rs, const int32_t offset) {
    load_base<OP_LOAD_MAJOR, width>(Rd->raw_encoding(), Rs, offset);
  }

  template <LoadWidthFunct3 width>
  void load_base(FloatRegister Rd, Register Rs, const int32_t offset) {
    load_base<OP_FP_LOAD_MAJOR, width>(Rd->raw_encoding(), Rs, offset);
  }

 public:

  void lb(Register Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_BYTE>(Rd, Rs, offset);
  }

  void _lbu(Register Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_BYTE_UNSIGNED>(Rd, Rs, offset);
  }

  void _lh(Register Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_HALFWORD>(Rd, Rs, offset);
  }

  void _lhu(Register Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_HALFWORD_UNSIGNED>(Rd, Rs, offset);
  }

  void _lw(Register Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_WORD>(Rd, Rs, offset);
  }

  void lwu(Register Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_WORD_UNSIGNED>(Rd, Rs, offset);
  }

  void _ld(Register Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_DOUBLEWORD>(Rd, Rs, offset);
  }

  void flh(FloatRegister Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_HALFWORD>(Rd, Rs, offset);
  }

  void flw(FloatRegister Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_WORD>(Rd, Rs, offset);
  }

  void _fld(FloatRegister Rd, Register Rs, const int32_t offset) {
    load_base<LOAD_WIDTH_DOUBLEWORD>(Rd, Rs, offset);
  }

#define INSN(NAME, op, funct3)                                                                           \
  void NAME(Register Rs1, Register Rs2, const int64_t offset) {                                          \
    guarantee(is_simm13(offset) && ((offset % 2) == 0), "offset is invalid.");                           \
    unsigned insn = 0;                                                                                   \
    uint32_t val  = offset & 0x1fff;                                                                     \
    uint32_t val11 = (val >> 11) & 0x1;                                                                  \
    uint32_t val12 = (val >> 12) & 0x1;                                                                  \
    uint32_t low  = (val >> 1) & 0xf;                                                                    \
    uint32_t high = (val >> 5) & 0x3f;                                                                   \
    patch((address)&insn, 6, 0, op);                                                                     \
    patch((address)&insn, 14, 12, funct3);                                                               \
    patch_reg((address)&insn, 15, Rs1);                                                                  \
    patch_reg((address)&insn, 20, Rs2);                                                                  \
    patch((address)&insn, 7, val11);                                                                     \
    patch((address)&insn, 11, 8, low);                                                                   \
    patch((address)&insn, 30, 25, high);                                                                 \
    patch((address)&insn, 31, val12);                                                                    \
    emit(insn);                                                                                          \
  }

  INSN(beq,  0b1100011, 0b000);
  INSN(bne,  0b1100011, 0b001);
  INSN(bge,  0b1100011, 0b101);
  INSN(bgeu, 0b1100011, 0b111);
  INSN(blt,  0b1100011, 0b100);
  INSN(bltu, 0b1100011, 0b110);

#undef INSN

 private:

  enum StoreWidthFunct3 : uint8_t {
    STORE_WIDTH_BYTE        = 0b000,
    STORE_WIDTH_HALFWORD    = 0b001,
    STORE_WIDTH_WORD        = 0b010,
    STORE_WIDTH_DOUBLEWORD  = 0b011,
    // 0b100 to 0b111 are reserved for this opcode
  };

  static constexpr uint8_t OP_STORE_MAJOR    = 0b0100011;
  static constexpr uint8_t OP_FP_STORE_MAJOR = 0b0100111;

  template <uint8_t op_code, StoreWidthFunct3 width>
  void store_base(uint8_t Rs2, Register Rs1, const int32_t offset) {
    guarantee(is_simm12(offset), "offset is invalid.");
    unsigned insn = 0;
    uint32_t val  = offset & 0xfff;
    uint32_t low  = val & 0x1f;
    uint32_t high = (val >> 5) & 0x7f;
    patch((address)&insn,  6,  0, op_code);
    patch((address)&insn, 11,  7, low);
    patch((address)&insn, 14, 12, width);
    patch_reg((address)&insn, 15, Rs1);
    patch((address)&insn, 24, 20, Rs2);
    patch((address)&insn, 31, 25, high);
    emit(insn);
  }

  template <StoreWidthFunct3 width>
  void store_base(Register Rs2, Register Rs1, const int32_t offset) {
    store_base<OP_STORE_MAJOR, width>(Rs2->raw_encoding(), Rs1, offset);
  }

  template <StoreWidthFunct3 width>
  void store_base(FloatRegister Rs2, Register Rs1, const int32_t offset) {
    store_base<OP_FP_STORE_MAJOR, width>(Rs2->raw_encoding(), Rs1, offset);
  }

 public:

  void _sb(Register Rs2, Register Rs1, const int32_t offset) {
    store_base<STORE_WIDTH_BYTE>(Rs2, Rs1, offset);
  }

  void _sh(Register Rs2, Register Rs1, const int32_t offset) {
    store_base<STORE_WIDTH_HALFWORD>(Rs2, Rs1, offset);
  }

  void _sw(Register Rs2, Register Rs1, const int32_t offset) {
    store_base<STORE_WIDTH_WORD>(Rs2, Rs1, offset);
  }

  void _sd(Register Rs2, Register Rs1, const int32_t offset) {
    store_base<STORE_WIDTH_DOUBLEWORD>(Rs2, Rs1, offset);
  }

  void fsw(FloatRegister Rs2, Register Rs1, const int32_t offset) {
    store_base<STORE_WIDTH_WORD>(Rs2, Rs1, offset);
  }

  void _fsd(FloatRegister Rs2, Register Rs1, const int32_t offset) {
    store_base<STORE_WIDTH_DOUBLEWORD>(Rs2, Rs1, offset);
  }

#define INSN(NAME, op, funct3)                                                        \
  void NAME(Register Rd, const uint32_t csr, Register Rs1) {                          \
    guarantee(is_uimm12(csr), "csr is invalid");                                      \
    unsigned insn = 0;                                                                \
    patch((address)&insn, 6, 0, op);                                                  \
    patch((address)&insn, 14, 12, funct3);                                            \
    patch_reg((address)&insn, 7, Rd);                                                 \
    patch_reg((address)&insn, 15, Rs1);                                               \
    patch((address)&insn, 31, 20, csr);                                               \
    emit(insn);                                                                       \
  }

  INSN(csrrw, 0b1110011, 0b001);
  INSN(csrrs, 0b1110011, 0b010);
  INSN(csrrc, 0b1110011, 0b011);

#undef INSN

#define INSN(NAME, op, funct3)                                                        \
  void NAME(Register Rd, const uint32_t csr, const uint32_t uimm) {                   \
    guarantee(is_uimm12(csr), "csr is invalid");                                      \
    guarantee(is_uimm5(uimm), "uimm is invalid");                                     \
    unsigned insn = 0;                                                                \
    uint32_t val  = uimm & 0x1f;                                                      \
    patch((address)&insn, 6, 0, op);                                                  \
    patch((address)&insn, 14, 12, funct3);                                            \
    patch_reg((address)&insn, 7, Rd);                                                 \
    patch((address)&insn, 19, 15, val);                                               \
    patch((address)&insn, 31, 20, csr);                                               \
    emit(insn);                                                                       \
  }

  INSN(csrrwi, 0b1110011, 0b101);
  INSN(csrrsi, 0b1110011, 0b110);
  INSN(csrrci, 0b1110011, 0b111);

#undef INSN

 private:
  // All calls and jumps must go via MASM.
  // Format J-type
  void _jal(Register Rd, const int32_t offset) {
    guarantee(is_simm21(offset) && ((offset % 2) == 0), "offset is invalid.");
    unsigned insn = 0;
    patch((address)&insn, 6, 0, 0b1101111);
    patch_reg((address)&insn, 7, Rd);
    patch((address)&insn, 19, 12, (uint32_t)((offset >> 12) & 0xff));
    patch((address)&insn, 20, (uint32_t)((offset >> 11) & 0x1));
    patch((address)&insn, 30, 21, (uint32_t)((offset >> 1) & 0x3ff));
    patch((address)&insn, 31, (uint32_t)((offset >> 20) & 0x1));
    emit(insn);
  }

  // Format I-type
  void _jalr(Register Rd, Register Rs, const int32_t offset) {
    guarantee(is_simm12(offset), "offset is invalid.");
    unsigned insn = 0;
    patch((address)&insn, 6, 0, 0b1100111);
    patch_reg((address)&insn, 7, Rd);
    patch((address)&insn, 14, 12, 0b000);
    patch_reg((address)&insn, 15, Rs);
    int32_t val = offset & 0xfff;
    patch((address)&insn, 31, 20, val);
    emit(insn);
  }

 public:

  static uint32_t encode_csrrw(Register Rd, const uint32_t csr, Register Rs1) {
    guarantee(is_uimm12(csr), "csr is invalid");
    uint32_t insn = 0;
    patch((address)&insn, 6, 0, 0b1110011);
    patch((address)&insn, 14, 12, 0b001);
    patch_reg((address)&insn, 7, Rd);
    patch_reg((address)&insn, 15, Rs1);
    patch((address)&insn, 31, 20, csr);
    return insn;
  }

  static uint32_t encode_jal(Register Rd, const int32_t offset) {
    guarantee(is_simm21(offset) && ((offset % 2) == 0), "offset is invalid.");
    uint32_t insn = 0;
    patch((address)&insn, 6, 0, 0b1101111);
    patch_reg((address)&insn, 7, Rd);
    patch((address)&insn, 19, 12, (uint32_t)((offset >> 12) & 0xff));
    patch((address)&insn, 20, (uint32_t)((offset >> 11) & 0x1));
    patch((address)&insn, 30, 21, (uint32_t)((offset >> 1) & 0x3ff));
    patch((address)&insn, 31, (uint32_t)((offset >> 20) & 0x1));
    return insn;
  }

  static uint32_t encode_jalr(Register Rd, Register Rs, const int32_t offset) {
    guarantee(is_simm12(offset), "offset is invalid.");
    uint32_t insn = 0;
    patch((address)&insn, 6, 0, 0b1100111);
    patch_reg((address)&insn, 7, Rd);
    patch((address)&insn, 14, 12, 0b000);
    patch_reg((address)&insn, 15, Rs);
    int32_t val = offset & 0xfff;
    patch((address)&insn, 31, 20, val);
    return insn;
  }

 protected:

  enum barrier {
    i = 0b1000, o = 0b0100, r = 0b0010, w = 0b0001,
    ir = i | r, ow = o | w, iorw = i | o | r | w
  };

  void fence(const uint32_t predecessor, const uint32_t successor) {
    unsigned insn = 0;
    guarantee(predecessor < 16, "predecessor is invalid");
    guarantee(successor < 16, "successor is invalid");
    patch((address)&insn, 6, 0, 0b001111);      // opcode
    patch((address)&insn, 11, 7, 0b00000);      // rd
    patch((address)&insn, 14, 12, 0b000);
    patch((address)&insn, 19, 15, 0b00000);     // rs1
    patch((address)&insn, 23, 20, successor);   // succ
    patch((address)&insn, 27, 24, predecessor); // pred
    patch((address)&insn, 31, 28, 0b0000);      // fm
    emit(insn);
  }

  void fencei() {
    unsigned insn = 0;
    patch((address)&insn,  6,  0, 0b0001111);      // opcode
    patch((address)&insn, 11,  7, 0b00000);        // rd
    patch((address)&insn, 14, 12, 0b001);          // func
    patch((address)&insn, 19, 15, 0b00000);        // rs1
    patch((address)&insn, 31, 20, 0b000000000000); // fm
    emit(insn);
  }

 public:

#define INSN(NAME, op, funct3, funct7)                      \
  void NAME() {                                             \
    unsigned insn = 0;                                      \
    patch((address)&insn, 6, 0, op);                        \
    patch((address)&insn, 11, 7, 0b00000);                  \
    patch((address)&insn, 14, 12, funct3);                  \
    patch((address)&insn, 19, 15, 0b00000);                 \
    patch((address)&insn, 31, 20, funct7);                  \
    emit(insn);                                             \
  }

  INSN(ecall,   0b1110011, 0b000, 0b000000000000);
  INSN(_ebreak, 0b1110011, 0b000, 0b000000000001);

#undef INSN

  enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};

 private:

  enum AmoWidthFunct3 : uint8_t {
    AMO_WIDTH_BYTE        = 0b000, // Zabha extension
    AMO_WIDTH_HALFWORD    = 0b001, // Zabha extension
    AMO_WIDTH_WORD        = 0b010,
    AMO_WIDTH_DOUBLEWORD  = 0b011,
    AMO_WIDTH_QUADWORD    = 0b100,
    // 0b101 to 0b111 are reserved
  };

  enum AmoOperationFunct5 : uint8_t {
    AMO_ADD  = 0b00000,
    AMO_SWAP = 0b00001,
    AMO_LR   = 0b00010,
    AMO_SC   = 0b00011,
    AMO_XOR  = 0b00100,
    AMO_OR   = 0b01000,
    AMO_AND  = 0b01100,
    AMO_MIN  = 0b10000,
    AMO_MAX  = 0b10100,
    AMO_MINU = 0b11000,
    AMO_MAXU = 0b11100,
    AMO_CAS  = 0b00101 // Zacas
  };

  static constexpr uint32_t OP_AMO_MAJOR = 0b0101111;

  template <AmoOperationFunct5 funct5, AmoWidthFunct3 width>
  void amo_base(Register Rd, Register Rs1, uint8_t Rs2, Aqrl memory_order = aqrl) {
    assert(width > AMO_WIDTH_HALFWORD || UseZabha, "Must be");
    assert(funct5 != AMO_CAS || UseZacas, "Must be");
    unsigned insn = 0;
    patch((address)&insn,  6,  0, OP_AMO_MAJOR);
    patch_reg((address)&insn,  7, Rd);
    patch((address)&insn, 14, 12, width);
    patch_reg((address)&insn, 15, Rs1);
    patch((address)&insn, 24, 20, Rs2);
    patch((address)&insn, 26, 25, memory_order);
    patch((address)&insn, 31, 27, funct5);
    emit(insn);
  }

  template <AmoOperationFunct5 funct5, AmoWidthFunct3 width>
  void amo_base(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<funct5, width>(Rd, Rs1, Rs2->raw_encoding(), memory_order);
  }

 public:

  void amoadd_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_ADD, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amoadd_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_ADD, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoadd_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_ADD, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoadd_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_ADD, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoswap_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_SWAP, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amoswap_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_SWAP, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoswap_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_SWAP, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoswap_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_SWAP, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoxor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_XOR, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amoxor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_XOR, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoxor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_XOR, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoxor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_XOR, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_OR, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amoor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_OR, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_OR, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_OR, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoand_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_AND, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amoand_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_AND, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoand_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_AND, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amoand_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_AND, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomin_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MIN, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amomin_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MIN, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomin_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MIN, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomin_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MIN, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amominu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MINU, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amominu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MINU, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amominu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MINU, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amominu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MINU, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomax_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAX, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amomax_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAX, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomax_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAX, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomax_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAX, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomaxu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAXU, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amomaxu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAXU, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomaxu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAXU, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amomaxu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_MAXU, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

 protected:

  void lr_w(Register Rd, Register Rs1, Aqrl memory_order = aqrl) {
    amo_base<AMO_LR, AMO_WIDTH_WORD>(Rd, Rs1, 0, memory_order);
  }

  void lr_d(Register Rd, Register Rs1, Aqrl memory_order = aqrl) {
    amo_base<AMO_LR, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, 0, memory_order);
  }

  void sc_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_SC, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void sc_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_SC, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amocas_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_CAS, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
  }

  void amocas_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_CAS, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amocas_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_CAS, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
  }

  void amocas_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
    amo_base<AMO_CAS, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
  }

 public:

  enum operand_size { int8, int16, int32, uint32, int64 };

// Immediate Instruction
#define INSN(NAME, op, funct3)                                                              \
  void NAME(Register Rd, Register Rs1, int64_t imm) {                                       \
    guarantee(is_simm12(imm), "Immediate is out of validity");                              \
    unsigned insn = 0;                                                                      \
    patch((address)&insn, 6, 0, op);                                                        \
    patch((address)&insn, 14, 12, funct3);                                                  \
    patch((address)&insn, 31, 20, imm & 0x00000fff);                                        \
    patch_reg((address)&insn, 7, Rd);                                                       \
    patch_reg((address)&insn, 15, Rs1);                                                     \
    emit(insn);                                                                             \
  }

  INSN(_addi,  0b0010011, 0b000);
  INSN(_addiw, 0b0011011, 0b000);
  INSN(_andi,  0b0010011, 0b111);
  INSN(ori,    0b0010011, 0b110);
  INSN(xori,   0b0010011, 0b100);
  INSN(slti,   0b0010011, 0b010);

#undef INSN

#define INSN(NAME, op, funct3)                                                              \
  void NAME(Register Rd, Register Rs1, uint64_t imm) {                                      \
    guarantee(is_uimm12(imm), "Immediate is out of validity");                              \
    unsigned insn = 0;                                                                      \
    patch((address)&insn,6, 0,  op);                                                        \
    patch((address)&insn, 14, 12, funct3);                                                  \
    patch((address)&insn, 31, 20, imm & 0x00000fff);                                        \
    patch_reg((address)&insn, 7, Rd);                                                       \
    patch_reg((address)&insn, 15, Rs1);                                                     \
    emit(insn);                                                                             \
  }

  INSN(sltiu, 0b0010011, 0b011);

#undef INSN

// Shift Immediate Instruction
#define INSN(NAME, op, funct3, funct6)                                   \
  void NAME(Register Rd, Register Rs1, unsigned shamt) {                 \
    guarantee(shamt <= 0x3f, "Shamt is invalid");                        \
    unsigned insn = 0;                                                   \
    patch((address)&insn, 6, 0, op);                                     \
    patch((address)&insn, 14, 12, funct3);                               \
    patch((address)&insn, 25, 20, shamt);                                \
    patch((address)&insn, 31, 26, funct6);                               \
    patch_reg((address)&insn, 7, Rd);                                    \
    patch_reg((address)&insn, 15, Rs1);                                  \
    emit(insn);                                                          \
  }

  INSN(_slli, 0b0010011, 0b001, 0b000000);
  INSN(_srai, 0b0010011, 0b101, 0b010000);
  INSN(_srli, 0b0010011, 0b101, 0b000000);

#undef INSN

// Shift Word Immediate Instruction
#define INSN(NAME, op, funct3, funct7)                                  \
  void NAME(Register Rd, Register Rs1, unsigned shamt) {                \
    guarantee(shamt <= 0x1f, "Shamt is invalid");                       \
    unsigned insn = 0;                                                  \
    patch((address)&insn, 6, 0, op);                                    \
    patch((address)&insn, 14, 12, funct3);                              \
    patch((address)&insn, 24, 20, shamt);                               \
    patch((address)&insn, 31, 25, funct7);                              \
    patch_reg((address)&insn, 7, Rd);                                   \
    patch_reg((address)&insn, 15, Rs1);                                 \
    emit(insn);                                                         \
  }

  INSN(slliw, 0b0011011, 0b001, 0b0000000);
  INSN(sraiw, 0b0011011, 0b101, 0b0100000);
  INSN(srliw, 0b0011011, 0b101, 0b0000000);

#undef INSN

// Upper Immediate Instruction
#define INSN(NAME, op)                                                  \
  void NAME(Register Rd, int32_t imm) {                                 \
    int32_t upperImm = imm >> 12;                                       \
    unsigned insn = 0;                                                  \
    patch((address)&insn, 6, 0, op);                                    \
    patch_reg((address)&insn, 7, Rd);                                   \
    upperImm &= 0x000fffff;                                             \
    patch((address)&insn, 31, 12, upperImm);                            \
    emit(insn);                                                         \
  }

  INSN(_lui,  0b0110111);
  INSN(auipc, 0b0010111);

#undef INSN

// ==========================
// Floating Point Instructions
// ==========================
  static constexpr uint32_t OP_FP_MAJOR = 0b1010011;

  enum FmtPrecision : uint8_t {
    S_32_sp  = 0b00,
    D_64_dp  = 0b01,
    H_16_hp  = 0b10,
    Q_128_qp = 0b11
  };

 private:

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(uint8_t Rd, uint8_t Rs1, uint8_t Rs2, RoundingMode rm) {
    assert(Fmt != H_16_hp || UseZfh || UseZfhmin, "No half precision enabled");
    assert_cond(Fmt != Q_128_qp);
    guarantee(is_uimm3(rm), "Rounding mode is out of validity");
    guarantee(is_uimm2(Fmt), "FMT is out of validity");
    guarantee(is_uimm5(funct5), "Funct5 is out of validity");
    uint32_t insn = 0;
    patch((address)&insn,   6, 0, OP_FP_MAJOR);
    patch((address)&insn, 11,  7, Rd);
    patch((address)&insn, 14, 12, rm);
    patch((address)&insn, 19, 15, Rs1);
    patch((address)&insn, 24, 20, Rs2);
    patch((address)&insn, 26, 25, Fmt);
    patch((address)&insn, 31, 27, funct5);
    emit(insn);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm) {
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2->raw_encoding(), rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, int8_t rm) {
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2->raw_encoding(), (RoundingMode)rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(Register Rd, FloatRegister Rs1, FloatRegister Rs2, int8_t rm) {
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2->raw_encoding(), (RoundingMode)rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(FloatRegister Rd, FloatRegister Rs1, int8_t Rs2, int8_t rm) {
    guarantee(is_uimm5(Rs2), "Rs2 is out of validity");
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2, (RoundingMode)rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(FloatRegister Rd, Register Rs1, FloatRegister Rs2, RoundingMode rm) {
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2->raw_encoding(), rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(Register Rd, FloatRegister Rs1, uint8_t Rs2, RoundingMode rm) {
    guarantee(is_uimm5(Rs2), "Rs2 is out of validity");
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2, rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(Register Rd, FloatRegister Rs1, uint8_t Rs2, uint8_t rm) {
    guarantee(is_uimm5(Rs2), "Rs2 is out of validity");
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2, (RoundingMode)rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(FloatRegister Rd, Register Rs1, uint8_t Rs2, RoundingMode rm) {
    guarantee(is_uimm5(Rs2), "Rs2 is out of validity");
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2, rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(FloatRegister Rd, Register Rs1, uint8_t Rs2, int8_t rm) {
    guarantee(is_uimm5(Rs2), "Rs2 is out of validity");
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1->raw_encoding(), Rs2, (RoundingMode)rm);
  }

  template <FmtPrecision Fmt, uint8_t funct5>
  void fp_base(FloatRegister Rd, uint8_t Rs1, uint8_t Rs2, int8_t rm) {
    guarantee(is_uimm5(Rs1), "Rs1 is out of validity");
    guarantee(is_uimm5(Rs2), "Rs2 is out of validity");
    fp_base<Fmt, funct5>(Rd->raw_encoding(), Rs1, Rs2, (RoundingMode)rm);
  }

 public:

  enum FClassBits {
    minf       = 1 << 0,   // negative infinite
    mnorm      = 1 << 1,   // negative normal number
    msubnorm   = 1 << 2,   // negative subnormal number
    mzero      = 1 << 3,   // negative zero
    pzero      = 1 << 4,   // positive zero
    psubnorm   = 1 << 5,   // positive subnormal number
    pnorm      = 1 << 6,   // positive normal number
    pinf       = 1 << 7,   // positive infinite
    snan       = 1 << 8,   // signaling NaN
    qnan       = 1 << 9,   // quiet NaN
    zero       = mzero    | pzero,
    subnorm    = msubnorm | psubnorm,
    norm       = mnorm    | pnorm,
    inf        = minf     | pinf,
    nan        = snan     | qnan,
    finite     = zero     | subnorm   | norm,
  };

  void fsqrt_s(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b01011>(Rd, Rs1, 0b00000, rm);
  }

  void fsqrt_d(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b01011>(Rd, Rs1, 0b00000, rm);
  }

  void fcvt_s_d(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b01000>(Rd, Rs1, 0b00001, rm);
  }

  void fcvt_d_s(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b01000>(Rd, Rs1, 0b00000, rm);
  }

  void fsgnj_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b00100>(Rd, Rs1, Rs2, 0b000);
  }

  void fsgnjn_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b00100>(Rd, Rs1, Rs2, 0b001);
  }

  void fsgnjx_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b00100>(Rd, Rs1, Rs2, 0b010);
  }

  void fmin_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b00101>(Rd, Rs1, Rs2, 0b000);
  }

  void fmax_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b00101>(Rd, Rs1, Rs2, 0b001);
  }

  void fsgnj_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b00100>(Rd, Rs1, Rs2, 0b000);
  }

  void fsgnjn_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b00100>(Rd, Rs1, Rs2, 0b001);
  }

  void fsgnjx_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b00100>(Rd, Rs1, Rs2, 0b010);
  }

  void fmin_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b00101>(Rd, Rs1, Rs2, 0b000);
  }

  void fmax_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b00101>(Rd, Rs1, Rs2, 0b001);
  }

  void feq_s(Register Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b10100>(Rd, Rs1, Rs2, 0b010);
  }

  void flt_s(Register Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b10100>(Rd, Rs1, Rs2, 0b001);
  }

  void fle_s(Register Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<S_32_sp, 0b10100>(Rd, Rs1, Rs2, 0b000);
  }

  void feq_d(Register Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b10100>(Rd, Rs1, Rs2, 0b010);
  }

  void fle_d(Register Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b10100>(Rd, Rs1, Rs2, 0b000);
  }

  void flt_d(Register Rd, FloatRegister Rs1, FloatRegister Rs2) {
    fp_base<D_64_dp, 0b10100>(Rd, Rs1, Rs2, 0b001);
  }

  void fadd_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b00000>(Rd, Rs1, Rs2, rm);
  }

  void fsub_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b00001>(Rd, Rs1, Rs2, rm);
  }

  void fmul_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b00010>(Rd, Rs1, Rs2, rm);
  }

  void fdiv_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b00011>(Rd, Rs1, Rs2, rm);
  }

  void fadd_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b00000>(Rd, Rs1, Rs2, rm);
  }

  void fsub_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b00001>(Rd, Rs1, Rs2, rm);
  }

  void fmul_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b00010>(Rd, Rs1, Rs2, rm);
  }

  void fdiv_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b00011>(Rd, Rs1, Rs2, rm);
  }

  void fcvt_s_w(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b11010>(Rd, Rs1, 0b00000, rm);
  }

  void fcvt_s_wu(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b11010>(Rd, Rs1, 0b00001, rm);
  }

  void fcvt_s_l(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b11010>(Rd, Rs1, 0b00010, rm);
  }

  void fcvt_s_lu(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<S_32_sp, 0b11010>(Rd, Rs1, 0b00011, rm);
  }

  void fcvt_d_w(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b11010>(Rd, Rs1, 0b00000, rm);
  }

  void fcvt_d_wu(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b11010>(Rd, Rs1, 0b00001, rm);
  }

  void fcvt_d_l(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b11010>(Rd, Rs1, 0b00010, rm);
  }

  void fcvt_d_lu(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {
    fp_base<D_64_dp, 0b11010>(Rd, Rs1, 0b00011, rm);
  }

  void fcvt_w_s(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<S_32_sp, 0b11000>(Rd, Rs1, 0b00000, rm);
  }

  void fcvt_l_s(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<S_32_sp, 0b11000>(Rd, Rs1, 0b00010, rm);
  }

  void fcvt_wu_s(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<S_32_sp, 0b11000>(Rd, Rs1, 0b00001, rm);
  }

  void fcvt_lu_s(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<S_32_sp, 0b11000>(Rd, Rs1, 0b00011, rm);
  }

  void fcvt_w_d(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<D_64_dp, 0b11000>(Rd, Rs1, 0b00000, rm);
  }

  void fcvt_wu_d(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<D_64_dp, 0b11000>(Rd, Rs1, 0b00001, rm);
  }

  void fcvt_l_d(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<D_64_dp, 0b11000>(Rd, Rs1, 0b00010, rm);
  }

  void fcvt_lu_d(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {
    fp_base<D_64_dp, 0b11000>(Rd, Rs1, 0b00011, rm);
  }

  void fmv_w_x(FloatRegister Rd, Register Rs1) {
    fp_base<S_32_sp, 0b11110>(Rd, Rs1, 0b00000, 0b000);
  }

  void fmv_d_x(FloatRegister Rd, Register Rs1) {
    fp_base<D_64_dp, 0b11110>(Rd, Rs1, 0b00000, 0b000);
  }

  void fclass_s(Register Rd, FloatRegister Rs1) {
    fp_base<S_32_sp, 0b11100>(Rd, Rs1, 0b00000, 0b001);
  }

  void fclass_d(Register Rd, FloatRegister Rs1) {
    fp_base<D_64_dp, 0b11100>(Rd, Rs1, 0b00000, 0b001);
  }

  void fmv_x_w(Register Rd, FloatRegister Rs1) {
    fp_base<S_32_sp, 0b11100>(Rd, Rs1, 0b00000, 0b000);
  }

  void fmv_x_d(Register Rd, FloatRegister Rs1) {
    fp_base<D_64_dp, 0b11100>(Rd, Rs1, 0b00000, 0b000);
  }

 private:
  template <FmtPrecision Fmt, uint8_t OpVal>
  void fp_fm(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm) {
    assert_cond(Fmt != Q_128_qp);
    guarantee(is_uimm3(rm), "Rounding mode is out of validity");
    guarantee(is_uimm2(Fmt), "FMT is out of validity");
    unsigned insn = 0;
    patch((address)&insn,   6, 0, OpVal);
    patch_reg((address)&insn,  7, Rd);
    patch((address)&insn, 14, 12, rm);
    patch_reg((address)&insn, 15, Rs1);
    patch_reg((address)&insn, 20, Rs2);
    patch((address)&insn, 26, 25, Fmt);
    patch_reg((address)&insn, 27, Rs3);
    emit(insn);
  }

 public:
  void fmadd_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne)  {
    fp_fm<S_32_sp, 0b1000011>(Rd, Rs1, Rs2, Rs3, rm);
  }

  void fmsub_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne)  {
    fp_fm<S_32_sp, 0b1000111>(Rd, Rs1, Rs2, Rs3, rm);
  }

  void fnmsub_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne) {
    fp_fm<S_32_sp, 0b1001011>(Rd, Rs1, Rs2, Rs3, rm);
  }

  void fnmadd_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne) {
    fp_fm<S_32_sp, 0b1001111>(Rd, Rs1, Rs2, Rs3, rm);
  }

  void fmadd_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne)  {
    fp_fm<D_64_dp, 0b1000011>(Rd, Rs1, Rs2, Rs3, rm);
  }

  void fmsub_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne)  {
    fp_fm<D_64_dp, 0b1000111>(Rd, Rs1, Rs2, Rs3, rm);
  }

  void fnmsub_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne) {
    fp_fm<D_64_dp, 0b1001011>(Rd, Rs1, Rs2, Rs3, rm);
  }

  void fnmadd_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne) {
    fp_fm<D_64_dp, 0b1001111>(Rd, Rs1, Rs2, Rs3, rm);
  }

// --------------  ZFH Instruction Definitions  --------------
// Zfh Standard Extensions for Half-Precision Floating-Point
  void fclass_h(Register Rd, FloatRegister Rs1) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b11100>(Rd, Rs1, 0b00000, 0b001);
  }

// Zfh and Zfhmin Half-Precision Floating-Point
  void fcvt_s_h(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {
    assert_cond(UseZfh || UseZfhmin);
    fp_base<S_32_sp, 0b01000>(Rd, Rs1, 0b00010, rm);
  }

  void fcvt_h_s(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {
    assert_cond(UseZfh || UseZfhmin);
    fp_base<H_16_hp, 0b01000>(Rd, Rs1, 0b00000, rm);
  }

  void fmv_h_x(FloatRegister Rd, Register Rs1) {
    assert_cond(UseZfh || UseZfhmin);
    fp_base<H_16_hp, 0b11110>(Rd, Rs1, 0b00000, 0b000);
  }

  void fmv_x_h(Register Rd, FloatRegister Rs1) {
    assert_cond(UseZfh || UseZfhmin);
    fp_base<H_16_hp, 0b11100>(Rd, Rs1, 0b00000, 0b000);
  }

  void fadd_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b00000>(Rd, Rs1, Rs2, rm);
  }

  void fsub_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b00001>(Rd, Rs1, Rs2, rm);
  }

  void fmul_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b00010>(Rd, Rs1, Rs2, rm);
  }

  void fdiv_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b00011>(Rd, Rs1, Rs2, rm);
  }

  void fsqrt_h(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b01011>(Rd, Rs1, 0b00000, rm);
  }

  void fmin_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b00101>(Rd, Rs1, Rs2, 0b000);
  }

  void fmax_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfh);
    fp_base<H_16_hp, 0b00101>(Rd, Rs1, Rs2, 0b001);
  }

  void fmadd_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne)  {
    assert_cond(UseZfh);
    fp_fm<H_16_hp, 0b1000011>(Rd, Rs1, Rs2, Rs3, rm);
  }

// --------------  ZFA Instruction Definitions  --------------
// Zfa Extension for Additional Floating-Point Instructions
  void _fli_h(FloatRegister Rd, uint8_t Rs1) {
    assert_cond(UseZfa && UseZfh);
    fp_base<H_16_hp, 0b11110>(Rd, Rs1, 0b00001, 0b000);
  }

  void _fli_s(FloatRegister Rd, uint8_t Rs1) {
    assert_cond(UseZfa);
    fp_base<S_32_sp, 0b11110>(Rd, Rs1, 0b00001, 0b000);
  }

  void _fli_d(FloatRegister Rd, uint8_t Rs1) {
    assert_cond(UseZfa);
    fp_base<D_64_dp, 0b11110>(Rd, Rs1, 0b00001, 0b000);
  }

  void fminm_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfa && UseZfh);
    fp_base<H_16_hp, 0b00101>(Rd, Rs1, Rs2, 0b010);
  }

  void fmaxm_h(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfa && UseZfh);
    fp_base<H_16_hp, 0b00101>(Rd, Rs1, Rs2, 0b011);
  }

  void fminm_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfa);
    fp_base<S_32_sp, 0b00101>(Rd, Rs1, Rs2, 0b010);
  }

  void fmaxm_s(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfa);
    fp_base<S_32_sp, 0b00101>(Rd, Rs1, Rs2, 0b011);
  }

  void fminm_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfa);
    fp_base<D_64_dp, 0b00101>(Rd, Rs1, Rs2, 0b010);
  }

  void fmaxm_d(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {
    assert_cond(UseZfa);
    fp_base<D_64_dp, 0b00101>(Rd, Rs1, Rs2, 0b011);
  }

// ==========================
// RISC-V Vector Extension
// ==========================
enum SEW {
  e8,
  e16,
  e32,
  e64,
  RESERVED,
};

enum LMUL {
  mf8 = 0b101,
  mf4 = 0b110,
  mf2 = 0b111,
  m1  = 0b000,
  m2  = 0b001,
  m4  = 0b010,
  m8  = 0b011,
};

enum VMA {
  mu, // undisturbed
  ma, // agnostic
};

enum VTA {
  tu, // undisturbed
  ta, // agnostic
};

static Assembler::SEW elembytes_to_sew(int ebytes) {
  assert(ebytes > 0 && ebytes <= 8, "unsupported element size");
  return (Assembler::SEW) exact_log2(ebytes);
}

static Assembler::SEW elemtype_to_sew(BasicType etype) {
  return Assembler::elembytes_to_sew(type2aelembytes(etype));
}

#define patch_vtype(hsb, lsb, vlmul, vsew, vta, vma, vill)   \
    /* If vill then other bits of vtype must be zero. */     \
    guarantee(!vill, "vill not supported");                  \
    patch((address)&insn, lsb + 2, lsb, vlmul);              \
    patch((address)&insn, lsb + 5, lsb + 3, vsew);           \
    patch((address)&insn, lsb + 6, vta);                     \
    patch((address)&insn, lsb + 7, vma);                     \
    patch((address)&insn, hsb - 1, lsb + 8, 0);              \
    patch((address)&insn, hsb, vill)

#define INSN(NAME, op, funct3)                                            \
  void NAME(Register Rd, Register Rs1, SEW sew, LMUL lmul = m1,           \
            VMA vma = mu, VTA vta = tu, bool vill = false) {              \
    unsigned insn = 0;                                                    \
    patch((address)&insn, 6, 0, op);                                      \
    patch((address)&insn, 14, 12, funct3);                                \
    patch_vtype(30, 20, lmul, sew, vta, vma, vill);                       \
    patch((address)&insn, 31, 0);                                         \
    patch_reg((address)&insn, 7, Rd);                                     \
    patch_reg((address)&insn, 15, Rs1);                                   \
    emit(insn);                                                           \
  }

  INSN(vsetvli, 0b1010111, 0b111);

#undef INSN

#define INSN(NAME, op, funct3)                                            \
  void NAME(Register Rd, uint32_t imm, SEW sew, LMUL lmul = m1,           \
            VMA vma = mu, VTA vta = tu, bool vill = false) {              \
    unsigned insn = 0;                                                    \
    guarantee(is_uimm5(imm), "uimm is invalid");                           \
    patch((address)&insn, 6, 0, op);                                      \
    patch((address)&insn, 14, 12, funct3);                                \
    patch((address)&insn, 19, 15, imm);                                   \
    patch_vtype(29, 20, lmul, sew, vta, vma, vill);                       \
    patch((address)&insn, 31, 30, 0b11);                                  \
    patch_reg((address)&insn, 7, Rd);                                     \
    emit(insn);                                                           \
  }

  INSN(vsetivli, 0b1010111, 0b111);

#undef INSN

#undef patch_vtype

#define INSN(NAME, op, funct3, funct7)                          \
  void NAME(Register Rd, Register Rs1, Register Rs2) {          \
    unsigned insn = 0;                                          \
    patch((address)&insn, 6,  0, op);                           \
    patch((address)&insn, 14, 12, funct3);                      \
    patch((address)&insn, 31, 25, funct7);                      \
    patch_reg((address)&insn, 7, Rd);                           \
    patch_reg((address)&insn, 15, Rs1);                         \
    patch_reg((address)&insn, 20, Rs2);                         \
    emit(insn);                                                 \
  }

  // Vector Configuration Instruction
  INSN(vsetvl, 0b1010111, 0b111, 0b1000000);

#undef INSN

enum VectorMask {
  v0_t = 0b0,
  unmasked = 0b1
};

#define patch_VArith(op, Reg, funct3, Reg_or_Imm5, Vs2, vm, funct6)            \
    unsigned insn = 0;                                                         \
    patch((address)&insn, 6, 0, op);                                           \
    patch((address)&insn, 14, 12, funct3);                                     \
    patch((address)&insn, 19, 15, Reg_or_Imm5);                                \
    patch((address)&insn, 25, vm);                                             \
    patch((address)&insn, 31, 26, funct6);                                     \
    patch_reg((address)&insn, 7, Reg);                                         \
    patch_reg((address)&insn, 20, Vs2);                                        \
    emit(insn)

// r2_vm
#define INSN(NAME, op, funct3, Vs1, funct6)                                    \
  void NAME(Register Rd, VectorRegister Vs2, VectorMask vm = unmasked) {       \
    patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6);                        \
  }

  // Vector Mask
  INSN(vcpop_m,  0b1010111, 0b010, 0b10000, 0b010000);
  INSN(vfirst_m, 0b1010111, 0b010, 0b10001, 0b010000);
#undef INSN

#define INSN(NAME, op, funct3, Vs1, funct6)                                    \
  void NAME(VectorRegister Vd, VectorRegister Vs2, VectorMask vm = unmasked) { \
    patch_VArith(op, Vd, funct3, Vs1, Vs2, vm, funct6);                        \
  }

  // Vector Integer Extension
  INSN(vzext_vf2, 0b1010111, 0b010, 0b00110, 0b010010);
  INSN(vzext_vf4, 0b1010111, 0b010, 0b00100, 0b010010);
  INSN(vzext_vf8, 0b1010111, 0b010, 0b00010, 0b010010);
  INSN(vsext_vf2, 0b1010111, 0b010, 0b00111, 0b010010);
  INSN(vsext_vf4, 0b1010111, 0b010, 0b00101, 0b010010);
  INSN(vsext_vf8, 0b1010111, 0b010, 0b00011, 0b010010);

  // Vector Mask
  INSN(vmsbf_m,   0b1010111, 0b010, 0b00001, 0b010100);
  INSN(vmsif_m,   0b1010111, 0b010, 0b00011, 0b010100);
  INSN(vmsof_m,   0b1010111, 0b010, 0b00010, 0b010100);
  INSN(viota_m,   0b1010111, 0b010, 0b10000, 0b010100);

  // Vector Single-Width Floating-Point/Integer Type-Convert Instructions
  INSN(vfcvt_x_f_v,      0b1010111, 0b001, 0b00001, 0b010010);
  INSN(vfcvt_f_x_v,      0b1010111, 0b001, 0b00011, 0b010010);
  INSN(vfcvt_rtz_x_f_v,  0b1010111, 0b001, 0b00111, 0b010010);

  // Vector Widening Floating-Point/Integer Type-Convert Instructions
  INSN(vfwcvt_f_x_v,      0b1010111, 0b001, 0b01011, 0b010010);
  INSN(vfwcvt_f_f_v,      0b1010111, 0b001, 0b01100, 0b010010);
  INSN(vfwcvt_rtz_x_f_v,  0b1010111, 0b001, 0b01111, 0b010010);

  // Vector Narrowing Floating-Point/Integer Type-Convert Instructions
  INSN(vfncvt_f_x_w,      0b1010111, 0b001, 0b10011, 0b010010);
  INSN(vfncvt_f_f_w,      0b1010111, 0b001, 0b10100, 0b010010);
  INSN(vfncvt_rtz_x_f_w,  0b1010111, 0b001, 0b10111, 0b010010);

  // Vector Floating-Point Instruction
  INSN(vfsqrt_v,  0b1010111, 0b001, 0b00000, 0b010011);
  INSN(vfclass_v, 0b1010111, 0b001, 0b10000, 0b010011);

#undef INSN

// r2rd
#define INSN(NAME, op, funct3, simm5, vm, funct6)         \
  void NAME(VectorRegister Vd, VectorRegister Vs2) {      \
    patch_VArith(op, Vd, funct3, simm5, Vs2, vm, funct6); \
  }

  // Vector Whole Vector Register Move
  INSN(vmv1r_v, 0b1010111, 0b011, 0b00000, 0b1, 0b100111);
  INSN(vmv2r_v, 0b1010111, 0b011, 0b00001, 0b1, 0b100111);
  INSN(vmv4r_v, 0b1010111, 0b011, 0b00011, 0b1, 0b100111);
  INSN(vmv8r_v, 0b1010111, 0b011, 0b00111, 0b1, 0b100111);

#undef INSN

#define INSN(NAME, op, funct3, Vs1, vm, funct6)           \
  void NAME(FloatRegister Rd, VectorRegister Vs2) {       \
    patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6);   \
  }

  // Vector Floating-Point Move Instruction
  INSN(vfmv_f_s, 0b1010111, 0b001, 0b00000, 0b1, 0b010000);

#undef INSN

#define INSN(NAME, op, funct3, Vs1, vm, funct6)          \
  void NAME(Register Rd, VectorRegister Vs2) {           \
    patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6);  \
  }

  // Vector Integer Scalar Move Instructions
  INSN(vmv_x_s, 0b1010111, 0b010, 0b00000, 0b1, 0b010000);

#undef INSN

// r_vm
#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {       \
    guarantee(is_uimm5(imm), "uimm is invalid");                                                    \
    patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);                         \
  }

  // Vector Single-Width Bit Shift Instructions
  INSN(vsra_vi,    0b1010111, 0b011, 0b101001);
  INSN(vsrl_vi,    0b1010111, 0b011, 0b101000);
  INSN(vsll_vi,    0b1010111, 0b011, 0b100101);

  // Vector Slide Instructions
  INSN(vslideup_vi,   0b1010111, 0b011, 0b001110);
  INSN(vslidedown_vi, 0b1010111, 0b011, 0b001111);

  // Vector Narrowing Integer Right Shift Instructions
  INSN(vnsra_wi, 0b1010111, 0b011, 0b101101);
  INSN(vnsrl_wi, 0b1010111, 0b011, 0b101100);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs1, VectorRegister Vs2, VectorMask vm = unmasked) { \
    patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
  INSN(vfnmsub_vv, 0b1010111, 0b001, 0b101011);
  INSN(vfmsub_vv,  0b1010111, 0b001, 0b101010);
  INSN(vfnmadd_vv, 0b1010111, 0b001, 0b101001);
  INSN(vfmadd_vv,  0b1010111, 0b001, 0b101000);
  INSN(vfnmsac_vv, 0b1010111, 0b001, 0b101111);
  INSN(vfmsac_vv,  0b1010111, 0b001, 0b101110);
  INSN(vfmacc_vv,  0b1010111, 0b001, 0b101100);
  INSN(vfnmacc_vv, 0b1010111, 0b001, 0b101101);

  // Vector Single-Width Integer Multiply-Add Instructions
  INSN(vnmsub_vv, 0b1010111, 0b010, 0b101011);
  INSN(vmadd_vv,  0b1010111, 0b010, 0b101001);
  INSN(vnmsac_vv, 0b1010111, 0b010, 0b101111);
  INSN(vmacc_vv,  0b1010111, 0b010, 0b101101);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, Register Rs1, VectorRegister Vs2, VectorMask vm = unmasked) {       \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Single-Width Integer Multiply-Add Instructions
  INSN(vnmsub_vx, 0b1010111, 0b110, 0b101011);
  INSN(vmadd_vx,  0b1010111, 0b110, 0b101001);
  INSN(vnmsac_vx, 0b1010111, 0b110, 0b101111);
  INSN(vmacc_vx,  0b1010111, 0b110, 0b101101);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, FloatRegister Rs1, VectorRegister Vs2, VectorMask vm = unmasked) {  \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
  INSN(vfnmsub_vf, 0b1010111, 0b101, 0b101011);
  INSN(vfmsub_vf,  0b1010111, 0b101, 0b101010);
  INSN(vfnmadd_vf, 0b1010111, 0b101, 0b101001);
  INSN(vfmadd_vf,  0b1010111, 0b101, 0b101000);
  INSN(vfnmsac_vf, 0b1010111, 0b101, 0b101111);
  INSN(vfmsac_vf,  0b1010111, 0b101, 0b101110);
  INSN(vfmacc_vf,  0b1010111, 0b101, 0b101100);
  INSN(vfnmacc_vf, 0b1010111, 0b101, 0b101101);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1, VectorMask vm = unmasked) { \
    patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Single-Width Floating-Point Reduction Instructions
  INSN(vfredusum_vs,  0b1010111, 0b001, 0b000001);
  INSN(vfredosum_vs,  0b1010111, 0b001, 0b000011);
  INSN(vfredmin_vs,   0b1010111, 0b001, 0b000101);
  INSN(vfredmax_vs,   0b1010111, 0b001, 0b000111);

  // Vector Single-Width Integer Reduction Instructions
  INSN(vredsum_vs,    0b1010111, 0b010, 0b000000);
  INSN(vredand_vs,    0b1010111, 0b010, 0b000001);
  INSN(vredor_vs,     0b1010111, 0b010, 0b000010);
  INSN(vredxor_vs,    0b1010111, 0b010, 0b000011);
  INSN(vredminu_vs,   0b1010111, 0b010, 0b000100);
  INSN(vredmin_vs,    0b1010111, 0b010, 0b000101);
  INSN(vredmaxu_vs,   0b1010111, 0b010, 0b000110);
  INSN(vredmax_vs,    0b1010111, 0b010, 0b000111);

  // Vector Widening Integer Reduction Instructions
  INSN(vwredsum_vs,    0b1010111, 0b000, 0b110001);
  INSN(vwredsumu_vs,   0b1010111, 0b000, 0b110000);

  // Vector Floating-Point Compare Instructions
  INSN(vmfle_vv, 0b1010111, 0b001, 0b011001);
  INSN(vmflt_vv, 0b1010111, 0b001, 0b011011);
  INSN(vmfne_vv, 0b1010111, 0b001, 0b011100);
  INSN(vmfeq_vv, 0b1010111, 0b001, 0b011000);

  // Vector Floating-Point Sign-Injection Instructions
  INSN(vfsgnj_vv, 0b1010111, 0b001, 0b001000);
  INSN(vfsgnjx_vv, 0b1010111, 0b001, 0b001010);
  INSN(vfsgnjn_vv, 0b1010111, 0b001, 0b001001);

  // Vector Floating-Point MIN/MAX Instructions
  INSN(vfmax_vv,   0b1010111, 0b001, 0b000110);
  INSN(vfmin_vv,   0b1010111, 0b001, 0b000100);

  // Vector Single-Width Floating-Point Multiply/Divide Instructions
  INSN(vfdiv_vv,   0b1010111, 0b001, 0b100000);
  INSN(vfmul_vv,   0b1010111, 0b001, 0b100100);

  // Vector Single-Width Floating-Point Add/Subtract Instructions
  INSN(vfsub_vv, 0b1010111, 0b001, 0b000010);
  INSN(vfadd_vv, 0b1010111, 0b001, 0b000000);

  // Vector Single-Width Fractional Multiply with Rounding and Saturation
  INSN(vsmul_vv, 0b1010111, 0b000, 0b100111);

  // Vector Integer Divide Instructions
  INSN(vrem_vv,  0b1010111, 0b010, 0b100011);
  INSN(vremu_vv, 0b1010111, 0b010, 0b100010);
  INSN(vdiv_vv,  0b1010111, 0b010, 0b100001);
  INSN(vdivu_vv, 0b1010111, 0b010, 0b100000);

  // Vector Single-Width Integer Multiply Instructions
  INSN(vmulhsu_vv, 0b1010111, 0b010, 0b100110);
  INSN(vmulhu_vv,  0b1010111, 0b010, 0b100100);
  INSN(vmulh_vv,   0b1010111, 0b010, 0b100111);
  INSN(vmul_vv,    0b1010111, 0b010, 0b100101);

  // Vector Widening Integer Multiply Instructions
  INSN(vwmul_vv,    0b1010111, 0b010, 0b111011);
  INSN(vwmulu_vv,   0b1010111, 0b010, 0b111000);

  // Vector Integer Min/Max Instructions
  INSN(vmax_vv,  0b1010111, 0b000, 0b000111);
  INSN(vmaxu_vv, 0b1010111, 0b000, 0b000110);
  INSN(vmin_vv,  0b1010111, 0b000, 0b000101);
  INSN(vminu_vv, 0b1010111, 0b000, 0b000100);

  // Vector Integer Comparison Instructions
  INSN(vmsle_vv,  0b1010111, 0b000, 0b011101);
  INSN(vmsleu_vv, 0b1010111, 0b000, 0b011100);
  INSN(vmslt_vv,  0b1010111, 0b000, 0b011011);
  INSN(vmsltu_vv, 0b1010111, 0b000, 0b011010);
  INSN(vmsne_vv,  0b1010111, 0b000, 0b011001);
  INSN(vmseq_vv,  0b1010111, 0b000, 0b011000);

  // Vector Single-Width Bit Shift Instructions
  INSN(vsra_vv, 0b1010111, 0b000, 0b101001);
  INSN(vsrl_vv, 0b1010111, 0b000, 0b101000);
  INSN(vsll_vv, 0b1010111, 0b000, 0b100101);

  // Vector Bitwise Logical Instructions
  INSN(vxor_vv, 0b1010111, 0b000, 0b001011);
  INSN(vor_vv,  0b1010111, 0b000, 0b001010);
  INSN(vand_vv, 0b1010111, 0b000, 0b001001);

  // Vector Single-Width Integer Add and Subtract
  INSN(vadd_vv, 0b1010111, 0b000, 0b000000);
  INSN(vsub_vv, 0b1010111, 0b000, 0b000010);

  // Vector Saturating Integer Add and Subtract
  INSN(vsadd_vv,  0b1010111, 0b000, 0b100001);
  INSN(vsaddu_vv, 0b1010111, 0b000, 0b100000);
  INSN(vssub_vv,  0b1010111, 0b000, 0b100011);
  INSN(vssubu_vv, 0b1010111, 0b000, 0b100010);

  // Vector Register Gather Instructions
  INSN(vrgather_vv,     0b1010111, 0b000, 0b001100);

#undef INSN


#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, Register Rs1, VectorMask vm = unmasked) {       \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Integer Divide Instructions
  INSN(vrem_vx,  0b1010111, 0b110, 0b100011);
  INSN(vremu_vx, 0b1010111, 0b110, 0b100010);
  INSN(vdiv_vx,  0b1010111, 0b110, 0b100001);
  INSN(vdivu_vx, 0b1010111, 0b110, 0b100000);

  // Vector Single-Width Integer Multiply Instructions
  INSN(vmulhsu_vx, 0b1010111, 0b110, 0b100110);
  INSN(vmulhu_vx,  0b1010111, 0b110, 0b100100);
  INSN(vmulh_vx,   0b1010111, 0b110, 0b100111);
  INSN(vmul_vx,    0b1010111, 0b110, 0b100101);

  // Vector Widening Integer Add/Subtract
  INSN(vwadd_vx, 0b1010111, 0b110, 0b110001);

  // Vector Integer Min/Max Instructions
  INSN(vmax_vx,  0b1010111, 0b100, 0b000111);
  INSN(vmaxu_vx, 0b1010111, 0b100, 0b000110);
  INSN(vmin_vx,  0b1010111, 0b100, 0b000101);
  INSN(vminu_vx, 0b1010111, 0b100, 0b000100);

  // Vector Integer Comparison Instructions
  INSN(vmsgt_vx,  0b1010111, 0b100, 0b011111);
  INSN(vmsgtu_vx, 0b1010111, 0b100, 0b011110);
  INSN(vmsle_vx,  0b1010111, 0b100, 0b011101);
  INSN(vmsleu_vx, 0b1010111, 0b100, 0b011100);
  INSN(vmslt_vx,  0b1010111, 0b100, 0b011011);
  INSN(vmsltu_vx, 0b1010111, 0b100, 0b011010);
  INSN(vmsne_vx,  0b1010111, 0b100, 0b011001);
  INSN(vmseq_vx,  0b1010111, 0b100, 0b011000);

  // Vector Narrowing Integer Right Shift Instructions
  INSN(vnsra_wx, 0b1010111, 0b100, 0b101101);
  INSN(vnsrl_wx, 0b1010111, 0b100, 0b101100);

  // Vector Single-Width Bit Shift Instructions
  INSN(vsra_vx, 0b1010111, 0b100, 0b101001);
  INSN(vsrl_vx, 0b1010111, 0b100, 0b101000);
  INSN(vsll_vx, 0b1010111, 0b100, 0b100101);

  // Vector Bitwise Logical Instructions
  INSN(vxor_vx, 0b1010111, 0b100, 0b001011);
  INSN(vor_vx,  0b1010111, 0b100, 0b001010);
  INSN(vand_vx, 0b1010111, 0b100, 0b001001);

  // Vector Single-Width Integer Add and Subtract
  INSN(vsub_vx,  0b1010111, 0b100, 0b000010);
  INSN(vadd_vx,  0b1010111, 0b100, 0b000000);

  // Vector Integer reverse subtract
  INSN(vrsub_vx, 0b1010111, 0b100, 0b000011);

  // Vector Slide Instructions
  INSN(vslidedown_vx, 0b1010111, 0b100, 0b001111);

#undef INSN

#define INSN(NAME, op, funct3, vm, funct6)                                                         \
  void NAME(VectorRegister Vd, VectorRegister Vs2, Register Rs1) {                                 \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Integer Merge Instructions
  INSN(vmerge_vxm,  0b1010111, 0b100, 0b0, 0b010111);

#undef INSN

#define INSN(NAME, op, funct3, vm, funct6)                                                         \
  void NAME(VectorRegister Vd, VectorRegister Vs2, FloatRegister Rs1) {                            \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Floating-Point Merge Instruction
  INSN(vfmerge_vfm,  0b1010111, 0b101, 0b0, 0b010111);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, FloatRegister Rs1, VectorMask vm = unmasked) {  \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Floating-Point Compare Instructions
  INSN(vmfge_vf, 0b1010111, 0b101, 0b011111);
  INSN(vmfgt_vf, 0b1010111, 0b101, 0b011101);
  INSN(vmfle_vf, 0b1010111, 0b101, 0b011001);
  INSN(vmflt_vf, 0b1010111, 0b101, 0b011011);
  INSN(vmfne_vf, 0b1010111, 0b101, 0b011100);
  INSN(vmfeq_vf, 0b1010111, 0b101, 0b011000);

  // Vector Floating-Point MIN/MAX Instructions
  INSN(vfmax_vf, 0b1010111, 0b101, 0b000110);
  INSN(vfmin_vf, 0b1010111, 0b101, 0b000100);

  // Vector Single-Width Floating-Point Multiply/Divide Instructions
  INSN(vfdiv_vf,  0b1010111, 0b101, 0b100000);
  INSN(vfmul_vf,  0b1010111, 0b101, 0b100100);
  INSN(vfrdiv_vf, 0b1010111, 0b101, 0b100001);

  // Vector Single-Width Floating-Point Add/Subtract Instructions
  INSN(vfsub_vf,  0b1010111, 0b101, 0b000010);
  INSN(vfadd_vf,  0b1010111, 0b101, 0b000000);
  INSN(vfrsub_vf, 0b1010111, 0b101, 0b100111);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, int32_t imm, VectorMask vm = unmasked) {        \
    guarantee(is_simm5(imm), "imm is invalid");                                                    \
    patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);                         \
  }

  // Vector Integer Comparison Instructions
  INSN(vmsgt_vi,  0b1010111, 0b011, 0b011111);
  INSN(vmsgtu_vi, 0b1010111, 0b011, 0b011110);
  INSN(vmsle_vi,  0b1010111, 0b011, 0b011101);
  INSN(vmsleu_vi, 0b1010111, 0b011, 0b011100);
  INSN(vmsne_vi,  0b1010111, 0b011, 0b011001);
  INSN(vmseq_vi,  0b1010111, 0b011, 0b011000);

  // Vector Bitwise Logical Instructions
  INSN(vxor_vi,   0b1010111, 0b011, 0b001011);
  INSN(vor_vi,    0b1010111, 0b011, 0b001010);
  INSN(vand_vi,   0b1010111, 0b011, 0b001001);

  // Vector Single-Width Integer Add and Subtract
  INSN(vadd_vi,   0b1010111, 0b011, 0b000000);

  // Vector Integer reverse subtract
  INSN(vrsub_vi,  0b1010111, 0b011, 0b000011);

#undef INSN

#define INSN(NAME, op, funct3, vm, funct6)                                    \
  void NAME(VectorRegister Vd, VectorRegister Vs2, int32_t imm) {             \
    guarantee(is_simm5(imm), "imm is invalid");                               \
    patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);    \
  }

  // Vector Integer Merge Instructions
  INSN(vmerge_vim,  0b1010111, 0b011, 0b0, 0b010111);

#undef INSN

#define INSN(NAME, op, funct3, vm, funct6)                                   \
  void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1) {     \
    patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6);      \
  }

  // Vector Compress Instruction
  INSN(vcompress_vm, 0b1010111, 0b010, 0b1, 0b010111);

  // Vector Mask-Register Logical Instructions
  INSN(vmxnor_mm,   0b1010111, 0b010, 0b1, 0b011111);
  INSN(vmorn_mm,    0b1010111, 0b010, 0b1, 0b011100);
  INSN(vmnor_mm,    0b1010111, 0b010, 0b1, 0b011110);
  INSN(vmor_mm,     0b1010111, 0b010, 0b1, 0b011010);
  INSN(vmxor_mm,    0b1010111, 0b010, 0b1, 0b011011);
  INSN(vmandn_mm,   0b1010111, 0b010, 0b1, 0b011000);
  INSN(vmnand_mm,   0b1010111, 0b010, 0b1, 0b011101);
  INSN(vmand_mm,    0b1010111, 0b010, 0b1, 0b011001);

  // Vector Integer Merge Instructions
  INSN(vmerge_vvm,  0b1010111, 0b000, 0b0, 0b010111);

#undef INSN

#define INSN(NAME, op, funct3, Vs2, vm, funct6)                            \
  void NAME(VectorRegister Vd, int32_t imm) {                              \
    guarantee(is_simm5(imm), "imm is invalid");                            \
    patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6); \
  }

  // Vector Integer Move Instructions
  INSN(vmv_v_i, 0b1010111, 0b011, v0, 0b1, 0b010111);

#undef INSN

#define INSN(NAME, op, funct3, Vs2, vm, funct6)                             \
  void NAME(VectorRegister Vd, FloatRegister Rs1) {                         \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);     \
  }

  // Floating-Point Scalar Move Instructions
  INSN(vfmv_s_f, 0b1010111, 0b101, v0, 0b1, 0b010000);
  // Vector Floating-Point Move Instruction
  INSN(vfmv_v_f, 0b1010111, 0b101, v0, 0b1, 0b010111);

#undef INSN

#define INSN(NAME, op, funct3, Vs2, vm, funct6)                             \
  void NAME(VectorRegister Vd, VectorRegister Vs1) {                        \
    patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6);     \
  }

  // Vector Integer Move Instructions
  INSN(vmv_v_v, 0b1010111, 0b000, v0, 0b1, 0b010111);

#undef INSN

#define INSN(NAME, op, funct3, Vs2, vm, funct6)                             \
   void NAME(VectorRegister Vd, Register Rs1) {                             \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);     \
   }

  // Integer Scalar Move Instructions
  INSN(vmv_s_x, 0b1010111, 0b110, v0, 0b1, 0b010000);

  // Vector Integer Move Instructions
  INSN(vmv_v_x, 0b1010111, 0b100, v0, 0b1, 0b010111);

#undef INSN

#define INSN(NAME, op, funct13, funct6)                    \
  void NAME(VectorRegister Vd, VectorMask vm = unmasked) { \
    unsigned insn = 0;                                     \
    patch((address)&insn, 6, 0, op);                       \
    patch((address)&insn, 24, 12, funct13);                \
    patch((address)&insn, 25, vm);                         \
    patch((address)&insn, 31, 26, funct6);                 \
    patch_reg((address)&insn, 7, Vd);                      \
    emit(insn);                                            \
  }

  // Vector Element Index Instruction
  INSN(vid_v, 0b1010111, 0b0000010001010, 0b010100);

#undef INSN

enum Nf {
  g1 = 0b000,
  g2 = 0b001,
  g3 = 0b010,
  g4 = 0b011,
  g5 = 0b100,
  g6 = 0b101,
  g7 = 0b110,
  g8 = 0b111
};

#define patch_VLdSt(op, VReg, width, Rs1, Reg_or_umop, vm, mop, mew, nf) \
    unsigned insn = 0;                                                   \
    patch((address)&insn, 6, 0, op);                                     \
    patch((address)&insn, 14, 12, width);                                \
    patch((address)&insn, 24, 20, Reg_or_umop);                          \
    patch((address)&insn, 25, vm);                                       \
    patch((address)&insn, 27, 26, mop);                                  \
    patch((address)&insn, 28, mew);                                      \
    patch((address)&insn, 31, 29, nf);                                   \
    patch_reg((address)&insn, 7, VReg);                                  \
    patch_reg((address)&insn, 15, Rs1);                                  \
    emit(insn)

#define INSN(NAME, op, width, lumop, vm, mop, mew, nf)                               \
  void NAME(VectorRegister Vd, Register Rs1) {                                       \
    guarantee(is_uimm3(width), "width is invalid");                                  \
    patch_VLdSt(op, Vd, width, Rs1, lumop, vm, mop, mew, nf);                        \
  }

  // Vector Load/Store Instructions
  INSN(vl1re8_v,  0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g1);
  INSN(vl1re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g1);
  INSN(vl1re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g1);
  INSN(vl1re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g1);
  INSN(vl2re8_v,  0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g2);
  INSN(vl2re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g2);
  INSN(vl2re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g2);
  INSN(vl2re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g2);
  INSN(vl4re8_v,  0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g4);
  INSN(vl4re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g4);
  INSN(vl4re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g4);
  INSN(vl4re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g4);
  INSN(vl8re8_v,  0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g8);
  INSN(vl8re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g8);
  INSN(vl8re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g8);
  INSN(vl8re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g8);

#undef INSN

#define INSN(NAME, op, width, sumop, vm, mop, mew, nf)           \
  void NAME(VectorRegister Vs3, Register Rs1) {                  \
    patch_VLdSt(op, Vs3, width, Rs1, sumop, vm, mop, mew, nf);   \
  }

  // Vector Load/Store Instructions
  INSN(vs1r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g1);
  INSN(vs2r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g2);
  INSN(vs4r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g4);
  INSN(vs8r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g8);

#undef INSN

// r2_nfvm
#define INSN(NAME, op, width, umop, mop, mew)                         \
  void NAME(VectorRegister Vd_or_Vs3, Register Rs1, Nf nf = g1) {     \
    patch_VLdSt(op, Vd_or_Vs3, width, Rs1, umop, 1, mop, mew, nf);    \
  }

  // Vector Unit-Stride Instructions
  INSN(vlm_v, 0b0000111, 0b000, 0b01011, 0b00, 0b0);
  INSN(vsm_v, 0b0100111, 0b000, 0b01011, 0b00, 0b0);

#undef INSN

#define INSN(NAME, op, width, umop, mop, mew)                                               \
  void NAME(VectorRegister Vd_or_Vs3, Register Rs1, VectorMask vm = unmasked, Nf nf = g1) { \
    patch_VLdSt(op, Vd_or_Vs3, width, Rs1, umop, vm, mop, mew, nf);                         \
  }

  // Vector Unit-Stride Instructions
  INSN(vle8_v,    0b0000111, 0b000, 0b00000, 0b00, 0b0);
  INSN(vle16_v,   0b0000111, 0b101, 0b00000, 0b00, 0b0);
  INSN(vle32_v,   0b0000111, 0b110, 0b00000, 0b00, 0b0);
  INSN(vle64_v,   0b0000111, 0b111, 0b00000, 0b00, 0b0);

  // Vector unit-stride fault-only-first Instructions
  INSN(vle8ff_v,  0b0000111, 0b000, 0b10000, 0b00, 0b0);
  INSN(vle16ff_v, 0b0000111, 0b101, 0b10000, 0b00, 0b0);
  INSN(vle32ff_v, 0b0000111, 0b110, 0b10000, 0b00, 0b0);
  INSN(vle64ff_v, 0b0000111, 0b111, 0b10000, 0b00, 0b0);

  INSN(vse8_v,  0b0100111, 0b000, 0b00000, 0b00, 0b0);
  INSN(vse16_v, 0b0100111, 0b101, 0b00000, 0b00, 0b0);
  INSN(vse32_v, 0b0100111, 0b110, 0b00000, 0b00, 0b0);
  INSN(vse64_v, 0b0100111, 0b111, 0b00000, 0b00, 0b0);

#undef INSN

#define INSN(NAME, op, width, umop, mop, mew, nf)                                               \
  void NAME(VectorRegister Vd_or_Vs3, Register Rs1, VectorMask vm = unmasked) { \
    patch_VLdSt(op, Vd_or_Vs3, width, Rs1, umop, vm, mop, mew, nf);                         \
  }

  // Vector Unit-Stride Segment Load Instructions
  INSN(vlseg3e8_v, 0b0000111, 0b000, 0b00000, 0b00, 0b0, g3);
  INSN(vlseg4e8_v, 0b0000111, 0b000, 0b00000, 0b00, 0b0, g4);

  // Vector Unit-Stride Segment Store Instructions
  INSN(vsseg3e8_v, 0b0100111, 0b000, 0b00000, 0b00, 0b0, g3);
  INSN(vsseg4e8_v, 0b0100111, 0b000, 0b00000, 0b00, 0b0, g4);

#undef INSN

#define INSN(NAME, op, width, mop, mew)                                                                  \
  void NAME(VectorRegister Vd, Register Rs1, VectorRegister Vs2, VectorMask vm = unmasked, Nf nf = g1) { \
    patch_VLdSt(op, Vd, width, Rs1, Vs2->raw_encoding(), vm, mop, mew, nf);                              \
  }

  // Vector unordered indexed load instructions
  INSN( vluxei8_v, 0b0000111, 0b000, 0b01, 0b0);
  INSN(vluxei32_v, 0b0000111, 0b110, 0b01, 0b0);
  INSN(vluxei64_v, 0b0000111, 0b111, 0b01, 0b0);

  // Vector unordered indexed store instructions
  INSN( vsuxei8_v, 0b0100111, 0b000, 0b01, 0b0);
  INSN(vsuxei32_v, 0b0100111, 0b110, 0b01, 0b0);
  INSN(vsuxei64_v, 0b0100111, 0b111, 0b01, 0b0);

#undef INSN

#define INSN(NAME, op, width, mop, mew)                                                                  \
  void NAME(VectorRegister Vd, Register Rs1, Register Rs2, VectorMask vm = unmasked, Nf nf = g1) {       \
    patch_VLdSt(op, Vd, width, Rs1, Rs2->raw_encoding(), vm, mop, mew, nf);                              \
  }

  // Vector Strided Instructions
  INSN(vlse8_v,  0b0000111, 0b000, 0b10, 0b0);
  INSN(vlse16_v, 0b0000111, 0b101, 0b10, 0b0);
  INSN(vlse32_v, 0b0000111, 0b110, 0b10, 0b0);
  INSN(vlse64_v, 0b0000111, 0b111, 0b10, 0b0);

  INSN(vsse8_v,  0b0100111, 0b000, 0b10, 0b0);
  INSN(vsse16_v, 0b0100111, 0b101, 0b10, 0b0);
  INSN(vsse32_v, 0b0100111, 0b110, 0b10, 0b0);
  INSN(vsse64_v, 0b0100111, 0b111, 0b10, 0b0);

#undef INSN
#undef patch_VLdSt

// ====================================
// RISC-V Vector Crypto Extension
// ====================================

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1, VectorMask vm = unmasked) { \
    patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Bit-manipulation used in Cryptography (Zvbb) Extension
  INSN(vandn_vv,   0b1010111, 0b000, 0b000001);
  INSN(vror_vv,    0b1010111, 0b000, 0b010100);
  INSN(vrol_vv,    0b1010111, 0b000, 0b010101);

  // Vector Bit-manipulation used in Cryptography (Zvbc) Extension
  INSN(vclmul_vv,  0b1010111, 0b010, 0b001100);
  INSN(vclmulh_vv, 0b1010111, 0b010, 0b001101);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, Register Rs1, VectorMask vm = unmasked) {       \
    patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6);                            \
  }

  // Vector Bit-manipulation used in Cryptography (Zvbb) Extension
  INSN(vandn_vx,   0b1010111, 0b100, 0b000001);
  INSN(vrol_vx,    0b1010111, 0b100, 0b010101);
  INSN(vror_vx,    0b1010111, 0b100, 0b010100);

#undef INSN

#define patch_VArith_imm6(op, Reg, funct3, Reg_or_Imm5, I5, Vs2, vm, funct6)   \
    unsigned insn = 0;                                                         \
    patch((address)&insn, 6, 0, op);                                           \
    patch((address)&insn, 14, 12, funct3);                                     \
    patch((address)&insn, 19, 15, Reg_or_Imm5);                                \
    patch((address)&insn, 25, vm);                                             \
    patch((address)&insn, 26, I5);                                             \
    patch((address)&insn, 31, 27, funct6);                                     \
    patch_reg((address)&insn, 7, Reg);                                         \
    patch_reg((address)&insn, 20, Vs2);                                        \
    emit(insn)

#define INSN(NAME, op, funct3, funct6)                                                             \
  void NAME(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {       \
    guarantee(is_uimm6(imm), "uimm is invalid");                                                   \
    patch_VArith_imm6(op, Vd, funct3, (uint32_t)(imm & 0x1f), (uint32_t)((imm >> 5) & 0x1), Vs2, vm, funct6);  \
  }

  // Vector Bit-manipulation used in Cryptography (Zvbb) Extension
  // NOTE: there is no corresponding vrol.vi supplied by the extension, but it can be emulated with vror.vi easily.
  INSN(vror_vi,    0b1010111, 0b011, 0b01010);

#undef INSN
#undef patch_VArith_imm6

#define INSN(NAME, op, funct3, Vs1, funct6)                                    \
  void NAME(VectorRegister Vd, VectorRegister Vs2, VectorMask vm = unmasked) { \
    patch_VArith(op, Vd, funct3, Vs1, Vs2, vm, funct6);                        \
  }

  // Vector Bit-manipulation used in Cryptography (Zvkb) Extension
  INSN(vbrev_v,  0b1010111, 0b010, 0b01010, 0b010010); // reverse bits in every element
  INSN(vbrev8_v, 0b1010111, 0b010, 0b01000, 0b010010); // reverse bits in every byte of element
  INSN(vrev8_v,  0b1010111, 0b010, 0b01001, 0b010010); // reverse bytes in every elememt

  // Vector AES instructions (Zvkned extension)
  INSN(vaesem_vv,   0b1110111, 0b010, 0b00010, 0b101000);
  INSN(vaesef_vv,   0b1110111, 0b010, 0b00011, 0b101000);

  INSN(vaesdm_vv,   0b1110111, 0b010, 0b00000, 0b101000);
  INSN(vaesdf_vv,   0b1110111, 0b010, 0b00001, 0b101000);

  INSN(vclz_v,  0b1010111, 0b010, 0b01100, 0b010010); // count leading zeros
  INSN(vctz_v,  0b1010111, 0b010, 0b01101, 0b010010); // count trailing zeros

#undef INSN

#define INSN(NAME, op, funct3, vm, funct6)                                   \
  void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1) {     \
    patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6);      \
  }

  // Vector SHA-2 Secure Hash (Zvknh[ab]) Extension
  INSN(vsha2ms_vv,  0b1110111, 0b010, 0b1, 0b101101);
  INSN(vsha2ch_vv,  0b1110111, 0b010, 0b1, 0b101110);
  INSN(vsha2cl_vv,  0b1110111, 0b010, 0b1, 0b101111);

  // Vector GHASH (Zvkg) Extension
  INSN(vghsh_vv,    0b1110111, 0b010, 0b1, 0b101100);

#undef INSN

#define INSN(NAME, op, funct3, Vs1, funct6)                                    \
  void NAME(VectorRegister Vd, VectorRegister Vs2, VectorMask vm = unmasked) { \
    patch_VArith(op, Vd, funct3, Vs1, Vs2, vm, funct6);                        \
  }

  // Vector Basic Bit-manipulation (Zvbb) Extension
  INSN(vcpop_v,  0b1010111, 0b010, 0b01110, 0b010010);

#undef INSN

#undef patch_VArith

// ====================================
// RISC-V Bit-Manipulation Extension
// Currently only support Zba, Zbb and Zbs bitmanip extensions.
// ====================================
#define INSN(NAME, op, funct3, funct7)                  \
  void NAME(Register Rd, Register Rs1, Register Rs2) {  \
    unsigned insn = 0;                                  \
    patch((address)&insn, 6,  0, op);                   \
    patch((address)&insn, 14, 12, funct3);              \
    patch((address)&insn, 31, 25, funct7);              \
    patch_reg((address)&insn, 7, Rd);                   \
    patch_reg((address)&insn, 15, Rs1);                 \
    patch_reg((address)&insn, 20, Rs2);                 \
    emit(insn);                                         \
  }

  INSN(add_uw,    0b0111011, 0b000, 0b0000100);
  INSN(rolr,      0b0110011, 0b001, 0b0110000);
  INSN(rolrw,     0b0111011, 0b001, 0b0110000);
  INSN(rorr,      0b0110011, 0b101, 0b0110000);
  INSN(rorrw,     0b0111011, 0b101, 0b0110000);
  INSN(sh1add,    0b0110011, 0b010, 0b0010000);
  INSN(sh2add,    0b0110011, 0b100, 0b0010000);
  INSN(sh3add,    0b0110011, 0b110, 0b0010000);
  INSN(sh1add_uw, 0b0111011, 0b010, 0b0010000);
  INSN(sh2add_uw, 0b0111011, 0b100, 0b0010000);
  INSN(sh3add_uw, 0b0111011, 0b110, 0b0010000);
  INSN(andn,      0b0110011, 0b111, 0b0100000);
  INSN(orn,       0b0110011, 0b110, 0b0100000);
  INSN(xnor,      0b0110011, 0b100, 0b0100000);
  INSN(max,       0b0110011, 0b110, 0b0000101);
  INSN(maxu,      0b0110011, 0b111, 0b0000101);
  INSN(min,       0b0110011, 0b100, 0b0000101);
  INSN(minu,      0b0110011, 0b101, 0b0000101);
  INSN(clmul,     0b0110011, 0b001, 0b0000101);
  INSN(clmulh,    0b0110011, 0b011, 0b0000101);
  INSN(clmulr,    0b0110011, 0b010, 0b0000101);

#undef INSN

#define INSN(NAME, op, funct3, funct12)                 \
  void NAME(Register Rd, Register Rs1) {                \
    unsigned insn = 0;                                  \
    patch((address)&insn, 6, 0, op);                    \
    patch((address)&insn, 14, 12, funct3);              \
    patch((address)&insn, 31, 20, funct12);             \
    patch_reg((address)&insn, 7, Rd);                   \
    patch_reg((address)&insn, 15, Rs1);                 \
    emit(insn);                                         \
  }

  INSN(brev8,  0b0010011, 0b101, 0b011010000111);
  INSN(rev8,   0b0010011, 0b101, 0b011010111000);
  INSN(_sext_b, 0b0010011, 0b001, 0b011000000100);
  INSN(_sext_h, 0b0010011, 0b001, 0b011000000101);
  INSN(_zext_h, 0b0111011, 0b100, 0b000010000000);
  INSN(clz,    0b0010011, 0b001, 0b011000000000);
  INSN(clzw,   0b0011011, 0b001, 0b011000000000);
  INSN(ctz,    0b0010011, 0b001, 0b011000000001);
  INSN(ctzw,   0b0011011, 0b001, 0b011000000001);
  INSN(cpop,   0b0010011, 0b001, 0b011000000010);
  INSN(cpopw,  0b0011011, 0b001, 0b011000000010);
  INSN(orc_b,  0b0010011, 0b101, 0b001010000111);

#undef INSN

#define INSN(NAME, op, funct3, funct6)                  \
  void NAME(Register Rd, Register Rs1, unsigned shamt) {\
    guarantee(shamt <= 0x3f, "Shamt is invalid");       \
    unsigned insn = 0;                                  \
    patch((address)&insn, 6, 0, op);                    \
    patch((address)&insn, 14, 12, funct3);              \
    patch((address)&insn, 25, 20, shamt);               \
    patch((address)&insn, 31, 26, funct6);              \
    patch_reg((address)&insn, 7, Rd);                   \
    patch_reg((address)&insn, 15, Rs1);                 \
    emit(insn);                                         \
  }

  INSN(rori,    0b0010011, 0b101, 0b011000);
  INSN(slli_uw, 0b0011011, 0b001, 0b000010);
  INSN(bexti,   0b0010011, 0b101, 0b010010);

#undef INSN

#define INSN(NAME, op, funct3, funct7)                  \
  void NAME(Register Rd, Register Rs1, unsigned shamt) {\
    guarantee(shamt <= 0x1f, "Shamt is invalid");       \
    unsigned insn = 0;                                  \
    patch((address)&insn, 6, 0, op);                    \
    patch((address)&insn, 14, 12, funct3);              \
    patch((address)&insn, 24, 20, shamt);               \
    patch((address)&insn, 31, 25, funct7);              \
    patch_reg((address)&insn, 7, Rd);                   \
    patch_reg((address)&insn, 15, Rs1);                 \
    emit(insn);                                         \
  }

  INSN(roriw, 0b0011011, 0b101, 0b0110000);

#undef INSN

// ========================================
// RISC-V Compressed Instructions Extension
// ========================================
// Note:
// 1. Assembler functions encoding 16-bit compressed instructions always begin with a 'c_'
//    prefix, such as 'c_add'. Correspondingly, assembler functions encoding normal 32-bit
//    instructions with begin with a '_' prefix, such as "_add". Most of time users have no
//    need to explicitly emit these compressed instructions. Instead, they still use unified
//    wrappers such as 'add' which do the compressing work through 'c_add' depending on the
//    the operands of the instruction and availability of the RVC hardware extension.
//
// 2. 'CompressibleScope' and 'IncompressibleScope' are introduced to mark assembler scopes
//     within which instructions are qualified or unqualified to be compressed into their 16-bit
//     versions. An example:
//
//      CompressibleScope scope(_masm);
//      __ add(...);       // this instruction will be compressed into 'c.add' when possible
//      {
//         IncompressibleScope scope(_masm);
//         __ add(...);    // this instruction will not be compressed
//         {
//            CompressibleScope scope(_masm);
//            __ add(...); // this instruction will be compressed into 'c.add' when possible
//         }
//      }
//
// 3. When printing JIT assembly code, using -XX:PrintAssemblyOptions=no-aliases could help
//    distinguish compressed 16-bit instructions from normal 32-bit ones.

private:
  bool _in_compressible_scope;
public:
  bool in_compressible_scope() const { return _in_compressible_scope; }
  void set_in_compressible_scope(bool b) { _in_compressible_scope = b; }
public:

  // An abstract compressible scope
  class AbstractCompressibleScope : public StackObj {
  protected:
    Assembler *_masm;
    bool _saved_in_compressible_scope;
  protected:
    AbstractCompressibleScope(Assembler *_masm)
    : _masm(_masm)
    , _saved_in_compressible_scope(_masm->in_compressible_scope()) {}
  };
  // A compressible scope
  class CompressibleScope : public AbstractCompressibleScope {
  public:
    CompressibleScope(Assembler *_masm) : AbstractCompressibleScope(_masm) {
      _masm->set_in_compressible_scope(true);
    }
    ~CompressibleScope() {
      _masm->set_in_compressible_scope(_saved_in_compressible_scope);
    }
  };
  // An incompressible scope
  class IncompressibleScope : public AbstractCompressibleScope {
  public:
    IncompressibleScope(Assembler *_masm) : AbstractCompressibleScope(_masm) {
      _masm->set_in_compressible_scope(false);
    }
    ~IncompressibleScope() {
      _masm->set_in_compressible_scope(_saved_in_compressible_scope);
    }
  };

public:
  // Emit a relocation.
  void relocate(RelocationHolder const& rspec, int format = 0) {
    AbstractAssembler::relocate(rspec, format);
  }
  void relocate(relocInfo::relocType rtype, int format = 0) {
    AbstractAssembler::relocate(rtype, format);
  }
  template <typename Callback>
  void relocate(RelocationHolder const& rspec, Callback emit_insts, int format = 0) {
    AbstractAssembler::relocate(rspec, format);
    IncompressibleScope scope(this); // relocations
    emit_insts();
  }
  template <typename Callback>
  void relocate(relocInfo::relocType rtype, Callback emit_insts, int format = 0) {
    AbstractAssembler::relocate(rtype, format);
    IncompressibleScope scope(this); // relocations
    emit_insts();
  }

  // patch a 16-bit instruction.
  static void c_patch(address a, unsigned msb, unsigned lsb, uint16_t val) {
    assert_cond(a != nullptr);
    assert_cond(msb >= lsb && msb <= 15);
    unsigned nbits = msb - lsb + 1;
    guarantee(val < (1U << nbits), "Field too big for insn");
    uint16_t mask = (1U << nbits) - 1;
    val <<= lsb;
    mask <<= lsb;
    uint16_t target = ld_c_instr(a);
    target &= ~mask;
    target |= val;
    sd_c_instr(a, target);
  }

  static void c_patch(address a, unsigned bit, uint16_t val) {
    c_patch(a, bit, bit, val);
  }

  // patch a 16-bit instruction with a general purpose register ranging [0, 31] (5 bits)
  static void c_patch_reg(address a, unsigned lsb, Register reg) {
    c_patch(a, lsb + 4, lsb, reg->raw_encoding());
  }

  // patch a 16-bit instruction with a general purpose register ranging [8, 15] (3 bits)
  static void c_patch_compressed_reg(address a, unsigned lsb, Register reg) {
    c_patch(a, lsb + 2, lsb, reg->compressed_raw_encoding());
  }

  // patch a 16-bit instruction with a float register ranging [0, 31] (5 bits)
  static void c_patch_reg(address a, unsigned lsb, FloatRegister reg) {
    c_patch(a, lsb + 4, lsb, reg->raw_encoding());
  }

  // patch a 16-bit instruction with a float register ranging [8, 15] (3 bits)
  static void c_patch_compressed_reg(address a, unsigned lsb, FloatRegister reg) {
    c_patch(a, lsb + 2, lsb, reg->compressed_raw_encoding());
  }

// --------------  RVC Instruction Definitions  --------------

  void c_nop() {
    c_addi(x0, 0);
  }

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd_Rs1, int64_t imm) {                                                  \
    assert_cond(is_simm6(imm));                                                              \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 6, 2, (imm & right_n_bits(5)));                                  \
    c_patch_reg((address)&insn, 7, Rd_Rs1);                                                  \
    c_patch((address)&insn, 12, 12, (imm & nth_bit(5)) >> 5);                                \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_addi,   0b000, 0b01);
  INSN(c_addiw,  0b001, 0b01);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(int64_t imm) {                                                                   \
    assert_cond(is_simm10(imm));                                                             \
    assert_cond((imm & 0b1111) == 0);                                                        \
    assert_cond(imm != 0);                                                                   \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 2, 2, (imm & nth_bit(5)) >> 5);                                  \
    c_patch((address)&insn, 4, 3, (imm & right_n_bits(9)) >> 7);                             \
    c_patch((address)&insn, 5, 5, (imm & nth_bit(6)) >> 6);                                  \
    c_patch((address)&insn, 6, 6, (imm & nth_bit(4)) >> 4);                                  \
    c_patch_reg((address)&insn, 7, sp);                                                      \
    c_patch((address)&insn, 12, 12, (imm & nth_bit(9)) >> 9);                                \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_addi16sp, 0b011, 0b01);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd, uint64_t uimm) {                                                    \
    assert_cond(is_uimm10(uimm));                                                            \
    assert_cond((uimm & 0b11) == 0);                                                         \
    assert_cond(uimm != 0);                                                                  \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch_compressed_reg((address)&insn, 2, Rd);                                           \
    c_patch((address)&insn, 5, 5, (uimm & nth_bit(3)) >> 3);                                 \
    c_patch((address)&insn, 6, 6, (uimm & nth_bit(2)) >> 2);                                 \
    c_patch((address)&insn, 10, 7, (uimm & right_n_bits(10)) >> 6);                          \
    c_patch((address)&insn, 12, 11, (uimm & right_n_bits(6)) >> 4);                          \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_addi4spn, 0b000, 0b00);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd_Rs1, uint32_t shamt) {                                               \
    assert_cond(is_uimm6(shamt));                                                            \
    assert_cond(shamt != 0);                                                                 \
    assert_cond(Rd_Rs1 != x0);                                                               \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 6, 2, (shamt & right_n_bits(5)));                                \
    c_patch_reg((address)&insn, 7, Rd_Rs1);                                                  \
    c_patch((address)&insn, 12, 12, (shamt & nth_bit(5)) >> 5);                              \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_slli, 0b000, 0b10);

#undef INSN

#define INSN(NAME, funct3, funct2, op)                                                       \
  void NAME(Register Rd_Rs1, uint32_t shamt) {                                               \
    assert_cond(is_uimm6(shamt));                                                            \
    assert_cond(shamt != 0);                                                                 \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 6, 2, (shamt & right_n_bits(5)));                                \
    c_patch_compressed_reg((address)&insn, 7, Rd_Rs1);                                       \
    c_patch((address)&insn, 11, 10, funct2);                                                 \
    c_patch((address)&insn, 12, 12, (shamt & nth_bit(5)) >> 5);                              \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_srli, 0b100, 0b00, 0b01);
  INSN(c_srai, 0b100, 0b01, 0b01);

#undef INSN

#define INSN(NAME, funct3, funct2, op)                                                       \
  void NAME(Register Rd_Rs1, int64_t imm) {                                                  \
    assert_cond(is_simm6(imm));                                                              \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 6, 2, (imm & right_n_bits(5)));                                  \
    c_patch_compressed_reg((address)&insn, 7, Rd_Rs1);                                       \
    c_patch((address)&insn, 11, 10, funct2);                                                 \
    c_patch((address)&insn, 12, 12, (imm & nth_bit(5)) >> 5);                                \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_andi, 0b100, 0b10, 0b01);

#undef INSN

#define INSN(NAME, funct6, funct2, op)                                                       \
  void NAME(Register Rd_Rs1, Register Rs2) {                                                 \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch_compressed_reg((address)&insn, 2, Rs2);                                          \
    c_patch((address)&insn, 6, 5, funct2);                                                   \
    c_patch_compressed_reg((address)&insn, 7, Rd_Rs1);                                       \
    c_patch((address)&insn, 15, 10, funct6);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_sub,  0b100011, 0b00, 0b01);
  INSN(c_xor,  0b100011, 0b01, 0b01);
  INSN(c_or,   0b100011, 0b10, 0b01);
  INSN(c_and,  0b100011, 0b11, 0b01);
  INSN(c_subw, 0b100111, 0b00, 0b01);
  INSN(c_addw, 0b100111, 0b01, 0b01);

#undef INSN

#define INSN(NAME, funct4, op)                                                               \
  void NAME(Register Rd_Rs1, Register Rs2) {                                                 \
    assert_cond(Rd_Rs1 != x0);                                                               \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch_reg((address)&insn, 2, Rs2);                                                     \
    c_patch_reg((address)&insn, 7, Rd_Rs1);                                                  \
    c_patch((address)&insn, 15, 12, funct4);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_mv,  0b1000, 0b10);
  INSN(c_add, 0b1001, 0b10);

#undef INSN

 private:
  // All calls and jumps must go via MASM.
  // Format CR, c.jr/c.jalr
  // Note C instruction can't be changed, i.e. relocation patching.
  template <uint8_t InstructionType, uint8_t FunctionType>
  void c_cr_if(Register Rs1) {
    assert_cond(Rs1 != x0);
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, FunctionType);
    c_patch_reg((address)&insn, 2, x0);
    c_patch_reg((address)&insn, 7, Rs1);
    c_patch((address)&insn, 15, 12, InstructionType);
    emit_int16(insn);
  }

  void c_jr(Register Rs1)   { c_cr_if<0b1000, 0b10>(Rs1); }
  void c_jalr(Register Rs1) { c_cr_if<0b1001, 0b10>(Rs1); }

  typedef void (Assembler::* j_c_insn)(address dest);
  typedef void (Assembler::* compare_and_branch_c_insn)(Register Rs1, address dest);

  void wrap_label(Label &L, j_c_insn insn) {
    if (L.is_bound()) {
      (this->*insn)(target(L));
    } else {
      L.add_patch_at(code(), locator());
      (this->*insn)(pc());
    }
  }

  void wrap_label(Label &L, Register r, compare_and_branch_c_insn insn) {
    if (L.is_bound()) {
      (this->*insn)(r, target(L));
    } else {
      L.add_patch_at(code(), locator());
      (this->*insn)(r, pc());
    }
  }

  // Format CJ, c.j (c.jal)
  // Note C instruction can't be changed, i.e. relocation patching.
  void c_j(int32_t offset) {
    assert(is_simm12(offset) && ((offset % 2) == 0), "invalid encoding");
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, 0b01);
    c_patch((address)&insn, 2, 2, (offset & nth_bit(5)) >> 5);
    c_patch((address)&insn, 5, 3, (offset & right_n_bits(4)) >> 1);
    c_patch((address)&insn, 6, 6, (offset & nth_bit(7)) >> 7);
    c_patch((address)&insn, 7, 7, (offset & nth_bit(6)) >> 6);
    c_patch((address)&insn, 8, 8, (offset & nth_bit(10)) >> 10);
    c_patch((address)&insn, 10, 9, (offset & right_n_bits(10)) >> 8);
    c_patch((address)&insn, 11, 11, (offset & nth_bit(4)) >> 4);
    c_patch((address)&insn, 12, 12, (offset & nth_bit(11)) >> 11);
    c_patch((address)&insn, 15, 13, 0b101);
    emit_int16(insn);
  }

  void c_j(address dest) {
    assert_cond(dest != nullptr);
    int64_t distance = dest - pc();
    assert(is_simm12(distance) && ((distance % 2) == 0), "invalid encoding");
    c_j(distance);
  }

  void c_j(Label &L) {
    wrap_label(L, &Assembler::c_j);
  }

  public:

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rs1, int32_t imm) {                                                     \
    assert(is_simm9(imm) && ((imm % 2) == 0), "invalid encoding");                           \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 2, 2, (imm & nth_bit(5)) >> 5);                                  \
    c_patch((address)&insn, 4, 3, (imm & right_n_bits(3)) >> 1);                             \
    c_patch((address)&insn, 6, 5, (imm & right_n_bits(8)) >> 6);                             \
    c_patch_compressed_reg((address)&insn, 7, Rs1);                                          \
    c_patch((address)&insn, 11, 10, (imm & right_n_bits(5)) >> 3);                           \
    c_patch((address)&insn, 12, 12, (imm & nth_bit(8)) >> 8);                                \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }                                                                                          \
  void NAME(Register Rs1, address dest) {                                                    \
    assert_cond(dest != nullptr);                                                            \
    int64_t distance = dest - pc();                                                          \
    assert(is_simm9(distance) && ((distance % 2) == 0), "invalid encoding");                 \
    NAME(Rs1, distance);                                                                     \
  }                                                                                          \
  void NAME(Register Rs1, Label &L) {                                                        \
    wrap_label(L, Rs1, &Assembler::NAME);                                                    \
  }

  INSN(c_beqz, 0b110, 0b01);
  INSN(c_bnez, 0b111, 0b01);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd, int32_t imm) {                                                      \
    assert_cond(is_simm18(imm));                                                             \
    assert_cond((imm & 0xfff) == 0);                                                         \
    assert_cond(imm != 0);                                                                   \
    assert_cond(Rd != x0 && Rd != x2);                                                       \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 6, 2, (imm & right_n_bits(17)) >> 12);                           \
    c_patch_reg((address)&insn, 7, Rd);                                                      \
    c_patch((address)&insn, 12, 12, (imm & nth_bit(17)) >> 17);                              \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_lui, 0b011, 0b01);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd, int32_t imm) {                                                      \
    assert_cond(is_simm6(imm));                                                              \
    assert_cond(Rd != x0);                                                                   \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 6, 2, (imm & right_n_bits(5)));                                  \
    c_patch_reg((address)&insn, 7, Rd);                                                      \
    c_patch((address)&insn, 12, 12, (imm & right_n_bits(6)) >> 5);                           \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_li, 0b010, 0b01);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd, uint32_t uimm) {                                                    \
    assert_cond(is_uimm9(uimm));                                                             \
    assert_cond((uimm & 0b111) == 0);                                                        \
    assert_cond(Rd != x0);                                                                   \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 4, 2, (uimm & right_n_bits(9)) >> 6);                            \
    c_patch((address)&insn, 6, 5, (uimm & right_n_bits(5)) >> 3);                            \
    c_patch_reg((address)&insn, 7, Rd);                                                      \
    c_patch((address)&insn, 12, 12, (uimm & nth_bit(5)) >> 5);                               \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_ldsp,  0b011, 0b10);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(FloatRegister Rd, uint32_t uimm) {                                               \
    assert_cond(is_uimm9(uimm));                                                             \
    assert_cond((uimm & 0b111) == 0);                                                        \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 4, 2, (uimm & right_n_bits(9)) >> 6);                            \
    c_patch((address)&insn, 6, 5, (uimm & right_n_bits(5)) >> 3);                            \
    c_patch_reg((address)&insn, 7, Rd);                                                      \
    c_patch((address)&insn, 12, 12, (uimm & nth_bit(5)) >> 5);                               \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_fldsp, 0b001, 0b10);

#undef INSN

#define INSN(NAME, funct3, op, REGISTER_TYPE)                                                \
  void NAME(REGISTER_TYPE Rd_Rs2, Register Rs1, uint32_t uimm) {                             \
    assert_cond(is_uimm8(uimm));                                                             \
    assert_cond((uimm & 0b111) == 0);                                                        \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);                                       \
    c_patch((address)&insn, 6, 5, (uimm & right_n_bits(8)) >> 6);                            \
    c_patch_compressed_reg((address)&insn, 7, Rs1);                                          \
    c_patch((address)&insn, 12, 10, (uimm & right_n_bits(6)) >> 3);                          \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_ld,  0b011, 0b00, Register);
  INSN(c_sd,  0b111, 0b00, Register);
  INSN(c_fld, 0b001, 0b00, FloatRegister);
  INSN(c_fsd, 0b101, 0b00, FloatRegister);

#undef INSN

#define INSN(NAME, funct3, op, REGISTER_TYPE)                                                \
  void NAME(REGISTER_TYPE Rs2, uint32_t uimm) {                                              \
    assert_cond(is_uimm9(uimm));                                                             \
    assert_cond((uimm & 0b111) == 0);                                                        \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch_reg((address)&insn, 2, Rs2);                                                     \
    c_patch((address)&insn, 9, 7, (uimm & right_n_bits(9)) >> 6);                            \
    c_patch((address)&insn, 12, 10, (uimm & right_n_bits(6)) >> 3);                          \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_sdsp,  0b111, 0b10, Register);
  INSN(c_fsdsp, 0b101, 0b10, FloatRegister);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rs2, uint32_t uimm) {                                                   \
    assert_cond(is_uimm8(uimm));                                                             \
    assert_cond((uimm & 0b11) == 0);                                                         \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch_reg((address)&insn, 2, Rs2);                                                     \
    c_patch((address)&insn, 8, 7, (uimm & right_n_bits(8)) >> 6);                            \
    c_patch((address)&insn, 12, 9, (uimm & right_n_bits(6)) >> 2);                           \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_swsp, 0b110, 0b10);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd, uint32_t uimm) {                                                    \
    assert_cond(is_uimm8(uimm));                                                             \
    assert_cond((uimm & 0b11) == 0);                                                         \
    assert_cond(Rd != x0);                                                                   \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 3, 2, (uimm & right_n_bits(8)) >> 6);                            \
    c_patch((address)&insn, 6, 4, (uimm & right_n_bits(5)) >> 2);                            \
    c_patch_reg((address)&insn, 7, Rd);                                                      \
    c_patch((address)&insn, 12, 12, (uimm & nth_bit(5)) >> 5);                               \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_lwsp, 0b010, 0b10);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME(Register Rd_Rs2, Register Rs1, uint32_t uimm) {                                  \
    assert_cond(is_uimm7(uimm));                                                             \
    assert_cond((uimm & 0b11) == 0);                                                         \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);                                       \
    c_patch((address)&insn, 5, 5, (uimm & nth_bit(6)) >> 6);                                 \
    c_patch((address)&insn, 6, 6, (uimm & nth_bit(2)) >> 2);                                 \
    c_patch_compressed_reg((address)&insn, 7, Rs1);                                          \
    c_patch((address)&insn, 12, 10, (uimm & right_n_bits(6)) >> 3);                          \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_lw, 0b010, 0b00);
  INSN(c_sw, 0b110, 0b00);

#undef INSN

#define INSN(NAME, funct3, op)                                                               \
  void NAME() {                                                                              \
    uint16_t insn = 0;                                                                       \
    c_patch((address)&insn, 1, 0, op);                                                       \
    c_patch((address)&insn, 11, 2, 0x0);                                                     \
    c_patch((address)&insn, 12, 12, 0b1);                                                    \
    c_patch((address)&insn, 15, 13, funct3);                                                 \
    emit_int16(insn);                                                                        \
  }

  INSN(c_ebreak, 0b100, 0b10);

#undef INSN

// --------------  RVC Transformation Functions  --------------

// --------------------------
// Register instructions
// --------------------------
#define INSN(NAME)                                                                             \
  void NAME(Register Rd, Register Rs1, Register Rs2) {                                         \
    /* add -> c.add */                                                                         \
    if (do_compress()) {                                                                       \
      Register src = noreg;                                                                    \
      if (Rs1 != x0 && Rs2 != x0 && ((src = Rs1, Rs2 == Rd) || (src = Rs2, Rs1 == Rd))) {      \
        c_add(Rd, src);                                                                        \
        return;                                                                                \
      }                                                                                        \
    }                                                                                          \
    _add(Rd, Rs1, Rs2);                                                                        \
  }

  INSN(add);

#undef INSN

// --------------------------
#define INSN(NAME, C_NAME, NORMAL_NAME)                                                      \
  void NAME(Register Rd, Register Rs1, Register Rs2) {                                       \
    /* sub/subw -> c.sub/c.subw */                                                           \
    if (do_compress() &&                                                                     \
        (Rd == Rs1 && Rd->is_compressed_valid() && Rs2->is_compressed_valid())) {            \
      C_NAME(Rd, Rs2);                                                                       \
      return;                                                                                \
    }                                                                                        \
    NORMAL_NAME(Rd, Rs1, Rs2);                                                               \
  }

  INSN(sub,  c_sub,  _sub);
  INSN(subw, c_subw, _subw);

#undef INSN

// --------------------------
#define INSN(NAME, C_NAME, NORMAL_NAME)                                                      \
  void NAME(Register Rd, Register Rs1, Register Rs2) {                                       \
    /* and/or/xor/addw -> c.and/c.or/c.xor/c.addw */                                         \
    if (do_compress()) {                                                                     \
      Register src = noreg;                                                                  \
      if (Rs1->is_compressed_valid() && Rs2->is_compressed_valid() &&                        \
        ((src = Rs1, Rs2 == Rd) || (src = Rs2, Rs1 == Rd))) {                                \
        C_NAME(Rd, src);                                                                     \
        return;                                                                              \
      }                                                                                      \
    }                                                                                        \
    NORMAL_NAME(Rd, Rs1, Rs2);                                                               \
  }

  INSN(andr, c_and,  _andr);
  INSN(orr,  c_or,   _orr);
  INSN(xorr, c_xor,  _xorr);
  INSN(addw, c_addw, _addw);

#undef INSN

private:
// some helper functions
#define FUNC(NAME, funct3, bits)                                                             \
  bool NAME(Register rs1, Register rd_rs2, int32_t imm12, bool ld) {                         \
    return rs1 == sp &&                                                                      \
      is_uimm(imm12, bits) &&                                                                \
      (intx(imm12) & funct3) == 0x0 &&                                                       \
      (!ld || rd_rs2 != x0);                                                                 \
  }                                                                                          \

  FUNC(is_c_ldsdsp,  0b111, 9);
  FUNC(is_c_lwswsp,  0b011, 8);

#undef FUNC

#define FUNC(NAME, funct3, bits)                                                             \
  bool NAME(Register rs1, int32_t imm12) {                                                   \
    return rs1 == sp &&                                                                      \
      is_uimm(imm12, bits) &&                                                                \
      (intx(imm12) & funct3) == 0x0;                                                         \
  }                                                                                          \

  FUNC(is_c_fldsdsp, 0b111, 9);

#undef FUNC

#define FUNC(NAME, REG_TYPE, funct3, bits)                                                   \
  bool NAME(Register rs1, REG_TYPE rd_rs2, int32_t imm12) {                                  \
    return rs1->is_compressed_valid() &&                                                     \
      rd_rs2->is_compressed_valid() &&                                                       \
      is_uimm(imm12, bits) &&                                                                \
      (intx(imm12) & funct3) == 0x0;                                                         \
  }                                                                                          \

  FUNC(is_c_ldsd,  Register,      0b111, 8);
  FUNC(is_c_lwsw,  Register,      0b011, 7);
  FUNC(is_c_fldsd, FloatRegister, 0b111, 8);

#undef FUNC

public:
  bool do_compress() const {
    return UseRVC && in_compressible_scope();
  }

  bool do_compress_zcb(Register reg1 = noreg, Register reg2 = noreg) const {
    return do_compress() && UseZcb &&
           (reg1 == noreg || reg1->is_compressed_valid()) && (reg2 == noreg || reg2->is_compressed_valid());
  }

  bool do_compress_zcb_zbb(Register reg1 = noreg, Register reg2 = noreg) const {
    return do_compress_zcb(reg1, reg2) && UseZbb;
  }

// --------------------------
// Load/store register
// --------------------------
  void lw(Register Rd, Register Rs, const int32_t offset) {
    /* lw -> c.lwsp/c.lw */
    if (do_compress()) {
      if (is_c_lwswsp(Rs, Rd, offset, true)) {
        c_lwsp(Rd, offset);
        return;
      } else if (is_c_lwsw(Rs, Rd, offset)) {
        c_lw(Rd, Rs, offset);
        return;
      }
    }
    _lw(Rd, Rs, offset);
  }

// --------------------------
  void ld(Register Rd, Register Rs, const int32_t offset) {
    /* ld -> c.ldsp/c.ld */
    if (do_compress()) {
      if (is_c_ldsdsp(Rs, Rd, offset, true)) {
        c_ldsp(Rd, offset);
        return;
      } else if (is_c_ldsd(Rs, Rd, offset)) {
        c_ld(Rd, Rs, offset);
        return;
      }
    }
    _ld(Rd, Rs, offset);
  }

// --------------------------
  void fld(FloatRegister Rd, Register Rs, const int32_t offset) {
    /* fld -> c.fldsp/c.fld */
    if (do_compress()) {
      if (is_c_fldsdsp(Rs, offset)) {
        c_fldsp(Rd, offset);
        return;
      } else if (is_c_fldsd(Rs, Rd, offset)) {
        c_fld(Rd, Rs, offset);
        return;
      }
    }
    _fld(Rd, Rs, offset);
  }

// --------------------------
  void sd(Register Rs2, Register Rs1, const int32_t offset) {
    /* sd -> c.sdsp/c.sd */
    if (do_compress()) {
      if (is_c_ldsdsp(Rs1, Rs2, offset, false)) {
        c_sdsp(Rs2, offset);
        return;
      } else if (is_c_ldsd(Rs1, Rs2, offset)) {
        c_sd(Rs2, Rs1, offset);
        return;
      }
    }
    _sd(Rs2, Rs1, offset);
  }

// --------------------------
  void sw(Register Rs2, Register Rs1, const int32_t offset) {
    /* sw -> c.swsp/c.sw */
    if (do_compress()) {
      if (is_c_lwswsp(Rs1, Rs2, offset, false)) {
        c_swsp(Rs2, offset);
        return;
      } else if (is_c_lwsw(Rs1, Rs2, offset)) {
        c_sw(Rs2, Rs1, offset);
        return;
      }
    }
    _sw(Rs2, Rs1, offset);
  }

// --------------------------
  void fsd(FloatRegister Rs2, Register Rs1, const int32_t offset) {
    /* fsd -> c.fsdsp/c.fsd */
    if (do_compress()) {
      if (is_c_fldsdsp(Rs1, offset)) {
        c_fsdsp(Rs2, offset);
        return;
      } else if (is_c_fldsd(Rs1, Rs2, offset)) {
        c_fsd(Rs2, Rs1, offset);
        return;
      }
    }
    _fsd(Rs2, Rs1, offset);
  }

// --------------------------
// Unconditional branch instructions
// --------------------------
 protected:
  // All calls and jumps must go via MASM. Only use x1 (aka ra) as link register for now.
  void jalr(Register Rd, Register Rs, const int32_t offset) {
    assert(Rd != x5 && Rs != x5, "Register x5 must not be used for calls/jumps.");
    /* jalr -> c.jr/c.jalr */
    if (do_compress() && (offset == 0 && Rs != x0)) {
      if (Rd == x1) {
        c_jalr(Rs);
        return;
      } else if (Rd == x0) {
        c_jr(Rs);
        return;
      }
    }
    _jalr(Rd, Rs, offset);
  }

  void jal(Register Rd, const int32_t offset) {
    assert(Rd != x5, "Register x5 must not be used for calls/jumps.");
    /* jal -> c.j, note c.jal is RV32C only */
    if (do_compress() &&
        Rd == x0 &&
        is_simm12(offset) && ((offset % 2) == 0)) {
      c_j(offset);
      return;
    }
    _jal(Rd, offset);
  }

  public:

// --------------------------
// Miscellaneous Instructions
// --------------------------
#define INSN(NAME)                                                     \
  void NAME() {                                                        \
    /* ebreak -> c.ebreak */                                           \
    if (do_compress()) {                                               \
      c_ebreak();                                                      \
      return;                                                          \
    }                                                                  \
    _ebreak();                                                         \
  }

  INSN(ebreak);

#undef INSN

// --------------------------
// Immediate Instructions
// --------------------------
#define INSN(NAME)                                                                           \
  void NAME(Register Rd, Register Rs1, int64_t imm) {                                        \
    /* addi -> c.addi/c.nop/c.mv/c.addi16sp/c.addi4spn */                                    \
    if (do_compress()) {                                                                     \
      if (Rd == Rs1 && is_simm6(imm)) {                                                      \
        c_addi(Rd, imm);                                                                     \
        return;                                                                              \
      } else if (imm == 0 && Rd != x0 && Rs1 != x0) {                                        \
        c_mv(Rd, Rs1);                                                                       \
        return;                                                                              \
      } else if (Rs1 == sp && imm != 0) {                                                    \
        if (Rd == Rs1 && (imm & 0b1111) == 0x0 && is_simm10(imm)) {                          \
          c_addi16sp(imm);                                                                   \
          return;                                                                            \
        } else if (Rd->is_compressed_valid() && (imm & 0b11) == 0x0 && is_uimm10(imm)) {     \
          c_addi4spn(Rd, imm);                                                               \
          return;                                                                            \
        }                                                                                    \
      }                                                                                      \
    }                                                                                        \
    _addi(Rd, Rs1, imm);                                                                     \
  }

  INSN(addi);

#undef INSN

// --------------------------
#define INSN(NAME)                                                                           \
  void NAME(Register Rd, Register Rs1, int64_t imm) {                                        \
    /* addiw -> c.addiw */                                                                   \
    if (do_compress() && (Rd == Rs1 && Rd != x0 && is_simm6(imm))) {                         \
      c_addiw(Rd, imm);                                                                      \
      return;                                                                                \
    }                                                                                        \
    _addiw(Rd, Rs1, imm);                                                                    \
  }

  INSN(addiw);

#undef INSN

// --------------------------
#define INSN(NAME)                                                                           \
  void NAME(Register Rd, Register Rs1, int64_t imm) {                                        \
    /* andi -> c.andi */                                                                     \
    if (do_compress() &&                                                                     \
        (Rd == Rs1 && Rd->is_compressed_valid() && is_simm6(imm))) {                         \
      c_andi(Rd, imm);                                                                       \
      return;                                                                                \
    }                                                                                        \
    _andi(Rd, Rs1, imm);                                                                     \
  }

  INSN(andi);

#undef INSN

// --------------------------
// Shift Immediate Instructions
// --------------------------
#define INSN(NAME)                                                                           \
  void NAME(Register Rd, Register Rs1, unsigned shamt) {                                     \
    /* slli -> c.slli */                                                                     \
    if (do_compress() && (Rd == Rs1 && Rd != x0 && shamt != 0)) {                            \
      c_slli(Rd, shamt);                                                                     \
      return;                                                                                \
    }                                                                                        \
    if (shamt != 0) {                                                                        \
      _slli(Rd, Rs1, shamt);                                                                 \
    } else {                                                                                 \
      if (Rd != Rs1) {                                                                       \
        addi(Rd, Rs1, 0);                                                                    \
      }                                                                                      \
    }                                                                                        \
  }

  INSN(slli);

#undef INSN

// --------------------------
#define INSN(NAME, C_NAME, NORMAL_NAME)                                                      \
  void NAME(Register Rd, Register Rs1, unsigned shamt) {                                     \
    /* srai/srli -> c.srai/c.srli */                                                         \
    if (do_compress() && (Rd == Rs1 && Rd->is_compressed_valid() && shamt != 0)) {           \
      C_NAME(Rd, shamt);                                                                     \
      return;                                                                                \
    }                                                                                        \
    if (shamt != 0) {                                                                        \
      NORMAL_NAME(Rd, Rs1, shamt);                                                           \
    } else {                                                                                 \
      if (Rd != Rs1) {                                                                       \
        addi(Rd, Rs1, 0);                                                                    \
      }                                                                                      \
    }                                                                                        \
  }

  INSN(srai, c_srai, _srai);
  INSN(srli, c_srli, _srli);

#undef INSN

// --------------------------
// Upper Immediate Instruction
// --------------------------
  void lui(Register Rd, int32_t imm) {
    /* lui -> c.lui */
    if (do_compress() && (Rd != x0 && Rd != x2 && imm != 0 && is_simm18(imm))) {
      c_lui(Rd, imm);
      return;
    }
    _lui(Rd, imm);
  }


// Cache Management Operations
// These instruction may be turned off for user space.
 private:
  enum CBO_FUNCT : unsigned int {
    CBO_INVAL = 0b0000000000000,
    CBO_CLEAN = 0b0000000000001,
    CBO_FLUSH = 0b0000000000010,
    CBO_ZERO  = 0b0000000000100
  };

  template <CBO_FUNCT FUNCT>
  void cbo_base(Register Rs1) {
    assert((UseZicbom && FUNCT != CBO_ZERO) || UseZicboz, "sanity");
    unsigned insn = 0;
    patch((address)&insn, 6,  0, 0b0001111);
    patch((address)&insn, 14, 12, 0b010);
    patch_reg((address)&insn, 15, Rs1);
    patch((address)&insn, 31, 20, FUNCT);
    emit(insn);
  }

  // This instruction have some security implication.
  // At this time it's not likely to be enabled for user mode.
  void cbo_inval(Register Rs1) { cbo_base<CBO_INVAL>(Rs1); }
 public:
  // Zicbom
  void cbo_clean(Register Rs1) { cbo_base<CBO_CLEAN>(Rs1); }
  void cbo_flush(Register Rs1) { cbo_base<CBO_FLUSH>(Rs1); }
  // Zicboz
  void cbo_zero(Register Rs1)  { cbo_base<CBO_ZERO>(Rs1); }

 private:
  enum PREFETCH_FUNCT : unsigned int {
    PREFETCH_I = 0b0000000000000,
    PREFETCH_R = 0b0000000000001,
    PREFETCH_W = 0b0000000000011
  };

  template <PREFETCH_FUNCT FUNCT>
  void prefetch_base(Register Rs1, int32_t offset) {
    assert_cond(UseZicbop);
    guarantee((offset & 0x1f) == 0, "offset lowest 5 bits must be zero");
    int32_t upperOffset = offset >> 5;
    unsigned insn = 0;
    patch((address)&insn, 6,  0, 0b0010011);
    patch((address)&insn, 14, 12, 0b110);
    patch_reg((address)&insn, 15, Rs1);
    patch((address)&insn, 24, 20, FUNCT);
    upperOffset &= 0x7f;
    patch((address)&insn, 31, 25, upperOffset);
    emit(insn);
  }

 public:
  // Zicbop
  void prefetch_i(Register Rs1, int32_t offset) { prefetch_base<PREFETCH_I>(Rs1, offset); }
  void prefetch_r(Register Rs1, int32_t offset) { prefetch_base<PREFETCH_R>(Rs1, offset); }
  void prefetch_w(Register Rs1, int32_t offset) { prefetch_base<PREFETCH_W>(Rs1, offset); }

// --------------  Zicond Instruction Definitions  --------------
// Zicond conditional operations extension
  private:
  enum CZERO_OP : unsigned int {
    CZERO_NEZ = 0b111,
    CZERO_EQZ = 0b101
  };

  template <CZERO_OP OP_VALUE>
  void czero(Register Rd, Register Rs1, Register Rs2) {
    assert_cond(UseZicond);
    uint32_t insn = 0;
    patch    ((address)&insn,  6,  0, 0b0110011);  // bits:  7, name: 0x33, attr: ['OP']
    patch_reg((address)&insn,      7, Rd);         // bits:  5, name: 'rd'
    patch    ((address)&insn, 14, 12, OP_VALUE);   // bits:  3, name: 0x7, attr: ['CZERO.NEZ'] / 0x5, attr: ['CZERO.EQZ']}
    patch_reg((address)&insn,     15, Rs1);        // bits:  5, name: 'rs1', attr: ['value']
    patch_reg((address)&insn,     20, Rs2);        // bits:  5, name: 'rs2', attr: ['condition']
    patch    ((address)&insn, 31, 25, 0b0000111);  // bits:  7, name: 0x7, attr: ['CZERO']
    emit_int32(insn);
  }

  public:
  // Moves zero to a register rd, if the condition rs2 is equal to zero, otherwise moves rs1 to rd.
  void czero_eqz(Register rd, Register rs1_value, Register rs2_condition) {
    czero<CZERO_EQZ>(rd, rs1_value, rs2_condition);
  }

  // Moves zero to a register rd, if the condition rs2 is nonzero, otherwise moves rs1 to rd.
  void czero_nez(Register rd, Register rs1_value, Register rs2_condition) {
    czero<CZERO_NEZ>(rd, rs1_value, rs2_condition);
  }

// --------------  ZCB Instruction Definitions  --------------
// Zcb additional C instructions
 private:
  // Format CLH, c.lh/c.lhu
  template <bool Unsigned>
  void c_lh_if(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
    assert_cond(uimm == 0 || uimm == 2);
    assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, 0b00);
    c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
    c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
    c_patch((address)&insn, 6, 6, Unsigned ? 0 : 1);
    c_patch_compressed_reg((address)&insn, 7, Rs1);
    c_patch((address)&insn, 12, 10, 0b001);
    c_patch((address)&insn, 15, 13, 0b100);
    emit_int16(insn);
  }

  template <bool Unsigned>
  void lh_c_mux(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
    if (do_compress_zcb(Rd_Rs2, Rs1) &&
        (uimm == 0 || uimm == 2)) {
      c_lh_if<Unsigned>(Rd_Rs2, Rs1, uimm);
    } else {
      if (Unsigned) {
        _lhu(Rd_Rs2, Rs1, uimm);
      } else {
        _lh(Rd_Rs2, Rs1, uimm);
      }
    }
  }

  // Format CU, c.[sz]ext.*, c.not
  template <uint8_t InstructionType>
  void c_u_if(Register Rs1) {
    assert_cond(do_compress_zcb(Rs1));
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, 0b01);
    c_patch((address)&insn, 4, 2, InstructionType);
    c_patch((address)&insn, 6, 5, 0b11);
    c_patch_compressed_reg((address)&insn, 7, Rs1);
    c_patch((address)&insn, 12, 10, 0b111);
    c_patch((address)&insn, 15, 13, 0b100);
    emit_int16(insn);
  }

 public:

  // Prerequisites: Zcb
  void c_lh(Register Rd_Rs2, Register Rs1, const int32_t uimm)  {  c_lh_if<false>(Rd_Rs2, Rs1, uimm); }
  void lh(Register Rd_Rs2, Register Rs1, const int32_t uimm)    { lh_c_mux<false>(Rd_Rs2, Rs1, uimm); }

  // Prerequisites: Zcb
  void c_lhu(Register Rd_Rs2, Register Rs1, const int32_t uimm) {  c_lh_if<true>(Rd_Rs2, Rs1, uimm); }
  void lhu(Register Rd_Rs2, Register Rs1, const int32_t uimm)   { lh_c_mux<true>(Rd_Rs2, Rs1, uimm); }

  // Prerequisites: Zcb
  // Format CLB, single instruction
  void c_lbu(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
    assert_cond(uimm <= 3);
    assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, 0b00);
    c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
    c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
    c_patch((address)&insn, 6, 6, (uimm & nth_bit(0)) >> 0);
    c_patch_compressed_reg((address)&insn, 7, Rs1);
    c_patch((address)&insn, 12, 10, 0b000);
    c_patch((address)&insn, 15, 13, 0b100);
    emit_int16(insn);
  }

  void lbu(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
    if (do_compress_zcb(Rd_Rs2, Rs1) &&
        uimm >= 0 && uimm <= 3) {
      c_lbu(Rd_Rs2, Rs1, uimm);
    } else {
      _lbu(Rd_Rs2, Rs1, uimm);
    }
  }

  // Prerequisites: Zcb
  // Format CSB, single instruction
  void c_sb(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
    assert_cond(uimm <= 3);
    assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, 0b00);
    c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
    c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
    c_patch((address)&insn, 6, 6, (uimm & nth_bit(0)) >> 0);
    c_patch_compressed_reg((address)&insn, 7, Rs1);
    c_patch((address)&insn, 12, 10, 0b010);
    c_patch((address)&insn, 15, 13, 0b100);
    emit_int16(insn);
  }

  void sb(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
    if (do_compress_zcb(Rd_Rs2, Rs1) &&
        uimm >= 0 && uimm <= 3) {
      c_sb(Rd_Rs2, Rs1, uimm);
    } else {
      _sb(Rd_Rs2, Rs1, uimm);
    }
  }

  // Prerequisites: Zcb
  // Format CSH, single instruction
  void c_sh(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
    assert_cond(uimm == 0 || uimm == 2);
    assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, 0b00);
    c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
    c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
    c_patch((address)&insn, 6, 6, 0);
    c_patch_compressed_reg((address)&insn, 7, Rs1);
    c_patch((address)&insn, 12, 10, 0b011);
    c_patch((address)&insn, 15, 13, 0b100);
    emit_int16(insn);
  }

  void sh(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
    if (do_compress_zcb(Rd_Rs2, Rs1) &&
        (uimm == 0 || uimm == 2)) {
      c_sh(Rd_Rs2, Rs1, uimm);
    } else {
      _sh(Rd_Rs2, Rs1, uimm);
    }
  }

  // Prerequisites: Zcb
  // Format CS
  void c_zext_b(Register Rs1) {
    assert_cond(do_compress_zcb(Rs1));
    c_u_if<0b000>(Rs1);
  }

  // Prerequisites: Zbb
  void sext_b(Register Rd_Rs2, Register Rs1) {
    assert_cond(UseZbb);
    if (do_compress_zcb_zbb(Rd_Rs2, Rs1) && (Rd_Rs2 == Rs1)) {
      c_sext_b(Rd_Rs2);
    } else {
      _sext_b(Rd_Rs2, Rs1);
    }
  }

  // Prerequisites: Zcb, Zbb
  // Format CS
  void c_sext_b(Register Rs1) {
    c_u_if<0b001>(Rs1);
  }

  // Prerequisites: Zbb
  void zext_h(Register Rd_Rs2, Register Rs1) {
    assert_cond(UseZbb);
    if (do_compress_zcb_zbb(Rd_Rs2, Rs1) && (Rd_Rs2 == Rs1)) {
      c_zext_h(Rd_Rs2);
    } else {
      _zext_h(Rd_Rs2, Rs1);
    }
  }

  // Prerequisites: Zcb, Zbb
  // Format CS
  void c_zext_h(Register Rs1) {
    c_u_if<0b010>(Rs1);
  }

  // Prerequisites: Zbb
  void sext_h(Register Rd_Rs2, Register Rs1) {
    assert_cond(UseZbb);
    if (do_compress_zcb_zbb(Rd_Rs2, Rs1) && (Rd_Rs2 == Rs1)) {
      c_sext_h(Rd_Rs2);
    } else {
      _sext_h(Rd_Rs2, Rs1);
    }
  }

  // Prerequisites: Zcb, Zbb
  // Format CS
  void c_sext_h(Register Rs1) {
    c_u_if<0b011>(Rs1);
  }

  // Prerequisites: Zcb, Zba
  // Format CS
  void c_zext_w(Register Rs1) {
    c_u_if<0b100>(Rs1);
  }

  // Prerequisites: Zcb
  // Format CS
  void c_not(Register Rs1) {
    c_u_if<0b101>(Rs1);
  }

  // Prerequisites: Zcb (M or Zmmul)
  // Format CA, c.mul
  void c_mul(Register Rd_Rs1, Register Rs2) {
    uint16_t insn = 0;
    c_patch((address)&insn, 1, 0, 0b01);
    c_patch_compressed_reg((address)&insn, 2, Rs2);
    c_patch((address)&insn, 6, 5, 0b10);
    c_patch_compressed_reg((address)&insn, 7, Rd_Rs1);
    c_patch((address)&insn, 12, 10, 0b111);
    c_patch((address)&insn, 15, 13, 0b100);
    emit_int16(insn);
  }

  void mul(Register Rd, Register Rs1, Register Rs2) {
    if (Rd != Rs1 && Rd != Rs2) {
      // Three registers needed without a mv, emit uncompressed
      _mul(Rd, Rs1, Rs2);
      return;
    }

    // Rd is either Rs1 or Rs2
    if (!do_compress_zcb(Rs2, Rs1)) {
      _mul(Rd, Rs1, Rs2);
    } else {
      if (Rd == Rs2) {
        Rs2 = Rs1;
      } else {
        assert(Rd == Rs1, "must be");
      }
      c_mul(Rd, Rs2);
    }
  }

  // Stack overflow checking
  virtual void bang_stack_with_offset(int offset) { Unimplemented(); }

  static bool is_simm5(int64_t x);
  static bool is_simm6(int64_t x);
  static bool is_simm12(int64_t x);
  static bool is_simm13(int64_t x);
  static bool is_simm18(int64_t x);
  static bool is_simm21(int64_t x);

  static bool is_uimm2(uint64_t x);
  static bool is_uimm3(uint64_t x);
  static bool is_uimm5(uint64_t x);
  static bool is_uimm6(uint64_t x);
  static bool is_uimm7(uint64_t x);
  static bool is_uimm8(uint64_t x);
  static bool is_uimm9(uint64_t x);
  static bool is_uimm10(uint64_t x);

  // The maximum range of a branch is fixed for the RISCV architecture.
  static const unsigned long branch_range = 1 * M;

  static bool reachable_from_branch_at(address branch, address target) {
    return g_uabs(target - branch) < branch_range;
  }

  // Decode the given instruction, checking if it's a 16-bit compressed
  // instruction and return the address of the next instruction.
  static address locate_next_instruction(address inst) {
    // Instruction wider than 16 bits has the two least-significant bits set.
    if ((0x3 & *inst) == 0x3) {
      return inst + instruction_size;
    } else {
      return inst + compressed_instruction_size;
    }
  }

  Assembler(CodeBuffer* code) : AbstractAssembler(code), _in_compressible_scope(true) {}
};

#endif // CPU_RISCV_ASSEMBLER_RISCV_HPP