Merge pull request #2151 from riscv-software-src/speed-up-fetch

Further speed up instruction fetch

Author: aswaterman

riscv/common.h

@@ -19,4 +19,16 @@
 # define UNUSED
 #endif
 
+#ifndef __has_builtin
+# define __has_builtin(x) 0
+#endif
+
+#if __has_cpp_attribute(assume)
+# define assume(x) [[assume(x)]]
+#elif __has_builtin(__builtin_assume)
+# define assume(x) __builtin_assume(x)
+#else
+# define assume(x) ((void) 0)
+#endif
+
 #endif

riscv/csrs.cc

@@ -773,6 +773,8 @@ bool misa_csr_t::unlogged_write(const reg_t val) noexcept {
     }
   }
 
+  proc->get_mmu()->flush_tlb();
+
   return basic_csr_t::unlogged_write(new_misa);
 }
 

riscv/decode_macros.h

@@ -225,7 +225,8 @@ static inline bool is_aligned(const unsigned val, const unsigned pos)
 #define zext_xlen(x) zext(x, xlen)
 
 #define set_pc(x) \
-  do { p->check_pc_alignment(x); \
+  do { if (unlikely((x) & ~p->pc_alignment_mask())) \
+        return p->throw_instruction_address_misaligned(x); \
        npc = sext_xlen(x); \
      } while (0)
 

riscv/execute.cc

@@ -229,19 +229,19 @@ void processor_t::step(size_t n)
     state.prv_changed = false;
     state.v_changed = false;
 
-    #define advance_pc() \
+    #define advance_pc() { \
       if (unlikely(invalid_pc(pc))) { \
         switch (pc) { \
           case PC_SERIALIZE_BEFORE: state.serialized = true; break; \
           case PC_SERIALIZE_AFTER: ++instret; break; \
           default: abort(); \
         } \
         pc = state.pc; \
-        break; \
+        goto serialize; \
       } else { \
         state.pc = pc; \
         instret++; \
-      }
+      }}
 
     try
     {
@@ -302,19 +302,21 @@ void processor_t::step(size_t n)
       else while (instret < n)
       {
         // Main simulation loop, fast path.
-        for (auto ic_entry = _mmu->access_icache(pc); ; ) {
+        for (auto ic_entry = _mmu->access_icache(pc); instret < n; instret++) {
           auto fetch = ic_entry->data;
-          pc = execute_insn_fast(this, pc, fetch);
-          ic_entry = &_mmu->icache[_mmu->icache_index(pc)];
-          if (unlikely(ic_entry->tag != pc))
-            break;
-          if (unlikely(instret + 1 == n))
-            break;
-          instret++;
-          state.pc = pc;
+          ic_entry = ic_entry->next;
+          auto new_pc = execute_insn_fast(this, pc, fetch);
+          if (unlikely(ic_entry->tag != new_pc)) {
+            ic_entry = &_mmu->icache[_mmu->icache_index(new_pc)];
+            _mmu->icache[_mmu->icache_index(pc)].next = ic_entry;
+            if (ic_entry->tag != new_pc) {
+              pc = new_pc;
+              advance_pc();
+              break;
+            }
+          }
+          state.pc = pc = ic_entry->tag;
         }
-
-        advance_pc();
       }
     }
     catch(trap_t& t)
@@ -361,6 +363,7 @@ void processor_t::step(size_t n)
       in_wfi = true;
     }
 
+serialize:
     state.minstret->bump((state.mcountinhibit->read() & MCOUNTINHIBIT_IR) ? 0 : instret);
 
     // Model a hart whose CPI is 1.

riscv/insn_template.cc

@@ -6,7 +6,8 @@
 #define DECODE_MACRO_USAGE_LOGGED 0
 
 #define PROLOGUE \
-  reg_t npc = sext_xlen(pc + insn_length(OPCODE))
+  reg_t npc = sext_xlen(pc + insn_length(OPCODE)); \
+  if (!p->extension_enabled(EXT_ZCA)) assume(insn_length(OPCODE) % 4 == 0)
 
 #define EPILOGUE \
   trace_opcode(p, OPCODE, insn); \

riscv/insns/c_add.h

@@ -1,3 +1,2 @@
 require_extension(EXT_ZCA);
-require(insn.rvc_rs2() != 0);
 WRITE_RD(sext_xlen(RVC_RS1 + RVC_RS2));

riscv/insns/c_jalr.h

@@ -1,5 +1,4 @@
 require_extension(EXT_ZCA);
-require(insn.rvc_rs1() != 0);
 reg_t tmp = npc;
 set_pc(RVC_RS1 & ~reg_t(1));
 WRITE_REG(X_RA, tmp);

riscv/insns/c_mv.h

@@ -1,3 +1,2 @@
 require_extension(EXT_ZCA);
-require(insn.rvc_rs2() != 0);
 WRITE_RD(RVC_RS2);

riscv/mmu.cc

@@ -39,6 +39,7 @@ void mmu_t::flush_tlb()
   memset(tlb_insn, -1, sizeof(tlb_insn));
   memset(tlb_load, -1, sizeof(tlb_load));
   memset(tlb_store, -1, sizeof(tlb_store));
+  memset(pte_cache, -1, sizeof(pte_cache));
 
   flush_icache();
 }
@@ -106,7 +107,9 @@ mmu_t::insn_parcel_t mmu_t::fetch_slow_path(reg_t vaddr)
 
   auto [tlb_hit, host_addr, paddr] = access_tlb(tlb_insn, vaddr, TLB_FLAGS);
   auto access_info = generate_access_info(vaddr, FETCH, {});
-  check_triggers(triggers::OPERATION_EXECUTE, vaddr, access_info.effective_virt);
+
+  if (check_triggers_fetch)
+    check_triggers(triggers::OPERATION_EXECUTE, vaddr, access_info.effective_virt);
 
   if (!tlb_hit) {
     paddr = translate(access_info, sizeof(insn_parcel_t));
@@ -125,7 +128,8 @@ mmu_t::insn_parcel_t mmu_t::fetch_slow_path(reg_t vaddr)
 
   auto res = perform_intrapage_fetch(vaddr, host_addr, paddr);
 
-  check_triggers(triggers::OPERATION_EXECUTE, vaddr, access_info.effective_virt, from_le(res));
+  if (check_triggers_fetch)
+    check_triggers(triggers::OPERATION_EXECUTE, vaddr, access_info.effective_virt, from_le(res));
 
   return res;
 }
@@ -248,16 +252,17 @@ void mmu_t::load_slow_path_intrapage(reg_t len, uint8_t* bytes, mem_access_info_
 {
   reg_t vaddr = access_info.vaddr;
   auto [tlb_hit, host_addr, paddr] = access_tlb(tlb_load, vaddr, TLB_FLAGS);
-  if (!tlb_hit || access_info.flags.is_special_access()) {
+  bool special = access_info.flags.is_special_access() && !access_info.flags.lr;
+  if (!tlb_hit || special) {
     paddr = translate(access_info, len);
     host_addr = (uintptr_t)sim->addr_to_mem(paddr);
 
-    if (!access_info.flags.is_special_access())
+    if (!special)
       refill_tlb(vaddr, paddr, (char*)host_addr, LOAD);
+  }
 
-    if (access_info.flags.lr && !sim->reservable(paddr)) {
-      throw trap_load_access_fault(access_info.effective_virt, access_info.transformed_vaddr, 0, 0);
-    }
+  if (access_info.flags.lr && !sim->reservable(paddr)) {
+    throw trap_load_access_fault(access_info.effective_virt, access_info.transformed_vaddr, 0, 0);
   }
 
   perform_intrapage_load(vaddr, host_addr, paddr, len, bytes, access_info.flags);
@@ -282,7 +287,9 @@ void mmu_t::load_slow_path(reg_t original_addr, reg_t len, uint8_t* bytes, xlate
 
   auto access_info = generate_access_info(original_addr, LOAD, xlate_flags);
   reg_t transformed_addr = access_info.transformed_vaddr;
-  check_triggers(triggers::OPERATION_LOAD, transformed_addr, access_info.effective_virt);
+
+  if (check_triggers_load)
+    check_triggers(triggers::OPERATION_LOAD, transformed_addr, access_info.effective_virt);
 
   if ((transformed_addr & (len - 1)) == 0) {
     load_slow_path_intrapage(len, bytes, access_info);
@@ -302,12 +309,14 @@ void mmu_t::load_slow_path(reg_t original_addr, reg_t len, uint8_t* bytes, xlate
     }
   }
 
-  while (len > sizeof(reg_t)) {
-    check_triggers(triggers::OPERATION_LOAD, transformed_addr, access_info.effective_virt, reg_from_bytes(sizeof(reg_t), bytes));
-    len -= sizeof(reg_t);
-    bytes += sizeof(reg_t);
+  if (check_triggers_load) {
+    while (len > sizeof(reg_t)) {
+        check_triggers(triggers::OPERATION_LOAD, transformed_addr, access_info.effective_virt, reg_from_bytes(sizeof(reg_t), bytes));
+      len -= sizeof(reg_t);
+      bytes += sizeof(reg_t);
+    }
+    check_triggers(triggers::OPERATION_LOAD, transformed_addr, access_info.effective_virt, reg_from_bytes(len, bytes));
   }
-  check_triggers(triggers::OPERATION_LOAD, transformed_addr, access_info.effective_virt, reg_from_bytes(len, bytes));
 
   if (proc && unlikely(proc->get_log_commits_enabled()))
     proc->state.log_mem_read.push_back(std::make_tuple(original_addr, 0, len));
@@ -367,7 +376,8 @@ void mmu_t::store_slow_path(reg_t original_addr, reg_t len, const uint8_t* bytes
 
   auto access_info = generate_access_info(original_addr, STORE, xlate_flags);
   reg_t transformed_addr = access_info.transformed_vaddr;
-  if (actually_store) {
+
+  if (actually_store && check_triggers_store) {
     reg_t trig_len = len;
     const uint8_t* trig_bytes = bytes;
     while (trig_len > sizeof(reg_t)) {

riscv/mmu.h

@@ -43,6 +43,7 @@ struct insn_fetch_t
 
 struct icache_entry_t {
   reg_t tag;
+  icache_entry_t* next;
   insn_fetch_t data;
 };
 
@@ -56,6 +57,11 @@ struct dtlb_entry_t {
   reg_t tag;
 };
 
+struct pte_cache_entry_t {
+  reg_t paddr;
+  reg_t pte;
+};
+
 struct xlate_flags_t {
   const bool forced_virt : 1 {false};
   const bool hlvx : 1 {false};
@@ -129,7 +135,7 @@ class mmu_t
   T ss_load(reg_t addr) {
     if ((addr & (sizeof(T) - 1)) != 0)
       throw trap_store_access_fault((proc) ? proc->state.v : false, addr, 0, 0);
-    return load<T>(addr, {.forced_virt=false, .hlvx=false, .lr=false, .ss_access=true});
+    return load<T>(addr, {.ss_access=true});
   }
 
   template<typename T>
@@ -156,7 +162,7 @@ class mmu_t
   void ss_store(reg_t addr, T val) {
     if ((addr & (sizeof(T) - 1)) != 0)
       throw trap_store_access_fault((proc) ? proc->state.v : false, addr, 0, 0);
-    store<T>(addr, val, {.forced_virt=false, .hlvx=false, .lr=false, .ss_access=true});
+    store<T>(addr, val, {.ss_access=true});
   }
 
   // AMO/Zicbom faults should be reported as store faults
@@ -188,13 +194,9 @@ class mmu_t
   // for shadow stack amoswap
   template<typename T>
   T ssamoswap(reg_t addr, reg_t value) {
-      bool forced_virt = false;
-      bool hlvx = false;
-      bool lr = false;
-      bool ss_access = true;
-      store_slow_path(addr, sizeof(T), nullptr, {forced_virt, hlvx, lr, ss_access}, false, true);
-      auto data = load<T>(addr, {forced_virt, hlvx, lr, ss_access});
-      store<T>(addr, value, {forced_virt, hlvx, lr, ss_access});
+      store_slow_path(addr, sizeof(T), nullptr, {.ss_access=true}, false, true);
+      auto data = load<T>(addr, {.ss_access=true});
+      store<T>(addr, value, {.ss_access=true});
       return data;
   }
 
@@ -272,7 +274,10 @@ class mmu_t
       store_slow_path(vaddr, size, nullptr, {}, false, true);
     }
 
-    reg_t paddr = translate(generate_access_info(vaddr, STORE, {}), 1);
+    auto [tlb_hit, host_addr, paddr] = access_tlb(tlb_store, vaddr);
+    if (!tlb_hit)
+      paddr = translate(generate_access_info(vaddr, STORE, {}), 1);
+
     if (sim->reservable(paddr))
       return load_reservation_address == paddr;
     else
@@ -321,6 +326,7 @@ class mmu_t
 
     insn_fetch_t fetch = {proc->decode_insn(insn), insn};
     entry->tag = addr;
+    entry->next = &icache[icache_index(addr + length)];
     entry->data = fetch;
 
     auto [check_tracer, _, paddr] = access_tlb(tlb_insn, addr, TLB_FLAGS, TLB_CHECK_TRACER);
@@ -407,6 +413,9 @@ class mmu_t
   dtlb_entry_t tlb_store[TLB_ENTRIES];
   dtlb_entry_t tlb_insn[TLB_ENTRIES];
 
+  static const reg_t PTE_CACHE_ENTRIES = 251;
+  pte_cache_entry_t pte_cache[PTE_CACHE_ENTRIES];
+
   typedef bloom_filter_t<reg_t, simple_hash1, simple_hash2, TLB_ENTRIES * 16, 3> reverse_tags_t;
   reverse_tags_t tlb_store_reverse_tags;
   reverse_tags_t tlb_insn_reverse_tags;
@@ -463,6 +472,9 @@ class mmu_t
 
   template<typename T> inline reg_t pte_load(reg_t pte_paddr, reg_t addr, bool virt, access_type trap_type)
   {
+    if (auto [hit, pte] = pte_cache_access(pte_paddr); hit)
+      return pte;
+
     const size_t ptesize = sizeof(T);
 
     if (!pmp_ok(pte_paddr, ptesize, LOAD, PRV_S, false))
@@ -475,7 +487,10 @@ class mmu_t
     } else if (!mmio_load(pte_paddr, ptesize, (uint8_t*)&target_pte)) {
       throw_access_exception(virt, addr, trap_type);
     }
-    return from_target(target_pte);
+
+    auto res = from_target(target_pte);
+    pte_cache_insert(pte_paddr, res);
+    return res;
   }
 
   template<typename T> inline void pte_store(reg_t pte_paddr, reg_t new_pte, reg_t addr, bool virt, access_type trap_type)
@@ -492,6 +507,20 @@ class mmu_t
     } else if (!mmio_store(pte_paddr, ptesize, (uint8_t*)&target_pte)) {
       throw_access_exception(virt, addr, trap_type);
     }
+
+    pte_cache_insert(pte_paddr, new_pte);
+  }
+
+  std::tuple<bool, reg_t> pte_cache_access(reg_t key)
+  {
+    auto e = pte_cache[key % PTE_CACHE_ENTRIES];
+    return std::make_tuple(e.paddr == key, e.pte);
+  }
+
+  void pte_cache_insert(reg_t key, reg_t value)
+  {
+    if (value & PTE_V)
+      pte_cache[key % PTE_CACHE_ENTRIES] = {key, value};
   }
 
   inline insn_parcel_t fetch_insn_parcel(reg_t addr) {