feat(stinkytofu): per-window WMMA hide budget analysis

shared/stinkytofu/include/stinkytofu/analysis/asm/WmmaHideBudgetAnalysis.hpp

@@ -0,0 +1,134 @@
+/* ************************************************************************
+ * Copyright (C) 2025-2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * ************************************************************************ */
+#pragma once
+
+#include <cstdint>
+#include <unordered_map>
+#include <vector>
+
+#include "stinkytofu/Export.hpp"
+
+namespace stinkytofu {
+
+class PassContext;
+struct StinkyInstruction;
+
+namespace dag {
+struct RegionDAG;
+}
+
+// -------------------------------------------------------------------------
+// WMMA latency-window hide budget
+//
+// "Hiding" is what the DAG scheduler does when it parks non-WMMA work inside a matrix
+// op's latency shadow so those cycles cost nothing. The shadow is finite, and two
+// separate limits bound it:
+//
+//   * cycles -- every cycle after the op's own issue slot can take SOMETHING (SALU,
+//               memory, VALU), except the ones HwInstDesc::blockedScaleMask reserves
+//               for the hardware itself (the LD_SCALE half of a VOP3PX2/VOP3PX3 scale
+//               pair; no pipe issues there).
+//   * VALU   -- a VALU pick additionally needs a set bit in coIssueWindow, so its
+//               budget is the co-issue bits that are not also blocked. On gfx1250 this
+//               can be zero: v_wmma_scale16_* at FP4/FP4 resolves to latency 4 with
+//               coIssueWindow 0x0008, and blockedScaleMask 0x0001 blocks that very
+//               cycle.
+//
+// The budget is per window, not per region, because the work is not interchangeable: a
+// ds_load feeding WMMA 6 has to be issued before WMMA 6 whether or not a shadow has room
+// for it, while an independent SALU can wait forever. So the question each window
+// answers is "may I issue more than my slot?".
+// -------------------------------------------------------------------------
+
+/// What one matrix op's window can absorb, and what it is obliged to absorb anyway.
+struct WmmaWindowBudget {
+    StinkyInstruction* wmma = nullptr;
+    int capacityCycles = 0;  ///< issue cycles this window can hide
+    int capacityValu = 0;    ///< the VALU-capable subset of them
+    /// Issue cycles this window must take BEYOND capacityCycles. Non-zero when the work
+    /// some later WMMA depends on cannot fit in the shadow available before that WMMA, so
+    /// this window has to overrun or that WMMA is left waiting on a load. Granted to the
+    /// latest window that can still meet the deadline -- a consumer is free to spread the
+    /// same total earlier, which issues the loads sooner, but not later.
+    int extraIssue = 0;
+
+    bool mustIssuePastSlot() const {
+        return extraIssue > 0;
+    }
+};
+
+/// Summed hide budget of one scheduling region.
+struct RegionHideBudget {
+    std::vector<WmmaWindowBudget> windows;  ///< region program order
+    std::unordered_map<const StinkyInstruction*, int> windowIndex;
+
+    /// Work that must precede the FIRST WMMA. No window exists yet, so it is nobody's
+    /// overrun -- it is simply the region's prologue.
+    int prologueCycles = 0;
+    /// Work some WMMA transitively depends on (prologue included): it has a deadline.
+    int deadlinedCycles = 0;
+    /// Work no WMMA depends on. It still competes for window space at pick time, but it
+    /// can always be deferred, so it never forces a window past its slot.
+    int floatingCycles = 0;
+
+    int numWindows() const {
+        return static_cast<int>(windows.size());
+    }
+    /// How many cycles \p wmma may issue beyond its slot. 0 when it fits.
+    int extraIssueFor(const StinkyInstruction* wmma) const {
+        auto it = windowIndex.find(wmma);
+        return it == windowIndex.end() ? 0 : windows[static_cast<size_t>(it->second)].extraIssue;
+    }
+    int windowsPastSlot() const;
+    /// Windows that HAD co-issue slots and lost every one to blockedScaleMask. A
+    /// matrix op that declares no co-issue window at all is not counted -- nothing
+    /// was blocked there.
+    int windowsWithValuBlockedOut() const;
+};
+
+/// True when \p pos -- cycles elapsed since a matrix op issued -- lands on a cycle its
+/// blockedScaleMask reserves. The mask is END-anchored (bit 0 = the window's LAST cycle)
+/// so a single declaration stays correct across every per-format latency override; see
+/// HwInstDesc::blockedScaleMask. Shared with the scheduler, which asks the same question
+/// of its live window.
+///
+/// Defined inline: the scheduler calls this once per window cycle from advanceTime(),
+/// computeValuAdvanceCycles() and freeCoIssueSpace(), so it must not become a call.
+inline bool isBlockedWindowCycle(int pos, int latency, uint16_t blockedMask) {
+    if (blockedMask == 0 || pos < 0 || pos >= latency) return false;
+    const int fromEnd = latency - 1 - pos;
+    constexpr int kBlockedBits = static_cast<int>(sizeof(blockedMask) * 8);
+    return fromEnd < kBlockedBits && ((blockedMask >> fromEnd) & 1u) != 0u;
+}
+
+/// Analyse \p regionDag -- the same graph the scheduler drains -- for its per-window hide
+/// budget.
+STINKYTOFU_EXPORT RegionHideBudget analyzeWmmaHideBudget(const dag::RegionDAG& regionDag);
+
+/// Report \p budget through the optimization-remark channel: which windows are obliged to
+/// issue past their slot, and which have no VALU slot to hide anything in. A region whose
+/// work all fits says nothing. Self-gated on --remarks by emitRemark.
+STINKYTOFU_EXPORT void reportWmmaHideBudget(const PassContext& passCtx,
+                                            const RegionHideBudget& budget);
+
+}  // namespace stinkytofu

shared/stinkytofu/include/stinkytofu/core/Types.hpp

@@ -88,6 +88,16 @@ struct PassFeatureConfig {
         /// Internal tuning knob only — deliberately not surfaced as a module
         /// option, so TensileLite cannot set it.
         int mergeBarrierThreshold = 0;
+        /// Run the per-window WMMA hide-budget pre-scan (analyzeWmmaHideBudget) at the
+        /// top of each scheduling region, and report it through --remarks.
+        ///
+        /// OFF by default. The pre-scan is a pure measurement — nothing in the pick
+        /// paths gates on its verdict yet — so running it in production would be cost
+        /// for no decision. A follow-up wires the budget into the scheduler and turns
+        /// this on. Internal knob only, like mergeBarrierThreshold above: deliberately
+        /// not surfaced as a module option, so TensileLite cannot set it and only
+        /// stinkytofu-opt --enable-wmma-hide-budget-prescan reaches it.
+        bool enableWmmaHideBudgetPrescan = false;
         /// Mirrors ModuleOptions::ClusterBarrier: InsertClusterBarrierPass will run
         /// after the scheduler and plant SCC-clobbering handshakes around workgroup
         /// barriers. Enables the scheduler's cluster-barrier SCC rule and the

shared/stinkytofu/src/analysis/asm/CMakeLists.txt

@@ -4,4 +4,5 @@
 target_sources(stinkytofu_objs PRIVATE
     AsmVerifierPass.cpp
     HazardGapAnalysisPass.cpp
+    WmmaHideBudgetAnalysis.cpp
 )

shared/stinkytofu/src/analysis/asm/WmmaHideBudgetAnalysis.cpp

@@ -0,0 +1,202 @@
+/* ************************************************************************
+ * Copyright (C) 2025-2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * ************************************************************************ */
+#include "stinkytofu/analysis/asm/WmmaHideBudgetAnalysis.hpp"
+
+#include <algorithm>
+#include <sstream>
+
+#include "../../transforms/asm/dag/RegionDAG.hpp"
+#include "stinkytofu/hardware/GfxIsa.hpp"
+#include "stinkytofu/ir/asm/StinkyAsmIR.hpp"
+#include "stinkytofu/support/OptimizationRemark.hpp"
+
+namespace stinkytofu {
+namespace {
+
+// The scheduler honours blockedScaleMask unconditionally (pickOneFromWMMA), so the
+// budget reads it unconditionally too and describes the scheduler that will actually
+// run rather than a hypothetical one.
+uint16_t blockedScaleMaskOf(const StinkyInstruction& inst) {
+    const HwInstDesc* desc = inst.getHwInstDesc();
+    return desc != nullptr ? desc->blockedScaleMask : 0;
+}
+
+// Issue cycles the window of \p inst can hide work in: everything after its own issue
+// slot, less the cycles its blockedScaleMask reserves.
+int wmmaHideCapacityCycles(const StinkyInstruction& inst) {
+    const int latency = inst.latencyCycles;
+    const int issue = std::max(1, inst.issueCycles);
+    if (latency <= issue) return 0;
+    const uint16_t blocked = blockedScaleMaskOf(inst);
+    int cycles = 0;
+    for (int pos = issue; pos < latency; ++pos)
+        if (!isBlockedWindowCycle(pos, latency, blocked)) ++cycles;
+    return cycles;
+}
+
+// VALU-pipe slots the same window offers: co-issue bits that are not also blocked.
+// Reads the per-instruction coIssueWindow (matrix-format overrides already resolved),
+// the same value the pick paths gate on -- not the unresolved descriptor field.
+int wmmaHideCapacityValu(const StinkyInstruction& inst, uint16_t blocked) {
+    const int latency = inst.latencyCycles;
+    constexpr int kCoIssueBits = static_cast<int>(sizeof(inst.coIssueWindow) * 8);
+    int slots = 0;
+    for (int pos = 0; pos < latency && pos < kCoIssueBits; ++pos) {
+        if (((inst.coIssueWindow >> pos) & 1u) == 0u) continue;
+        if (!isBlockedWindowCycle(pos, latency, blocked)) ++slots;
+    }
+    return slots;
+}
+
+}  // namespace
+
+int RegionHideBudget::windowsPastSlot() const {
+    int n = 0;
+    for (const WmmaWindowBudget& w : windows)
+        if (w.mustIssuePastSlot()) ++n;
+    return n;
+}
+
+// Only windows that HAD co-issue slots and lost every one of them to blockedScaleMask.
+// A matrix op that simply declares no co-issue window (v_wmma_f32_16x16x4_f32 carries
+// coIssueWindow 0x0000) also has capacityValu == 0, but nothing was blocked there and
+// saying LD_SCALE took its slots would be false.
+int RegionHideBudget::windowsWithValuBlockedOut() const {
+    int n = 0;
+    for (const WmmaWindowBudget& w : windows) {
+        if (w.capacityValu != 0 || w.wmma == nullptr) continue;
+        if (wmmaHideCapacityValu(*w.wmma, /*blocked=*/0) > 0) ++n;
+    }
+    return n;
+}
+
+// One pass over the region DAG, no IR mutation.
+//
+// Demand counts VALU at its issueCycles even though a VALU inside a window can cost more
+// (computeValuAdvanceCycles walks to the next co-issue bit). That makes demand a
+// deliberate LOWER bound, so an overrun is only ever demanded where one is owed beyond
+// doubt. Barriers are left out -- they are not window fillers (updateWMMAStatus charges a
+// barrier its full latency), and placing them is the barrier-threshold work, not this --
+// as are pseudo nodes, which never reach an issue pipe.
+RegionHideBudget analyzeWmmaHideBudget(const dag::RegionDAG& regionDag) {
+    RegionHideBudget budget;
+    const unsigned n = static_cast<unsigned>(regionDag.nodes.size());
+    if (n == 0) return budget;
+
+    // (1) Number the matrix ops in program order. Those are the windows.
+    std::vector<int> wmmaOrder(n, -1);
+    for (unsigned i = 0; i < n; ++i) {
+        StinkyInstruction* inst = regionDag.nodes[i].inst;
+        if (!isMatrixInstruction(*inst)) continue;
+        wmmaOrder[i] = budget.numWindows();
+        budget.windowIndex[inst] = budget.numWindows();
+        budget.windows.push_back({inst, wmmaHideCapacityCycles(*inst),
+                                  wmmaHideCapacityValu(*inst, blockedScaleMaskOf(*inst)), 0});
+    }
+    const int numWindows = budget.numWindows();
+    if (numWindows == 0) return budget;
+
+    // (2) Deadline per node: the earliest WMMA that transitively depends on it, so the
+    // last window it can still hide in is deadline-1. RegionDAG ids are program indices
+    // and its edges run strictly forward, so one reverse sweep settles every node --
+    // every successor is already final by the time we read it.
+    const int kNoDeadline = numWindows;
+    std::vector<int> deadline(n, kNoDeadline);
+    for (unsigned i = n; i-- > 0;) {
+        int d = wmmaOrder[i] >= 0 ? wmmaOrder[i] : kNoDeadline;
+        for (unsigned succ : regionDag.graph[i]) d = std::min(d, deadline[succ]);
+        deadline[i] = d;
+    }
+
+    // (3) Charge the issue cycles of each filler to the deadline it inherited.
+    std::vector<int> demandAt(static_cast<size_t>(numWindows), 0);
+    for (unsigned i = 0; i < n; ++i) {
+        StinkyInstruction* inst = regionDag.nodes[i].inst;
+        if (wmmaOrder[i] >= 0 || isPseudoInst(inst) || isBarrier(*inst)) continue;
+        const int cycles = std::max(1, inst->issueCycles);
+        if (deadline[i] >= kNoDeadline) {
+            budget.floatingCycles += cycles;
+            continue;
+        }
+        budget.deadlinedCycles += cycles;
+        demandAt[static_cast<size_t>(deadline[i])] += cycles;
+    }
+    budget.prologueCycles = demandAt[0];
+
+    // (4) Walk the deadlines in order and hand each window the overrun a later WMMA
+    // forces on it. Everything due before WMMA i has only windows 0..i-1 to hide in; when
+    // that shadow runs short the shortfall is granted to window i-1, the latest one that
+    // can still meet the deadline. Prologue work (deadline 0) is excluded -- it precedes
+    // every window, so no window can be blamed for it.
+    int cumDemand = 0, cumCapacity = 0, granted = 0;
+    for (int i = 1; i < numWindows; ++i) {
+        cumDemand += demandAt[static_cast<size_t>(i)];
+        cumCapacity += budget.windows[static_cast<size_t>(i - 1)].capacityCycles;
+        const int need = cumDemand - cumCapacity - granted;
+        if (need > 0) {
+            budget.windows[static_cast<size_t>(i - 1)].extraIssue += need;
+            granted += need;
+        }
+    }
+    return budget;
+}
+
+void reportWmmaHideBudget(const PassContext& passCtx, const RegionHideBudget& budget) {
+    const char* const kRemarkPass = "StinkyDAGScheduler";
+    if (budget.numWindows() == 0) return;
+
+    if (const int pastSlot = budget.windowsPastSlot(); pastSlot > 0) {
+        // Name the windows, so a kernel author can find them, but keep the line bounded
+        // on a region with many of them.
+        constexpr int kMaxListed = 6;
+        std::ostringstream oss;
+        oss << pastSlot << " of " << budget.numWindows()
+            << " WMMA windows must issue past their slot (";
+        int listed = 0;
+        for (int i = 0; i < budget.numWindows(); ++i) {
+            const WmmaWindowBudget& w = budget.windows[static_cast<size_t>(i)];
+            if (!w.mustIssuePastSlot()) continue;
+            if (listed == kMaxListed) {
+                oss << ", ...";
+                break;
+            }
+            if (listed++ > 0) oss << ", ";
+            oss << "#" << i << " +" << w.extraIssue << " over " << w.capacityCycles;
+        }
+        oss << " issue cycles); work a later WMMA depends on does not fit the shadow "
+               "before it";
+        emitRemark(passCtx, {OptimizationRemark::Kind::Analysis, kRemarkPass, "WmmaWindowPastSlot",
+                             oss.str()});
+    }
+
+    if (const int noValu = budget.windowsWithValuBlockedOut(); noValu > 0) {
+        std::ostringstream oss;
+        oss << noValu << " of " << budget.numWindows()
+            << " WMMA windows lost every VALU co-issue slot they had to the LD_SCALE cycle "
+               "of a scale pair, so no VALU can be hidden in them at all";
+        emitRemark(passCtx, {OptimizationRemark::Kind::Analysis, kRemarkPass, "NoValuCoIssueSlot",
+                             oss.str()});
+    }
+}
+
+}  // namespace stinkytofu

shared/stinkytofu/src/transforms/asm/StinkyDAGSchedulerPass.cpp

@@ -587,7 +587,9 @@ static void scheduleRegionWithMovableSideEffects(
     // (these orderings are heuristic, not derived from real data dependencies, so
     // contradictory requests across barrier groups are possible).
     std::vector<HardSchedulingConstraint> requestedConstraints;
-    readyQueue.onInitRegion(regionStart, regionEnd, blockBegin, requestedConstraints);
+    const dag::RegionDependencies regionDeps{.dag = regionDag,
+                                             .requestedConstraints = requestedConstraints};
+    readyQueue.onInitRegion(regionStart, regionEnd, blockBegin, regionDeps);
     // Provenance only (not consulted by scheduling): which merged dagGraph edges are
     // policy-injected rather than real register dependencies, so debug output can still
     // tell them apart now that both live in the same graph.