Skip to content

Commit e3c5793

Browse files
krishnamtibrewalakrishnamtibrewala
committed
[AIEX] Reschedule multi-slot instruction for better packing/schedule
1 parent 5e2fb96 commit e3c5793

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1205
-970
lines changed

Diff for: ‎llvm/lib/Target/AIE/AIEAlternateDescriptors.h

+8-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222

2323
namespace llvm {
2424

25+
using MutateInstructionMap =
26+
std::unordered_map<MachineInstr *,
27+
std::pair<MachineInstr *, const MCInstrDesc *>>;
2528
using MIAltDescsMap = std::unordered_map<MachineInstr *, const MCInstrDesc *>;
2629

2730
class AIEAlternateDescriptors {
@@ -40,7 +43,11 @@ class AIEAlternateDescriptors {
4043
const AIEBaseSubtarget &STI = AIEBaseSubtarget::get(*MI->getMF());
4144
const AIEBaseInstrInfo *TII = STI.getInstrInfo();
4245

43-
AlternateDescs[MI] = &TII->get(AltInstOpcode);
46+
setAlternateDescriptor(MI, &TII->get(AltInstOpcode));
47+
}
48+
49+
void setAlternateDescriptor(MachineInstr *MI, const MCInstrDesc *AltDesc) {
50+
AlternateDescs[MI] = AltDesc;
4451
}
4552

4653
// Return the alternate descriptor for the given multi-opcode instruction.

Diff for: ‎llvm/lib/Target/AIE/AIEHazardRecognizer.cpp

+35-1
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,14 @@ auto toHazardType(bool Conflict) {
456456
}
457457
} // namespace
458458

459+
ScheduleHazardRecognizer::HazardType AIEHazardRecognizer::getHazardType(
460+
const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
461+
iterator_range<const MachineOperand *> MIOperands,
462+
const MachineRegisterInfo &MRI, int DeltaCycles) {
463+
return getHazardType(Scoreboard, Desc, MemoryBanks, MIOperands, MRI,
464+
DeltaCycles);
465+
}
466+
459467
// These functions interpret the itinerary, translating InstrStages
460468
// to ResourceCycles to apply.
461469
// We deviate from the standard ScoreboardHazardRecognizer by not
@@ -479,10 +487,28 @@ ScheduleHazardRecognizer::HazardType AIEHazardRecognizer::getHazardType(
479487
FUDepthLimit));
480488
}
481489

490+
ConflictTypeBits AIEHazardRecognizer::checkConflict(MachineInstr &MI,
491+
int DeltaCycles) {
492+
assert(!TII->getFormatInterface()->getAlternateInstsOpcode(MI.getOpcode()));
493+
return checkConflict(Scoreboard, MI, DeltaCycles);
494+
}
495+
482496
ConflictTypeBits AIEHazardRecognizer::checkConflict(
483497
const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, MachineInstr &MI,
484498
int DeltaCycles) const {
485-
const MCInstrDesc &Desc = MI.getDesc();
499+
assert(!TII->getFormatInterface()->getAlternateInstsOpcode(MI.getOpcode()));
500+
return checkConflict(Scoreboard, MI, MI.getDesc(), DeltaCycles);
501+
}
502+
503+
ConflictTypeBits AIEHazardRecognizer::checkConflict(MachineInstr &MI,
504+
const MCInstrDesc &Desc,
505+
int DeltaCycles) {
506+
return checkConflict(Scoreboard, MI, Desc, DeltaCycles);
507+
}
508+
509+
ConflictTypeBits AIEHazardRecognizer::checkConflict(
510+
const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, MachineInstr &MI,
511+
const MCInstrDesc &Desc, int DeltaCycles) const {
486512
const unsigned SchedClass =
487513
TII->getSchedClass(Desc, MI.operands(), MI.getMF()->getRegInfo());
488514
const MemoryBankBits MemoryBanks = getMemoryBanks(&MI);
@@ -600,6 +626,14 @@ void AIEHazardRecognizer::emitInScoreboard(
600626
TII->getMemoryCycles(SchedClass), DeltaCycles, FUDepthLimit);
601627
}
602628

629+
void AIEHazardRecognizer::releaseFromScoreboard(
630+
const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
631+
iterator_range<const MachineOperand *> MIOperands,
632+
const MachineRegisterInfo &MRI, int DeltaCycles) {
633+
releaseFromScoreboard(Scoreboard, Desc, MemoryBanks, MIOperands, MRI,
634+
DeltaCycles);
635+
}
636+
603637
void AIEHazardRecognizer::releaseFromScoreboard(
604638
ResourceScoreboard<FuncUnitWrapper> &TheScoreboard, const MCInstrDesc &Desc,
605639
MemoryBankBits MemoryBanks,

Diff for: ‎llvm/lib/Target/AIE/AIEHazardRecognizer.h

+19-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
namespace llvm {
3030

3131
class MachineInstr;
32-
using ConflictTypeBits = uint64_t;
32+
using ConflictTypeBits = std::uint32_t;
3333

3434
void applyFormatOrdering(AIE::MachineBundle &Bundle, const VLIWFormat &Format,
3535
MachineInstr *BundleRoot,
@@ -185,6 +185,11 @@ class AIEHazardRecognizer : public ScheduleHazardRecognizer {
185185
iterator_range<const MachineOperand *> MIOperands,
186186
const MachineRegisterInfo &MRI,
187187
int DeltaCycles) const;
188+
// Apply the above function to the local scoreboard.
189+
void releaseFromScoreboard(const MCInstrDesc &Desc,
190+
MemoryBankBits MemoryBanks,
191+
iterator_range<const MachineOperand *> MIOperands,
192+
const MachineRegisterInfo &MRI, int DeltaCycles);
188193

189194
/// Block all scoreboard resources at DeltaCycles
190195
void blockCycleInScoreboard(int DeltaCycle);
@@ -226,9 +231,22 @@ class AIEHazardRecognizer : public ScheduleHazardRecognizer {
226231
const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
227232
iterator_range<const MachineOperand *> MIOperands,
228233
const MachineRegisterInfo &MRI, int DeltaCycles) const;
234+
ScheduleHazardRecognizer::HazardType
235+
getHazardType(const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
236+
iterator_range<const MachineOperand *> MIOperands,
237+
const MachineRegisterInfo &MRI, int DeltaCycles);
238+
239+
ConflictTypeBits
240+
checkConflict(const ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
241+
MachineInstr &MI, const MCInstrDesc &Desc,
242+
int DeltaCycles) const;
243+
ConflictTypeBits checkConflict(MachineInstr &MI, const MCInstrDesc &Desc,
244+
int DeltaCycles);
245+
229246
ConflictTypeBits
230247
checkConflict(const ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
231248
MachineInstr &MI, int DeltaCycles) const;
249+
ConflictTypeBits checkConflict(MachineInstr &MI, int DeltaCycles);
232250

233251
protected:
234252
ScheduleHazardRecognizer::HazardType getHazardType(const MCInstrDesc &Desc,

Diff for: ‎llvm/lib/Target/AIE/AIEMachineScheduler.cpp

+196-1
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,13 @@ static cl::opt<bool> PreSchedFollowsSkipPipeliner(
9494
"aie-presched-follows-skip-pipeliner", cl::init(true),
9595
cl::desc("Don't run the prescheduler if the pipeliner is skipped"));
9696

97+
/// This option enables instruction mutuation to shift a multislot instruction
98+
/// in event of a slot conflict.
99+
static cl::opt<bool> InstructionMutation(
100+
"aie-instruction-mutation", cl::init(true),
101+
cl::desc("Allow instruction mutation to shift a multislot "
102+
"instruction in event of a slot conflict"));
103+
97104
namespace {
98105
// A sentinel value to represent an unknown SUnit.
99106
const constexpr unsigned UnknownSUNum = ~0;
@@ -643,6 +650,168 @@ bool AIEPostRASchedStrategy::isFreeSU(const SUnit &SU) const {
643650
SU.NodeNum < FirstBotFixedSU.value_or(NumUpperBound);
644651
}
645652

653+
static bool checkSlotConflict(const unsigned OpCodeA, const unsigned OpCodeB,
654+
const AIEBaseMCFormats &Formats) {
655+
656+
const MCSlotKind SlotKindA = Formats.getSlotKind(OpCodeA);
657+
const MCSlotKind SlotKindB = Formats.getSlotKind(OpCodeB);
658+
const MCSlotKind UnknownSlot = MCSlotKind();
659+
660+
if (SlotKindA == UnknownSlot || SlotKindB == UnknownSlot)
661+
return false;
662+
663+
const auto *SlotInfoA = Formats.getSlotInfo(SlotKindA);
664+
const auto *SlotInfoB = Formats.getSlotInfo(SlotKindB);
665+
666+
return SlotInfoA->getSlotSet() & SlotInfoB->getSlotSet();
667+
}
668+
669+
// Check if moving a scheduled multi-slot instruction to a different slot allows
670+
// us to schedule SU in the same DeltaCycle. If a multi-slot instruction
671+
// candidate is identified, it is preserved in a map so that when SU is
672+
// scheduled, the candidate is first moved to a new slot and then the SU is
673+
// emitted.
674+
bool AIEPostRASchedStrategy::canShiftSlot(SUnit &SU, SchedBoundary &Zone,
675+
const int DeltaCycle) {
676+
677+
if (!InstructionMutation)
678+
return false;
679+
680+
const ScheduleDAGMI &DAG = *Zone.DAG;
681+
const AIEBaseInstrInfo &TII = *getTII(DAG);
682+
const AIEBaseMCFormats &Formats = *TII.getFormatInterface();
683+
AIEHazardRecognizer &HR = *getAIEHazardRecognizer(Zone);
684+
MachineInstr *NewMI = SU.getInstr();
685+
std::vector<MachineInstr *> ScheduledMultiSlotInsts;
686+
bool CanShiftSlot = false;
687+
688+
// Find and cache if there are any multi-slot instructions scheduled in the
689+
// same delta cycle
690+
for (MachineInstr &MI : DAG) {
691+
SUnit *ZoneSU = DAG.getSUnit(&MI);
692+
if (!ZoneSU)
693+
continue;
694+
if (!ZoneSU->isScheduled)
695+
continue;
696+
697+
const int CurrCycle = Zone.getCurrCycle();
698+
if (ZoneSU->BotReadyCycle !=
699+
static_cast<unsigned int>(CurrCycle - DeltaCycle))
700+
continue;
701+
702+
// Check for a MultiSlot instruction scheduled in the same DeltaCycle,
703+
// we focus on multi-slot because they can be scheduled in different
704+
// slots
705+
auto AltOpcodes = Formats.getAlternateInstsOpcode(MI.getOpcode());
706+
if (!AltOpcodes)
707+
continue;
708+
ScheduledMultiSlotInsts.push_back(&MI);
709+
}
710+
711+
// If there are no multi-slot instructions scheduled in the same DeltaCycle we
712+
// cannot shift any instruction to a different slot.
713+
if (ScheduledMultiSlotInsts.empty())
714+
return false;
715+
716+
auto DefaultOpcode = std::vector<unsigned int>{SU.getInstr()->getOpcode()};
717+
const std::vector<unsigned int> *NewMIAltOpcodes =
718+
Formats.getAlternateInstsOpcode(SU.getInstr()->getOpcode())
719+
? Formats.getAlternateInstsOpcode(SU.getInstr()->getOpcode())
720+
: &DefaultOpcode;
721+
722+
for (const unsigned int NewMIAltOpcode : *NewMIAltOpcodes) {
723+
const MCInstrDesc &NewMIAltDesc = TII.get(NewMIAltOpcode);
724+
if (!(HR.checkConflict(*NewMI, NewMIAltDesc, DeltaCycle) &
725+
static_cast<uint32_t>(AIEHazardRecognizer::ConflictType::Format)))
726+
continue;
727+
728+
for (MachineInstr *MI : ScheduledMultiSlotInsts) {
729+
SUnit *ZoneSU = DAG.getSUnit(MI);
730+
const int CurrCycle = Zone.getCurrCycle();
731+
auto AltOpcodes = Formats.getAlternateInstsOpcode(MI->getOpcode());
732+
733+
// Check if the scheduled multi-slot instruction has a slot conflict
734+
// with the new instruction, if so we might have the possiblity to shift
735+
// the multi-slot and schedule the new instruction.
736+
if (!checkSlotConflict(HR.getSelectedAltDescs().getOpcode(MI),
737+
NewMIAltOpcode, Formats))
738+
continue;
739+
740+
// Release the multi-slot instruction from the scoreboard to check if
741+
// any other alternate opcode in presence of the new instruction will
742+
// not create a hazard.
743+
HR.releaseFromScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
744+
HR.getMemoryBanks(MI), MI->operands(),
745+
MI->getMF()->getRegInfo(),
746+
CurrCycle - ZoneSU->BotReadyCycle);
747+
748+
// Check if the new instruction can be scheduled after unscheduling
749+
// the conflicting multi-slot instruction.
750+
if (HR.getHazardType(NewMIAltDesc, HR.getMemoryBanks(NewMI),
751+
NewMI->operands(), NewMI->getMF()->getRegInfo(),
752+
DeltaCycle) !=
753+
ScheduleHazardRecognizer::HazardType::NoHazard) {
754+
// If the new instruction cannot be scheduled after unscheduling the
755+
// mulit-slot revert back the state of scoreboard to original state
756+
// and continue.
757+
758+
HR.emitInScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
759+
HR.getMemoryBanks(MI), MI->operands(),
760+
MI->getMF()->getRegInfo(),
761+
CurrCycle - ZoneSU->BotReadyCycle);
762+
continue;
763+
}
764+
765+
// Emit the new instruction in the scoreboard. This will help us
766+
// to check if the previously unscheduled multi-slot instruction
767+
// can be scheduled in the same cycle, with an alternate opcode.
768+
HR.emitInScoreboard(NewMIAltDesc, HR.getMemoryBanks(NewMI),
769+
NewMI->operands(), NewMI->getMF()->getRegInfo(),
770+
DeltaCycle);
771+
772+
// Check if the previously unscheduled multi-slot instruction
773+
// can be rescheduled in presense of the new instruction in the
774+
// same cycle, with a different opcode.
775+
for (const auto AltOpcodeInside : *AltOpcodes) {
776+
const MCInstrDesc &Desc = TII.get(AltOpcodeInside);
777+
if (HR.getHazardType(Desc, HR.getMemoryBanks(MI), MI->operands(),
778+
MI->getMF()->getRegInfo(), DeltaCycle) ==
779+
ScheduleHazardRecognizer::HazardType::NoHazard) {
780+
// Cache the information to mutate the instruction during bumpNode()
781+
MutateInstruction.insert(
782+
std::make_pair(NewMI, std::make_pair(MI, &Desc)));
783+
784+
// if the new instruction was a multi-slot instruction and it failed
785+
// the general check for isAvailabeNode() this means we have not set
786+
// the selected opcode for the instruction. Set the selected opcode
787+
// for the instruction.
788+
if (NewMIAltOpcodes->size() > 1)
789+
HR.getSelectedAltDescs().setAlternateDescriptor(NewMI,
790+
&NewMIAltDesc);
791+
792+
CanShiftSlot = true;
793+
break;
794+
}
795+
}
796+
797+
// Revert back the state of scoreboard to original state.
798+
HR.releaseFromScoreboard(NewMIAltDesc, HR.getMemoryBanks(NewMI),
799+
NewMI->operands(), NewMI->getMF()->getRegInfo(),
800+
DeltaCycle);
801+
HR.emitInScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
802+
HR.getMemoryBanks(MI), MI->operands(),
803+
MI->getMF()->getRegInfo(),
804+
CurrCycle - ZoneSU->BotReadyCycle);
805+
806+
if (CanShiftSlot)
807+
break;
808+
}
809+
if (CanShiftSlot)
810+
break;
811+
}
812+
return CanShiftSlot;
813+
}
814+
646815
bool AIEPostRASchedStrategy::isAvailableNode(SUnit &SU, SchedBoundary &Zone,
647816
bool /*VerifyReadyCycle*/) {
648817
// Note we use signed integers to avoid wrap-around behavior.
@@ -692,7 +861,8 @@ bool AIEPostRASchedStrategy::isAvailableNode(SUnit &SU, SchedBoundary &Zone,
692861
// ReadyCycle is always greater or equal to the current cycle,
693862
// so DeltaCycles will always be less or equal to 0.
694863
if (Zone.checkHazard(&SU, DeltaCycles))
695-
continue;
864+
if (!canShiftSlot(SU, Zone, DeltaCycles))
865+
continue;
696866
SU.BotReadyCycle = CurrCycle - DeltaCycles;
697867
return true;
698868
}
@@ -711,10 +881,35 @@ void AIEPostRASchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
711881
assert(DeltaCycles <= 0);
712882
Top.bumpNode(SU, DeltaCycles);
713883
} else {
884+
AIEHazardRecognizer &HR = *getAIEHazardRecognizer(Bot);
714885
int DeltaCycles = int(Bot.getCurrCycle()) - int(SU->BotReadyCycle);
715886
assert(DeltaCycles <= 0);
887+
888+
// Check if an instruction needs to be moved to a different slot for the
889+
// current SU to be scheduled in the DeltaCycles.
890+
if (MutateInstruction.find(SU->getInstr()) != MutateInstruction.end()) {
891+
auto &[MI, Desc] = MutateInstruction[SU->getInstr()];
892+
HR.releaseFromScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
893+
HR.getMemoryBanks(MI), MI->operands(),
894+
MI->getMF()->getRegInfo(), DeltaCycles);
895+
// Update the selected opcode for the instruction, refer
896+
// AIEPostRASchedStrategy::canShiftSlot()
897+
HR.getSelectedAltDescs().setAlternateDescriptor(MI, Desc);
898+
899+
assert(HR.getHazardType(*Desc, HR.getMemoryBanks(MI), MI->operands(),
900+
MI->getMF()->getRegInfo(), DeltaCycles) ==
901+
ScheduleHazardRecognizer::HazardType::NoHazard);
902+
// Reschedule the instruction with the new opcode.
903+
HR.emitInScoreboard(*Desc, HR.getMemoryBanks(MI), MI->operands(),
904+
MI->getMF()->getRegInfo(), DeltaCycles);
905+
}
906+
716907
Bot.bumpNode(SU, DeltaCycles);
717908
}
909+
// Clear the MutateInstruction map since after scheduling the instruction the
910+
// validity of mutation map can no longer be guaranteed.
911+
MutateInstruction.clear();
912+
SU->isScheduled = true;
718913
}
719914

720915
void AIEPostRASchedStrategy::enterFunction(MachineFunction *MF) {

Diff for: ‎llvm/lib/Target/AIE/AIEMachineScheduler.h

+2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ std::vector<AIE::MachineBundle> computeAndFinalizeBundles(SchedBoundary &Zone);
3737
class AIEPostRASchedStrategy : public PostGenericScheduler {
3838
/// Maintain the state of interblock/loop-aware scheduling
3939
AIE::InterBlockScheduling InterBlock;
40+
MutateInstructionMap MutateInstruction;
4041

4142
public:
4243
AIEPostRASchedStrategy(const MachineSchedContext *C);
@@ -50,6 +51,7 @@ class AIEPostRASchedStrategy : public PostGenericScheduler {
5051
SUnit *pickNodeAndCycle(bool &IsTopNode,
5152
std::optional<unsigned> &BotEmissionCycle) override;
5253

54+
bool canShiftSlot(SUnit &SU, SchedBoundary &Zone, const int DeltaCycle);
5355
bool isAvailableNode(SUnit &SU, SchedBoundary &Zone,
5456
bool VerifyReadyCycle) override;
5557

0 commit comments

Comments
 (0)