Skip to content

Commit e849513

Browse files
krishnamtibrewalakrishnamtibrewala
committed
[AIEX] Extend rescheduling when the new SU is a multi-slot
1 parent 2a6941b commit e849513

30 files changed

+517
-229
lines changed

Diff for: ‎llvm/lib/Target/AIE/AIEAlternateDescriptors.h

+8-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222

2323
namespace llvm {
2424

25+
using MutateInstructionMap =
26+
std::unordered_map<MachineInstr *,
27+
std::pair<MachineInstr *, const MCInstrDesc *>>;
2528
using MIAltDescsMap = std::unordered_map<MachineInstr *, const MCInstrDesc *>;
2629

2730
class AIEAlternateDescriptors {
@@ -40,7 +43,11 @@ class AIEAlternateDescriptors {
4043
const AIEBaseSubtarget &STI = AIEBaseSubtarget::get(*MI->getMF());
4144
const AIEBaseInstrInfo *TII = STI.getInstrInfo();
4245

43-
AlternateDescs[MI] = &TII->get(AltInstOpcode);
46+
setAlternateDescriptor(MI, &TII->get(AltInstOpcode));
47+
}
48+
49+
void setAlternateDescriptor(MachineInstr *MI, const MCInstrDesc *AltDesc) {
50+
AlternateDescs[MI] = AltDesc;
4451
}
4552

4653
// Return the alternate descriptor for the given multi-opcode instruction.

Diff for: ‎llvm/lib/Target/AIE/AIEHazardRecognizer.cpp

+35-1
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,14 @@ auto toHazardType(bool Conflict) {
440440
}
441441
} // namespace
442442

443+
ScheduleHazardRecognizer::HazardType AIEHazardRecognizer::getHazardType(
444+
const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
445+
iterator_range<const MachineOperand *> MIOperands,
446+
const MachineRegisterInfo &MRI, int DeltaCycles) {
447+
return getHazardType(Scoreboard, Desc, MemoryBanks, MIOperands, MRI,
448+
DeltaCycles);
449+
}
450+
443451
// These functions interpret the itinerary, translating InstrStages
444452
// to ResourceCycles to apply.
445453
// We deviate from the standard ScoreboardHazardRecognizer by not
@@ -457,10 +465,28 @@ ScheduleHazardRecognizer::HazardType AIEHazardRecognizer::getHazardType(
457465
FUDepthLimit));
458466
}
459467

468+
ConflictTypeBits AIEHazardRecognizer::checkConflict(MachineInstr &MI,
469+
int DeltaCycles) {
470+
assert(!TII->getFormatInterface()->getAlternateInstsOpcode(MI.getOpcode()));
471+
return checkConflict(Scoreboard, MI, DeltaCycles);
472+
}
473+
460474
ConflictTypeBits AIEHazardRecognizer::checkConflict(
461475
const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, MachineInstr &MI,
462476
int DeltaCycles) const {
463-
const MCInstrDesc &Desc = MI.getDesc();
477+
assert(!TII->getFormatInterface()->getAlternateInstsOpcode(MI.getOpcode()));
478+
return checkConflict(Scoreboard, MI, MI.getDesc(), DeltaCycles);
479+
}
480+
481+
ConflictTypeBits AIEHazardRecognizer::checkConflict(MachineInstr &MI,
482+
const MCInstrDesc &Desc,
483+
int DeltaCycles) {
484+
return checkConflict(Scoreboard, MI, Desc, DeltaCycles);
485+
}
486+
487+
ConflictTypeBits AIEHazardRecognizer::checkConflict(
488+
const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, MachineInstr &MI,
489+
const MCInstrDesc &Desc, int DeltaCycles) const {
464490
const unsigned SchedClass =
465491
TII->getSchedClass(Desc, MI.operands(), MI.getMF()->getRegInfo());
466492
const MemoryBankBits MemoryBanks = getMemoryBanks(&MI);
@@ -572,6 +598,14 @@ void AIEHazardRecognizer::emitInScoreboard(
572598
TII->getMemoryCycles(SchedClass), DeltaCycles, FUDepthLimit);
573599
}
574600

601+
void AIEHazardRecognizer::releaseFromScoreboard(
602+
const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
603+
iterator_range<const MachineOperand *> MIOperands,
604+
const MachineRegisterInfo &MRI, int DeltaCycles) {
605+
releaseFromScoreboard(Scoreboard, Desc, MemoryBanks, MIOperands, MRI,
606+
DeltaCycles);
607+
}
608+
575609
void AIEHazardRecognizer::releaseFromScoreboard(
576610
ResourceScoreboard<FuncUnitWrapper> &TheScoreboard, const MCInstrDesc &Desc,
577611
MemoryBankBits MemoryBanks,

Diff for: ‎llvm/lib/Target/AIE/AIEHazardRecognizer.h

+19-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
namespace llvm {
3030

3131
class MachineInstr;
32-
using ConflictTypeBits = uint64_t;
32+
using ConflictTypeBits = std::uint32_t;
3333

3434
void applyFormatOrdering(AIE::MachineBundle &Bundle, const VLIWFormat &Format,
3535
MachineInstr *BundleRoot,
@@ -164,6 +164,11 @@ class AIEHazardRecognizer : public ScheduleHazardRecognizer {
164164
iterator_range<const MachineOperand *> MIOperands,
165165
const MachineRegisterInfo &MRI,
166166
int DeltaCycles) const;
167+
// Apply the above function to the local scoreboard.
168+
void releaseFromScoreboard(const MCInstrDesc &Desc,
169+
MemoryBankBits MemoryBanks,
170+
iterator_range<const MachineOperand *> MIOperands,
171+
const MachineRegisterInfo &MRI, int DeltaCycles);
167172

168173
/// Block all scoreboard resources at DeltaCycles
169174
void blockCycleInScoreboard(int DeltaCycle);
@@ -205,9 +210,22 @@ class AIEHazardRecognizer : public ScheduleHazardRecognizer {
205210
const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
206211
iterator_range<const MachineOperand *> MIOperands,
207212
const MachineRegisterInfo &MRI, int DeltaCycles) const;
213+
ScheduleHazardRecognizer::HazardType
214+
getHazardType(const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
215+
iterator_range<const MachineOperand *> MIOperands,
216+
const MachineRegisterInfo &MRI, int DeltaCycles);
217+
218+
ConflictTypeBits
219+
checkConflict(const ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
220+
MachineInstr &MI, const MCInstrDesc &Desc,
221+
int DeltaCycles) const;
222+
ConflictTypeBits checkConflict(MachineInstr &MI, const MCInstrDesc &Desc,
223+
int DeltaCycles);
224+
208225
ConflictTypeBits
209226
checkConflict(const ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
210227
MachineInstr &MI, int DeltaCycles) const;
228+
ConflictTypeBits checkConflict(MachineInstr &MI, int DeltaCycles);
211229

212230
protected:
213231
ScheduleHazardRecognizer::HazardType getHazardType(const MCInstrDesc &Desc,

Diff for: ‎llvm/lib/Target/AIE/AIEMachineScheduler.cpp

+192-1
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,13 @@ static cl::opt<bool> PreSchedFollowsSkipPipeliner(
9393
"aie-presched-follows-skip-pipeliner", cl::init(true),
9494
cl::desc("Don't run the prescheduler if the pipeliner is skipped"));
9595

96+
/// This option enables instruction mutuation to shift a multislot instruction
97+
/// in event of a slot conflict.
98+
static cl::opt<bool> InstructionMutation(
99+
"aie-instruction-mutation", cl::init(true),
100+
cl::desc("Allow instruction mutation to shift a multislot "
101+
"instruction in event of a slot conflict"));
102+
96103
namespace {
97104
// A sentinel value to represent an unknown SUnit.
98105
const constexpr unsigned UnknownSUNum = ~0;
@@ -484,6 +491,164 @@ int AIEPostRASchedStrategy::getMaxDeltaCycles(const SchedBoundary &Zone) const {
484491
BottomUpDelta.getValue()});
485492
}
486493

494+
static bool checkSlotConflict(const unsigned OpCodeA, const unsigned OpCodeB,
495+
const AIEBaseMCFormats &Formats) {
496+
497+
const MCSlotKind SlotKindA = Formats.getSlotKind(OpCodeA);
498+
const MCSlotKind SlotKindB = Formats.getSlotKind(OpCodeB);
499+
const MCSlotKind UnknownSlot = MCSlotKind();
500+
501+
if (SlotKindA == UnknownSlot || SlotKindB == UnknownSlot)
502+
return false;
503+
504+
const auto *SlotInfoA = Formats.getSlotInfo(SlotKindA);
505+
const auto *SlotInfoB = Formats.getSlotInfo(SlotKindB);
506+
507+
return SlotInfoA->getSlotSet() & SlotInfoB->getSlotSet();
508+
}
509+
510+
bool AIEPostRASchedStrategy::canShiftSlot(SUnit &SU, SchedBoundary &Zone,
511+
const int DeltaCycle) {
512+
513+
if (!InstructionMutation)
514+
return false;
515+
516+
const ScheduleDAGMI &DAG = *Zone.DAG;
517+
const AIEBaseInstrInfo &TII = *getTII(DAG);
518+
const AIEBaseMCFormats &Formats = *TII.getFormatInterface();
519+
AIEHazardRecognizer &HR = *getAIEHazardRecognizer(Zone);
520+
MachineInstr *NewMI = SU.getInstr();
521+
std::vector<MachineInstr *> ScheduledMultiSlotInsts;
522+
bool CanShiftSlot = false;
523+
524+
// Find and cache if there are any multi-slot instructions scheduled in the
525+
// same delta cycle
526+
for (MachineInstr &MI : DAG) {
527+
SUnit *ZoneSU = DAG.getSUnit(&MI);
528+
if (!ZoneSU)
529+
continue;
530+
if (!ZoneSU->isScheduled)
531+
continue;
532+
533+
const int CurrCycle = Zone.getCurrCycle();
534+
if (ZoneSU->BotReadyCycle !=
535+
static_cast<unsigned int>(CurrCycle - DeltaCycle))
536+
continue;
537+
538+
// Check for a MultiSlot instruction scheduled in the same DeltaCycle,
539+
// we focus on multi-slot because they can be scheduled in different
540+
// slots
541+
auto AltOpcodes = Formats.getAlternateInstsOpcode(MI.getOpcode());
542+
if (!AltOpcodes)
543+
continue;
544+
ScheduledMultiSlotInsts.push_back(&MI);
545+
}
546+
547+
// If there are no multi-slot instructions scheduled in the same DeltaCycle we
548+
// cannot shift any instruction to a different slot.
549+
if (ScheduledMultiSlotInsts.empty())
550+
return false;
551+
552+
const std::vector<unsigned int> *NewMIAltOpcodes;
553+
auto DefaultOpcode = std::vector<unsigned int>{SU.getInstr()->getOpcode()};
554+
NewMIAltOpcodes =
555+
Formats.getAlternateInstsOpcode(SU.getInstr()->getOpcode())
556+
? Formats.getAlternateInstsOpcode(SU.getInstr()->getOpcode())
557+
: &DefaultOpcode;
558+
559+
for (const unsigned int NewMIAltOpcode : *NewMIAltOpcodes) {
560+
const MCInstrDesc &NewMIAltDesc = TII.get(NewMIAltOpcode);
561+
if (!(HR.checkConflict(*NewMI, NewMIAltDesc, DeltaCycle) &
562+
static_cast<uint32_t>(AIEHazardRecognizer::ConflictType::Format)))
563+
continue;
564+
565+
for (MachineInstr *MI : ScheduledMultiSlotInsts) {
566+
SUnit *ZoneSU = DAG.getSUnit(MI);
567+
const int CurrCycle = Zone.getCurrCycle();
568+
auto AltOpcodes = Formats.getAlternateInstsOpcode(MI->getOpcode());
569+
570+
// Check if the scheduled multi-slot instruction has a slot conflict
571+
// with the new instruction, if so we might have the possiblity to shift
572+
// the multi-slot and schedule the new instruction.
573+
if (!checkSlotConflict(HR.getSelectedAltDescs().getOpcode(MI),
574+
NewMIAltOpcode, Formats))
575+
continue;
576+
577+
// Release the multi-slot instruction from the scoreboard to check if
578+
// any other alternate opcode in presence of the new instruction will
579+
// not create a hazard.
580+
HR.releaseFromScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
581+
HR.getMemoryBanks(MI), MI->operands(),
582+
MI->getMF()->getRegInfo(),
583+
CurrCycle - ZoneSU->BotReadyCycle);
584+
585+
// Check if the new instruction can be scheduled after unscheduling
586+
// the conflicting multi-slot instruction.
587+
if (HR.getHazardType(NewMIAltDesc, HR.getMemoryBanks(NewMI),
588+
NewMI->operands(), NewMI->getMF()->getRegInfo(),
589+
DeltaCycle) !=
590+
ScheduleHazardRecognizer::HazardType::NoHazard) {
591+
// If the new instruction cannot be scheduled after unscheduling the
592+
// mulit-slot revert back the state of scoreboard to original state
593+
// and continue.
594+
595+
HR.emitInScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
596+
HR.getMemoryBanks(MI), MI->operands(),
597+
MI->getMF()->getRegInfo(),
598+
CurrCycle - ZoneSU->BotReadyCycle);
599+
continue;
600+
}
601+
602+
// Emit the new instruction in the scoreboard. This will help us
603+
// to check if the previously unscheduled multi-slot instruction
604+
// can be scheduled in the same cycle, with an alternate opcode.
605+
HR.emitInScoreboard(NewMIAltDesc, HR.getMemoryBanks(NewMI),
606+
NewMI->operands(), NewMI->getMF()->getRegInfo(),
607+
DeltaCycle);
608+
609+
// Check if the previously unscheduled multi-slot instruction
610+
// can be rescheduled in presense of the new instruction in the
611+
// same cycle, with a different opcode.
612+
for (const auto AltOpcodeInside : *AltOpcodes) {
613+
const MCInstrDesc &Desc = TII.get(AltOpcodeInside);
614+
if (HR.getHazardType(Desc, HR.getMemoryBanks(MI), MI->operands(),
615+
MI->getMF()->getRegInfo(), DeltaCycle) ==
616+
ScheduleHazardRecognizer::HazardType::NoHazard) {
617+
// Cache the information to mutate the instruction during bumpNode()
618+
MutateInstruction.insert(
619+
std::make_pair(NewMI, std::make_pair(MI, &Desc)));
620+
621+
// if the new instruction was a multi-slot instruction and it failed
622+
// the general check for isAvailabeNode() this means we have not set
623+
// the selected opcode for the instruction. Set the selected opcode
624+
// for the instruction.
625+
if (NewMIAltOpcodes->size() > 1)
626+
HR.getSelectedAltDescs().setAlternateDescriptor(NewMI,
627+
&NewMIAltDesc);
628+
629+
CanShiftSlot = true;
630+
break;
631+
}
632+
}
633+
634+
// Revert back the state of scoreboard to original state.
635+
HR.releaseFromScoreboard(NewMIAltDesc, HR.getMemoryBanks(NewMI),
636+
NewMI->operands(), NewMI->getMF()->getRegInfo(),
637+
DeltaCycle);
638+
HR.emitInScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
639+
HR.getMemoryBanks(MI), MI->operands(),
640+
MI->getMF()->getRegInfo(),
641+
CurrCycle - ZoneSU->BotReadyCycle);
642+
643+
if (CanShiftSlot)
644+
break;
645+
}
646+
if (CanShiftSlot)
647+
break;
648+
}
649+
return CanShiftSlot;
650+
}
651+
487652
bool AIEPostRASchedStrategy::isAvailableNode(SUnit &SU, SchedBoundary &Zone,
488653
bool /*VerifyReadyCycle*/) {
489654
// Whether or not the zone is Top or Bot, verify if SU is ready to be
@@ -502,7 +667,8 @@ bool AIEPostRASchedStrategy::isAvailableNode(SUnit &SU, SchedBoundary &Zone,
502667
// ReadyCycle is always greater or equal to the current cycle,
503668
// so DeltaCycles will always be less or equal to 0.
504669
if (Zone.checkHazard(&SU, DeltaCycles))
505-
continue;
670+
if (!canShiftSlot(SU, Zone, DeltaCycles))
671+
continue;
506672
SU.BotReadyCycle = CurrCycle - DeltaCycles;
507673
return true;
508674
}
@@ -519,10 +685,35 @@ void AIEPostRASchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
519685
if (IsTopNode) {
520686
PostGenericScheduler::schedNode(SU, IsTopNode);
521687
} else {
688+
AIEHazardRecognizer &HR = *getAIEHazardRecognizer(Bot);
522689
int DeltaCycles = int(Bot.getCurrCycle()) - int(SU->BotReadyCycle);
523690
assert(DeltaCycles <= 0);
691+
692+
// Check if an instruction needs to be moved to a different slot for the
693+
// current SU to be scheduled in the DeltaCycles.
694+
if (MutateInstruction.find(SU->getInstr()) != MutateInstruction.end()) {
695+
auto &[MI, Desc] = MutateInstruction[SU->getInstr()];
696+
HR.releaseFromScoreboard(*HR.getSelectedAltDescs().getDesc(MI),
697+
HR.getMemoryBanks(MI), MI->operands(),
698+
MI->getMF()->getRegInfo(), DeltaCycles);
699+
// Update the selected opcode for the instruction, refer
700+
// AIEPostRASchedStrategy::canShiftSlot()
701+
HR.getSelectedAltDescs().setAlternateDescriptor(MI, Desc);
702+
703+
assert(HR.getHazardType(*Desc, HR.getMemoryBanks(MI), MI->operands(),
704+
MI->getMF()->getRegInfo(), DeltaCycles) ==
705+
ScheduleHazardRecognizer::HazardType::NoHazard);
706+
// Reschedule the instruction with the new opcode.
707+
HR.emitInScoreboard(*Desc, HR.getMemoryBanks(MI), MI->operands(),
708+
MI->getMF()->getRegInfo(), DeltaCycles);
709+
}
710+
524711
Bot.bumpNode(SU, DeltaCycles);
525712
}
713+
// Clear the MutateInstruction map since after scheduling the instruction the
714+
// validity of mutation map can no longer be guaranteed.
715+
MutateInstruction.clear();
716+
SU->isScheduled = true;
526717
}
527718

528719
void AIEPostRASchedStrategy::enterFunction(MachineFunction *MF) {

Diff for: ‎llvm/lib/Target/AIE/AIEMachineScheduler.h

+2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ std::vector<AIE::MachineBundle> computeAndFinalizeBundles(SchedBoundary &Zone);
3737
class AIEPostRASchedStrategy : public PostGenericScheduler {
3838
/// Maintain the state of interblock/loop-aware scheduling
3939
AIE::InterBlockScheduling InterBlock;
40+
MutateInstructionMap MutateInstruction;
4041

4142
public:
4243
AIEPostRASchedStrategy(const MachineSchedContext *C);
@@ -50,6 +51,7 @@ class AIEPostRASchedStrategy : public PostGenericScheduler {
5051
SUnit *pickNodeAndCycle(bool &IsTopNode,
5152
std::optional<unsigned> &BotEmissionCycle) override;
5253

54+
bool canShiftSlot(SUnit &SU, SchedBoundary &Zone, const int DeltaCycle);
5355
bool isAvailableNode(SUnit &SU, SchedBoundary &Zone,
5456
bool VerifyReadyCycle) override;
5557

0 commit comments

Comments
 (0)