From 88b6442527880540cae334eef9aa63d10debb521 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Wed, 20 Sep 2023 21:20:59 -0700 Subject: [PATCH 1/5] irjit: Add facility for native reg transfer. --- Core/MIPS/ARM64/Arm64IRRegCache.cpp | 2 +- Core/MIPS/ARM64/Arm64IRRegCache.h | 2 +- Core/MIPS/IR/IRRegCache.cpp | 46 ++++++++++++++++++++--------- Core/MIPS/IR/IRRegCache.h | 3 +- Core/MIPS/RiscV/RiscVRegCache.cpp | 4 +-- Core/MIPS/RiscV/RiscVRegCache.h | 2 +- 6 files changed, 39 insertions(+), 20 deletions(-) diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.cpp b/Core/MIPS/ARM64/Arm64IRRegCache.cpp index 0420a808ce..f48207fa5f 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.cpp +++ b/Core/MIPS/ARM64/Arm64IRRegCache.cpp @@ -347,7 +347,7 @@ void Arm64IRRegCache::AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) { } } -bool Arm64IRRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) { +bool Arm64IRRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) { // No special flags, skip the check for a little speed. return true; } diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.h b/Core/MIPS/ARM64/Arm64IRRegCache.h index 3a9bf77ab9..9f0b0cbbac 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.h +++ b/Core/MIPS/ARM64/Arm64IRRegCache.h @@ -86,7 +86,7 @@ protected: const int *GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &count, int &base) const override; void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) override; - bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) override; + bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) override; void LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override; diff --git a/Core/MIPS/IR/IRRegCache.cpp b/Core/MIPS/IR/IRRegCache.cpp index b7ce1e265b..31a0da4d7d 100644 --- a/Core/MIPS/IR/IRRegCache.cpp +++ b/Core/MIPS/IR/IRRegCache.cpp @@ -501,7 +501,7 @@ IRNativeReg IRNativeRegCacheBase::FindBestToSpill(MIPSLoc type, MIPSMap flags, b return -1; } -bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) { +bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) { int allocCount = 0, base = 0; const int *allocOrder = GetAllocationOrder(type, flags, allocCount, base); @@ -514,6 +514,11 @@ bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, return false; } +bool IRNativeRegCacheBase::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) { + // To be overridden if the backend supports transfers. + return false; +} + void IRNativeRegCacheBase::DiscardNativeReg(IRNativeReg nreg) { _assert_msg_(nreg >= 0 && nreg < config_.totalNativeRegs, "DiscardNativeReg on invalid register %d", nreg); if (nr[nreg].mipsReg != IRREG_INVALID) { @@ -930,11 +935,14 @@ IRNativeReg IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRReg first, int la case MIPSLoc::REG: if (type != MIPSLoc::REG) { nreg = AllocateReg(type, flags); - } else if (!IsNativeRegCompatible(nreg, type, flags)) { + } else if (!IsNativeRegCompatible(nreg, type, flags, lanes)) { // If it's not compatible, we'll need to reallocate. - // TODO: Could do a transfer and avoid memory flush. - FlushNativeReg(nreg); - nreg = AllocateReg(type, flags); + if (TransferNativeReg(nreg, -1, type, first, lanes, flags)) { + nreg = mr[first].nReg; + } else { + FlushNativeReg(nreg); + nreg = AllocateReg(type, flags); + } } break; @@ -942,9 +950,13 @@ IRNativeReg IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRReg first, int la case MIPSLoc::VREG: if (type != mr[first].loc) { nreg = AllocateReg(type, flags); - } else if (!IsNativeRegCompatible(nreg, type, flags)) { - FlushNativeReg(nreg); - nreg = AllocateReg(type, flags); + } else if (!IsNativeRegCompatible(nreg, type, flags, lanes)) { + if (TransferNativeReg(nreg, -1, type, first, lanes, flags)) { + nreg = mr[first].nReg; + } else { + FlushNativeReg(nreg); + nreg = AllocateReg(type, flags); + } } break; @@ -981,10 +993,13 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi _assert_msg_(!mreg.isStatic, "Cannot MapNativeReg a static reg mismatch"); if ((flags & MIPSMap::NOINIT) != MIPSMap::NOINIT) { // If we need init, we have to flush mismatches. - // TODO: Do a shuffle if interior only? - // TODO: We may also be motivated to have multiple read-only "views" or an IRReg. - // For example Vec4Scale v0..v3, v0..v3, v3 - FlushNativeReg(mreg.nReg); + if (!TransferNativeReg(mreg.nReg, nreg, type, first, lanes, flags)) { + // TODO: We may also be motivated to have multiple read-only "views" or an IRReg. + // For example Vec4Scale v0..v3, v0..v3, v3 + FlushNativeReg(mreg.nReg); + } + // The mismatch has been "resolved" now. + mismatch = false; } else if (oldlanes != 1) { // Even if we don't care about the current contents, we can't discard outside. bool extendsBefore = oldlane > i; @@ -1017,6 +1032,9 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi DiscardNativeReg(mreg.nReg); else FlushNativeReg(mreg.nReg); + + // That took care of the mismatch, either by clobber or flush. + mismatch = false; } } } @@ -1027,8 +1045,8 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi if ((flags & MIPSMap::NOINIT) != MIPSMap::NOINIT) { // We better not be trying to map to a different nreg if it's in one now. // This might happen on some sort of transfer... - // TODO: Make a direct transfer, i.e. FREG -> VREG? - FlushNativeReg(mreg.nReg); + if (!TransferNativeReg(mreg.nReg, nreg, type, first, lanes, flags)) + FlushNativeReg(mreg.nReg); } else { DiscardNativeReg(mreg.nReg); } diff --git a/Core/MIPS/IR/IRRegCache.h b/Core/MIPS/IR/IRRegCache.h index c85bb41848..07067cbdcd 100644 --- a/Core/MIPS/IR/IRRegCache.h +++ b/Core/MIPS/IR/IRRegCache.h @@ -209,13 +209,14 @@ protected: IRNativeReg AllocateReg(MIPSLoc type, MIPSMap flags); IRNativeReg FindFreeReg(MIPSLoc type, MIPSMap flags) const; IRNativeReg FindBestToSpill(MIPSLoc type, MIPSMap flags, bool unusedOnly, bool *clobbered) const; - virtual bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags); + virtual bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes); virtual void DiscardNativeReg(IRNativeReg nreg); virtual void FlushNativeReg(IRNativeReg nreg); virtual void DiscardReg(IRReg mreg); virtual void FlushReg(IRReg mreg); virtual void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state); virtual void MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg first, int lanes, MIPSMap flags); + virtual bool TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags); virtual IRNativeReg MapNativeReg(MIPSLoc type, IRReg first, int lanes, MIPSMap flags); IRNativeReg MapNativeRegAsPointer(IRReg gpr); diff --git a/Core/MIPS/RiscV/RiscVRegCache.cpp b/Core/MIPS/RiscV/RiscVRegCache.cpp index 7a3e6505cb..25528aa3aa 100644 --- a/Core/MIPS/RiscV/RiscVRegCache.cpp +++ b/Core/MIPS/RiscV/RiscVRegCache.cpp @@ -303,11 +303,11 @@ void RiscVRegCache::AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) { } } -bool RiscVRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) { +bool RiscVRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) { // No special flags except VREG, skip the check for a little speed. if (type != MIPSLoc::VREG) return true; - return IRNativeRegCacheBase::IsNativeRegCompatible(nreg, type, flags); + return IRNativeRegCacheBase::IsNativeRegCompatible(nreg, type, flags, lanes); } void RiscVRegCache::LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) { diff --git a/Core/MIPS/RiscV/RiscVRegCache.h b/Core/MIPS/RiscV/RiscVRegCache.h index facfa52195..e0075f2c61 100644 --- a/Core/MIPS/RiscV/RiscVRegCache.h +++ b/Core/MIPS/RiscV/RiscVRegCache.h @@ -76,7 +76,7 @@ protected: const int *GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &count, int &base) const override; void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) override; - bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) override; + bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) override; void LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override; From d9f6bae1ff552899854f7ef54206f9c2bbfcae92 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sat, 23 Sep 2023 11:14:42 -0700 Subject: [PATCH 2/5] x64jit: Initial reg transfer. --- Common/x64Emitter.cpp | 6 +- Common/x64Emitter.h | 12 ++- Core/MIPS/x86/X64IRRegCache.cpp | 183 ++++++++++++++++++++++++++++++++ Core/MIPS/x86/X64IRRegCache.h | 1 + 4 files changed, 195 insertions(+), 7 deletions(-) diff --git a/Common/x64Emitter.cpp b/Common/x64Emitter.cpp index c2a5ba8c4d..814fc7e0d6 100644 --- a/Common/x64Emitter.cpp +++ b/Common/x64Emitter.cpp @@ -1697,7 +1697,6 @@ void XEmitter::MOVMSKPD(X64Reg dest, OpArg arg) {WriteSSEOp(0x66, 0x50, dest, ar void XEmitter::LDDQU(X64Reg dest, OpArg arg) {WriteSSEOp(0xF2, sseLDDQU, dest, arg);} // For integer data only -// THESE TWO ARE UNTESTED. void XEmitter::UNPCKLPS(X64Reg dest, OpArg arg) {WriteSSEOp(0x00, 0x14, dest, arg);} void XEmitter::UNPCKHPS(X64Reg dest, OpArg arg) {WriteSSEOp(0x00, 0x15, dest, arg);} @@ -1892,6 +1891,9 @@ void XEmitter::PTEST(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x3817, dest void XEmitter::PACKUSDW(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x382b, dest, arg);} void XEmitter::DPPS(X64Reg dest, OpArg arg, u8 mask) {WriteSSE41Op(0x66, 0x3A40, dest, arg, 1); Write8(mask);} +void XEmitter::INSERTPS(X64Reg dest, OpArg arg, u8 dstsubreg, u8 srcsubreg, u8 zmask) { WriteSSE41Op(0x66, 0x3A21, dest, arg, 1); Write8((srcsubreg << 6) | (dstsubreg << 4) | zmask); } +void XEmitter::EXTRACTPS(OpArg dest, X64Reg arg, u8 subreg) { WriteSSE41Op(0x66, 0x3A17, arg, dest, 1); Write8(subreg); } + void XEmitter::PMINSB(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x3838, dest, arg);} void XEmitter::PMINSD(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x3839, dest, arg);} void XEmitter::PMINUW(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x383a, dest, arg);} @@ -2084,7 +2086,7 @@ void XEmitter::VCVTTPD2DQ(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, void XEmitter::VCVTTSS2SI(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(0, 0xF3, 0x2C, regOp1, arg, 0, bits == 64 ? 1 : 0); } void XEmitter::VCVTTSD2SI(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(0, 0xF2, 0x2C, regOp1, arg, 0, bits == 64 ? 1 : 0); } void XEmitter::VEXTRACTPS(OpArg arg, X64Reg regOp1, u8 subreg) { WriteAVXOp(0, 0x66, 0x3A17, regOp1, arg, 1); Write8(subreg); } -void XEmitter::VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 subreg) { WriteAVXOp(0, 0x66, 0x3A21, regOp1, regOp2, arg, 1); Write8(subreg); } +void XEmitter::VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 dstsubreg, u8 srcsubreg, u8 zmask) { WriteAVXOp(0, 0x66, 0x3A21, regOp1, regOp2, arg, 1); Write8((srcsubreg << 6) | (dstsubreg << 4) | zmask); } void XEmitter::VLDDQU(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0xF2, sseLDDQU, regOp1, arg); } void XEmitter::VMOVAPS(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0x00, sseMOVAPfromRM, regOp1, arg); } void XEmitter::VMOVAPD(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0x66, sseMOVAPfromRM, regOp1, arg); } diff --git a/Common/x64Emitter.h b/Common/x64Emitter.h index 16f30a35b0..832ed767cb 100644 --- a/Common/x64Emitter.h +++ b/Common/x64Emitter.h @@ -684,12 +684,14 @@ public: // SSE4: Further horizontal operations - dot products. These are weirdly flexible, the arg contains both a read mask and a write "mask". void DPPD(X64Reg dest, OpArg src, u8 arg); - - // These are probably useful for VFPU emulation. - void INSERTPS(X64Reg dest, OpArg src, u8 arg); - void EXTRACTPS(OpArg dest, X64Reg src, u8 arg); #endif + // SSE4: Insert and extract for floats. + // Note: insert from memory or an XMM. + void INSERTPS(X64Reg dest, OpArg arg, u8 dstsubreg, u8 srcsubreg = 0, u8 zmask = 0); + // Extract to memory or GPR. + void EXTRACTPS(OpArg dest, X64Reg arg, u8 subreg); + // SSE3: Horizontal operations in SIMD registers. Very slow! shufps-based code beats it handily on Ivy. void HADDPS(X64Reg dest, OpArg src); @@ -1040,7 +1042,7 @@ public: // Can only extract from the low 128 bits. void VEXTRACTPS(OpArg arg, X64Reg regOp1, u8 subreg); // Can only insert into the low 128 bits, zeros upper bits. Inserts from XMM. - void VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 subreg); + void VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 dstsubreg, u8 srcsubreg = 0, u8 zmask = 0); void VLDDQU(int bits, X64Reg regOp1, OpArg arg); void VMOVAPS(int bits, X64Reg regOp1, OpArg arg); void VMOVAPD(int bits, X64Reg regOp1, OpArg arg); diff --git a/Core/MIPS/x86/X64IRRegCache.cpp b/Core/MIPS/x86/X64IRRegCache.cpp index ee176546c8..b8b2dd522f 100644 --- a/Core/MIPS/x86/X64IRRegCache.cpp +++ b/Core/MIPS/x86/X64IRRegCache.cpp @@ -453,6 +453,189 @@ void X64IRRegCache::StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) { } } +bool X64IRRegCache::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) { + bool allowed = !mr[nr[nreg].mipsReg].isStatic; + // There's currently no support for non-XMMs here. + allowed = allowed && type == MIPSLoc::FREG; + + if (dest == -1) + dest = nreg; + + if (allowed && (flags == MIPSMap::INIT || flags == MIPSMap::DIRTY)) { + // Alright, changing lane count (possibly including lane position.) + IRReg oldfirst = nr[nreg].mipsReg; + int oldlanes = 0; + while (mr[oldfirst + oldlanes].nReg == nreg) + oldlanes++; + _assert_msg_(oldlanes != 0, "TransferNativeReg encountered nreg mismatch"); + _assert_msg_(oldlanes != lanes, "TransferNativeReg transfer to same lanecount, misaligned?"); + + if (lanes == 1) { + // Okay, start by storing if dirty. + if (nr[nreg].isDirty) { + StoreNativeReg(nreg, oldfirst, oldlanes); + nr[nreg].isDirty = false; + } + // Next, shuffle the desired element into first place. + u8 shuf = VFPU_SWIZZLE(mr[first].lane, mr[first].lane, mr[first].lane, mr[first].lane); + if (mr[first].lane > 0 && cpu_info.bAVX && dest != nreg) { + emit_->VSHUFPS(128, FromNativeReg(dest), FromNativeReg(nreg), ::R(FromNativeReg(nreg)), shuf); + } else if (mr[first].lane <= 0 && dest != nreg) { + emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + } else if (mr[first].lane > 0) { + if (dest != nreg) + emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + emit_->SHUFPS(FromNativeReg(dest), ::R(FromNativeReg(dest)), shuf); + } + + // TODO: Consider moving the others to free regs if available? Likely will be wanted later. + + // Now update accounting. + for (int i = 0; i < oldlanes; ++i) { + auto &mreg = mr[oldfirst + i]; + if (oldfirst + i == first) { + mreg.lane = 0; + mreg.nReg = dest; + } else { + // No longer in a register. + mreg.nReg = -1; + mreg.lane = -1; + mreg.loc = MIPSLoc::MEM; + } + } + + if (dest != nreg) { + nr[dest].isDirty = nr[nreg].isDirty; + nr[nreg].mipsReg = -1; + nr[nreg].isDirty = false; + } + nr[dest].mipsReg = first; + + return true; + } + + if ((lanes == 4 || lanes == 2) && oldlanes == 1) { + X64Reg cur[4]{}; + int numInRegs = 0; + int numDirty = 0; + bool unavail = false; + for (int i = 0; i < lanes; ++i) { + if (mr[first + i].lane != -1 || (i != 0 && mr[first + i].spillLockIRIndex >= irIndex_)) { + unavail = true; + break; + } + + if (mr[first + i].nReg == -1) { + cur[i] = INVALID_REG; + } else { + cur[i] = FromNativeReg(mr[first + i].nReg); + numInRegs++; + if (nr[cur[i]].isDirty) + numDirty++; + } + } + + if (numInRegs == 0) + unavail = true; + + bool handled = false; + if (!unavail) { + // If everything's currently in a reg, move it into this reg. + if (lanes == 4) { + if (cur[0] == INVALID_REG) { + cur[0] = FromNativeReg(dest); + emit_->MOVSS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); + numInRegs++; + } + + // A lot of other methods are possible, but seem to make things slower in practice. + if (numInRegs == 4) { + // y = yw##, x = xz##, x = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->UNPCKLPS(cur[0], ::R(cur[2])); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + handled = true; + } else if (numInRegs == 2 && cur[1] != INVALID_REG) { + // x = xy##, then load zw. + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + emit_->MOVHPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2))); + handled = true; + } else if (cpu_info.bSSE4_1 && cur[1] != INVALID_REG && cur[2] != INVALID_REG) { + // x = xz##, z=w###, y=yw##, x=xyzw. + emit_->UNPCKLPS(cur[0], ::R(cur[2])); + emit_->MOVSS(cur[2], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 3))); + emit_->UNPCKLPS(cur[1], ::R(cur[2])); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + handled = true; + } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[1] != INVALID_REG && cur[3] != INVALID_REG) { + // y = yw##, load z into x[1], x = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2)), 1); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + handled = true; + } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[2] != INVALID_REG && cur[3] != INVALID_REG) { + // load y to x[1], z = zw##, x = xyzw. + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); + emit_->UNPCKLPS(cur[2], ::R(cur[3])); + emit_->MOVLHPS(cur[0], cur[2]); + handled = true; + } else if (cpu_info.bSSE4_1) { + // TODO: This might be worse than flushing depending? + for (int i = 1; i < 4; ++i) { + if (cur[i] == INVALID_REG) + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + i)), i); + else + emit_->INSERTPS(cur[0], ::R(cur[i]), i, 0); + } + handled = true; + } + } else if (lanes == 2) { + if (cur[0] != INVALID_REG && cur[1] != INVALID_REG) { + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + handled = true; + } else if (cur[0] != INVALID_REG && cpu_info.bSSE4_1) { + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); + handled = true; + } + } + } + + if (handled) { + mr[first].lane = 0; + for (int i = 0; i < lanes; ++i) { + if (mr[first + i].nReg != -1) { + // If this was dirty, the combined reg is now dirty. + if (nr[mr[first + i].nReg].isDirty) + nr[dest].isDirty = true; + + // Throw away the other register we're no longer using. + if (i != 0) + DiscardNativeReg(mr[first + i].nReg); + } + + // And set it as using the new one. + mr[first + i].lane = i; + mr[first + i].loc = type; + mr[first + i].nReg = dest; + } + + if (cur[0] != FromNativeReg(dest)) + emit_->MOVAPS(FromNativeReg(dest), ::R(cur[0])); + + if (dest != nreg) { + nr[dest].mipsReg = first; + nr[nreg].mipsReg = -1; + nr[nreg].isDirty = false; + } + + return true; + } + } + } + + return IRNativeRegCacheBase::TransferNativeReg(nreg, dest, type, first, lanes, flags); +} + void X64IRRegCache::SetNativeRegValue(IRNativeReg nreg, uint32_t imm) { X64Reg r = FromNativeReg(nreg); _dbg_assert_(nreg >= 0 && nreg < NUM_X_REGS); diff --git a/Core/MIPS/x86/X64IRRegCache.h b/Core/MIPS/x86/X64IRRegCache.h index f33e4e8d89..fd2a720bf8 100644 --- a/Core/MIPS/x86/X64IRRegCache.h +++ b/Core/MIPS/x86/X64IRRegCache.h @@ -117,6 +117,7 @@ protected: void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override; void StoreRegValue(IRReg mreg, uint32_t imm) override; + bool TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) override; private: IRNativeReg GPRToNativeReg(Gen::X64Reg r) { From 46e704f879c3857ddfc70bf5151ac2eb94400212 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 24 Sep 2023 16:58:41 -0700 Subject: [PATCH 3/5] x86jit: Cleanup and refactor transfer. --- Core/MIPS/x86/X64IRRegCache.cpp | 323 ++++++++++++++++---------------- Core/MIPS/x86/X64IRRegCache.h | 3 + 2 files changed, 166 insertions(+), 160 deletions(-) diff --git a/Core/MIPS/x86/X64IRRegCache.cpp b/Core/MIPS/x86/X64IRRegCache.cpp index b8b2dd522f..bd991c5018 100644 --- a/Core/MIPS/x86/X64IRRegCache.cpp +++ b/Core/MIPS/x86/X64IRRegCache.cpp @@ -470,172 +470,175 @@ bool X64IRRegCache::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLo _assert_msg_(oldlanes != 0, "TransferNativeReg encountered nreg mismatch"); _assert_msg_(oldlanes != lanes, "TransferNativeReg transfer to same lanecount, misaligned?"); - if (lanes == 1) { - // Okay, start by storing if dirty. - if (nr[nreg].isDirty) { - StoreNativeReg(nreg, oldfirst, oldlanes); - nr[nreg].isDirty = false; - } - // Next, shuffle the desired element into first place. - u8 shuf = VFPU_SWIZZLE(mr[first].lane, mr[first].lane, mr[first].lane, mr[first].lane); - if (mr[first].lane > 0 && cpu_info.bAVX && dest != nreg) { - emit_->VSHUFPS(128, FromNativeReg(dest), FromNativeReg(nreg), ::R(FromNativeReg(nreg)), shuf); - } else if (mr[first].lane <= 0 && dest != nreg) { - emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); - } else if (mr[first].lane > 0) { - if (dest != nreg) - emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); - emit_->SHUFPS(FromNativeReg(dest), ::R(FromNativeReg(dest)), shuf); - } - - // TODO: Consider moving the others to free regs if available? Likely will be wanted later. - - // Now update accounting. - for (int i = 0; i < oldlanes; ++i) { - auto &mreg = mr[oldfirst + i]; - if (oldfirst + i == first) { - mreg.lane = 0; - mreg.nReg = dest; - } else { - // No longer in a register. - mreg.nReg = -1; - mreg.lane = -1; - mreg.loc = MIPSLoc::MEM; - } - } - - if (dest != nreg) { - nr[dest].isDirty = nr[nreg].isDirty; - nr[nreg].mipsReg = -1; - nr[nreg].isDirty = false; - } - nr[dest].mipsReg = first; - + if (lanes == 1 && TransferVecTo1(nreg, dest, first, oldlanes)) + return true; + if (oldlanes == 1 && Transfer1ToVec(nreg, dest, first, lanes)) return true; - } - - if ((lanes == 4 || lanes == 2) && oldlanes == 1) { - X64Reg cur[4]{}; - int numInRegs = 0; - int numDirty = 0; - bool unavail = false; - for (int i = 0; i < lanes; ++i) { - if (mr[first + i].lane != -1 || (i != 0 && mr[first + i].spillLockIRIndex >= irIndex_)) { - unavail = true; - break; - } - - if (mr[first + i].nReg == -1) { - cur[i] = INVALID_REG; - } else { - cur[i] = FromNativeReg(mr[first + i].nReg); - numInRegs++; - if (nr[cur[i]].isDirty) - numDirty++; - } - } - - if (numInRegs == 0) - unavail = true; - - bool handled = false; - if (!unavail) { - // If everything's currently in a reg, move it into this reg. - if (lanes == 4) { - if (cur[0] == INVALID_REG) { - cur[0] = FromNativeReg(dest); - emit_->MOVSS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); - numInRegs++; - } - - // A lot of other methods are possible, but seem to make things slower in practice. - if (numInRegs == 4) { - // y = yw##, x = xz##, x = xyzw. - emit_->UNPCKLPS(cur[1], ::R(cur[3])); - emit_->UNPCKLPS(cur[0], ::R(cur[2])); - emit_->UNPCKLPS(cur[0], ::R(cur[1])); - handled = true; - } else if (numInRegs == 2 && cur[1] != INVALID_REG) { - // x = xy##, then load zw. - emit_->UNPCKLPS(cur[0], ::R(cur[1])); - emit_->MOVHPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2))); - handled = true; - } else if (cpu_info.bSSE4_1 && cur[1] != INVALID_REG && cur[2] != INVALID_REG) { - // x = xz##, z=w###, y=yw##, x=xyzw. - emit_->UNPCKLPS(cur[0], ::R(cur[2])); - emit_->MOVSS(cur[2], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 3))); - emit_->UNPCKLPS(cur[1], ::R(cur[2])); - emit_->UNPCKLPS(cur[0], ::R(cur[1])); - handled = true; - } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[1] != INVALID_REG && cur[3] != INVALID_REG) { - // y = yw##, load z into x[1], x = xyzw. - emit_->UNPCKLPS(cur[1], ::R(cur[3])); - emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2)), 1); - emit_->UNPCKLPS(cur[0], ::R(cur[1])); - handled = true; - } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[2] != INVALID_REG && cur[3] != INVALID_REG) { - // load y to x[1], z = zw##, x = xyzw. - emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); - emit_->UNPCKLPS(cur[2], ::R(cur[3])); - emit_->MOVLHPS(cur[0], cur[2]); - handled = true; - } else if (cpu_info.bSSE4_1) { - // TODO: This might be worse than flushing depending? - for (int i = 1; i < 4; ++i) { - if (cur[i] == INVALID_REG) - emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + i)), i); - else - emit_->INSERTPS(cur[0], ::R(cur[i]), i, 0); - } - handled = true; - } - } else if (lanes == 2) { - if (cur[0] != INVALID_REG && cur[1] != INVALID_REG) { - emit_->UNPCKLPS(cur[0], ::R(cur[1])); - handled = true; - } else if (cur[0] != INVALID_REG && cpu_info.bSSE4_1) { - emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); - handled = true; - } - } - } - - if (handled) { - mr[first].lane = 0; - for (int i = 0; i < lanes; ++i) { - if (mr[first + i].nReg != -1) { - // If this was dirty, the combined reg is now dirty. - if (nr[mr[first + i].nReg].isDirty) - nr[dest].isDirty = true; - - // Throw away the other register we're no longer using. - if (i != 0) - DiscardNativeReg(mr[first + i].nReg); - } - - // And set it as using the new one. - mr[first + i].lane = i; - mr[first + i].loc = type; - mr[first + i].nReg = dest; - } - - if (cur[0] != FromNativeReg(dest)) - emit_->MOVAPS(FromNativeReg(dest), ::R(cur[0])); - - if (dest != nreg) { - nr[dest].mipsReg = first; - nr[nreg].mipsReg = -1; - nr[nreg].isDirty = false; - } - - return true; - } - } } return IRNativeRegCacheBase::TransferNativeReg(nreg, dest, type, first, lanes, flags); } +bool X64IRRegCache::TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg first, int oldlanes) { + // Okay, start by storing if dirty. + // TODO: Consider moving the others to free regs if available? Likely will be wanted later. + IRReg oldfirst = nr[nreg].mipsReg; + if (nr[nreg].isDirty) { + StoreNativeReg(nreg, oldfirst, oldlanes); + nr[nreg].isDirty = false; + } + + // Next, shuffle the desired element into first place. + u8 shuf = VFPU_SWIZZLE(mr[first].lane, mr[first].lane, mr[first].lane, mr[first].lane); + if (mr[first].lane > 0 && cpu_info.bAVX && dest != nreg) { + emit_->VPERMILPS(128, FromNativeReg(dest), ::R(FromNativeReg(nreg)), shuf); + } else if (mr[first].lane <= 0 && dest != nreg) { + emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + } else if (mr[first].lane > 0) { + if (mr[first].lane == 2) { + emit_->MOVHLPS(FromNativeReg(dest), FromNativeReg(nreg)); + } else { + if (dest != nreg) + emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + emit_->SHUFPS(FromNativeReg(dest), ::R(FromNativeReg(dest)), shuf); + } + } + + // Now update accounting. + for (int i = 0; i < oldlanes; ++i) { + auto &mreg = mr[oldfirst + i]; + if (oldfirst + i == first) { + mreg.lane = 0; + mreg.nReg = dest; + } else { + // No longer in a register. + mreg.nReg = -1; + mreg.lane = -1; + mreg.loc = MIPSLoc::MEM; + } + } + + if (dest != nreg) { + nr[dest].isDirty = nr[nreg].isDirty; + nr[nreg].mipsReg = -1; + nr[nreg].isDirty = false; + } + nr[dest].mipsReg = first; + + return true; +} + +bool X64IRRegCache::Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg first, int lanes) { + X64Reg cur[4]{}; + int numInRegs = 0; + int numDirty = 0; + for (int i = 0; i < lanes; ++i) { + if (mr[first + i].lane != -1 || (i != 0 && mr[first + i].spillLockIRIndex >= irIndex_)) { + // Can't do it, either double mapped or overlapping vec. + return false; + } + + if (mr[first + i].nReg == -1) { + cur[i] = INVALID_REG; + } else { + cur[i] = FromNativeReg(mr[first + i].nReg); + numInRegs++; + if (nr[cur[i]].isDirty) + numDirty++; + } + } + + // Shouldn't happen, this should only get called to transfer one in a reg. + if (numInRegs == 0) + return false; + + // If everything's currently in a reg, move it into this reg. + if (lanes == 4) { + if (cur[0] == INVALID_REG) { + cur[0] = FromNativeReg(dest); + emit_->MOVSS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); + numInRegs++; + } + + // A lot of other methods are possible, but seem to make things slower in practice. + if (numInRegs == 4) { + // y = yw##, x = xz##, x = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->UNPCKLPS(cur[0], ::R(cur[2])); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (numInRegs == 2 && cur[1] != INVALID_REG) { + // x = xy##, then load zw. + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + emit_->MOVHPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2))); + } else if (cur[1] != INVALID_REG && cur[2] != INVALID_REG) { + // x = xz##, z=w###, y=yw##, x=xyzw. + emit_->UNPCKLPS(cur[0], ::R(cur[2])); + emit_->MOVSS(cur[2], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 3))); + emit_->UNPCKLPS(cur[1], ::R(cur[2])); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[1] != INVALID_REG && cur[3] != INVALID_REG) { + // y = yw##, load z into x[1], x = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2)), 1); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[2] != INVALID_REG && cur[3] != INVALID_REG) { + // load y to x[1], z = zw##, x = xyzw. + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); + emit_->UNPCKLPS(cur[2], ::R(cur[3])); + emit_->MOVLHPS(cur[0], cur[2]); + } else if (cpu_info.bSSE4_1) { + // TODO: This might be worse than flushing depending? + for (int i = 1; i < 4; ++i) { + if (cur[i] == INVALID_REG) + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + i)), i); + else + emit_->INSERTPS(cur[0], ::R(cur[i]), i, 0); + } + } else { + return false; + } + } else if (lanes == 2) { + if (cur[0] != INVALID_REG && cur[1] != INVALID_REG) { + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (cur[0] != INVALID_REG && cpu_info.bSSE4_1) { + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); + } else { + return false; + } + } else { + return false; + } + + mr[first].lane = 0; + for (int i = 0; i < lanes; ++i) { + if (mr[first + i].nReg != -1) { + // If this was dirty, the combined reg is now dirty. + if (nr[mr[first + i].nReg].isDirty) + nr[dest].isDirty = true; + + // Throw away the other register we're no longer using. + if (i != 0) + DiscardNativeReg(mr[first + i].nReg); + } + + // And set it as using the new one. + mr[first + i].lane = i; + mr[first + i].loc = MIPSLoc::FREG; + mr[first + i].nReg = dest; + } + + if (cur[0] != FromNativeReg(dest)) + emit_->MOVAPS(FromNativeReg(dest), ::R(cur[0])); + + if (dest != nreg) { + nr[dest].mipsReg = first; + nr[nreg].mipsReg = -1; + nr[nreg].isDirty = false; + } + + return true; +} + void X64IRRegCache::SetNativeRegValue(IRNativeReg nreg, uint32_t imm) { X64Reg r = FromNativeReg(nreg); _dbg_assert_(nreg >= 0 && nreg < NUM_X_REGS); diff --git a/Core/MIPS/x86/X64IRRegCache.h b/Core/MIPS/x86/X64IRRegCache.h index fd2a720bf8..8a21f563d8 100644 --- a/Core/MIPS/x86/X64IRRegCache.h +++ b/Core/MIPS/x86/X64IRRegCache.h @@ -120,6 +120,9 @@ protected: bool TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) override; private: + bool TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg first, int oldlanes); + bool Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg first, int lanes); + IRNativeReg GPRToNativeReg(Gen::X64Reg r) { return (IRNativeReg)r; } From 685d2acffe6c58a5199b48311126ee87a25e5cec Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 24 Sep 2023 17:31:25 -0700 Subject: [PATCH 4/5] x86jit: Retain old lanes when there's space. --- Core/MIPS/IR/IRRegCache.cpp | 21 ++++++++-- Core/MIPS/IR/IRRegCache.h | 3 +- Core/MIPS/x86/X64IRRegCache.cpp | 73 ++++++++++++++++++++++++++------- 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/Core/MIPS/IR/IRRegCache.cpp b/Core/MIPS/IR/IRRegCache.cpp index 31a0da4d7d..38c2fe2bd5 100644 --- a/Core/MIPS/IR/IRRegCache.cpp +++ b/Core/MIPS/IR/IRRegCache.cpp @@ -406,12 +406,12 @@ IRNativeReg IRNativeRegCacheBase::FindFreeReg(MIPSLoc type, MIPSMap flags) const bool IRNativeRegCacheBase::IsGPRClobbered(IRReg gpr) const { _dbg_assert_(IsValidGPR(gpr)); - return IsRegClobbered(MIPSLoc::REG, MIPSMap::INIT, gpr); + return IsRegClobbered(MIPSLoc::REG, gpr); } bool IRNativeRegCacheBase::IsFPRClobbered(IRReg fpr) const { _dbg_assert_(IsValidFPR(fpr)); - return IsRegClobbered(MIPSLoc::FREG, MIPSMap::INIT, fpr + 32); + return IsRegClobbered(MIPSLoc::FREG, fpr + 32); } IRUsage IRNativeRegCacheBase::GetNextRegUsage(const IRSituation &info, MIPSLoc type, IRReg r) const { @@ -423,7 +423,7 @@ IRUsage IRNativeRegCacheBase::GetNextRegUsage(const IRSituation &info, MIPSLoc t return IRUsage::UNKNOWN; } -bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) const { +bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, IRReg r) const { static const int UNUSED_LOOKAHEAD_OPS = 30; IRSituation info; @@ -450,6 +450,21 @@ bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) return false; } +bool IRNativeRegCacheBase::IsRegRead(MIPSLoc type, IRReg first) const { + static const int UNUSED_LOOKAHEAD_OPS = 30; + + IRSituation info; + info.lookaheadCount = UNUSED_LOOKAHEAD_OPS; + // We look starting one ahead, unlike spilling. + info.currentIndex = irIndex_ + 1; + info.instructions = irBlock_->GetInstructions(); + info.numInstructions = irBlock_->GetNumInstructions(); + + // Note: this intentionally doesn't look at the full reg, only the lane. + IRUsage usage = GetNextRegUsage(info, type, first); + return usage == IRUsage::READ; +} + IRNativeReg IRNativeRegCacheBase::FindBestToSpill(MIPSLoc type, MIPSMap flags, bool unusedOnly, bool *clobbered) const { int allocCount = 0, base = 0; const int *allocOrder = GetAllocationOrder(type, flags, allocCount, base); diff --git a/Core/MIPS/IR/IRRegCache.h b/Core/MIPS/IR/IRRegCache.h index 07067cbdcd..4301886b44 100644 --- a/Core/MIPS/IR/IRRegCache.h +++ b/Core/MIPS/IR/IRRegCache.h @@ -239,7 +239,8 @@ protected: void SetSpillLockIRIndex(IRReg reg, int index); int GetMipsRegOffset(IRReg r); - bool IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) const; + bool IsRegClobbered(MIPSLoc type, IRReg r) const; + bool IsRegRead(MIPSLoc type, IRReg r) const; IRUsage GetNextRegUsage(const IRSituation &info, MIPSLoc type, IRReg r) const; bool IsValidGPR(IRReg r) const; diff --git a/Core/MIPS/x86/X64IRRegCache.cpp b/Core/MIPS/x86/X64IRRegCache.cpp index bd991c5018..6a38303286 100644 --- a/Core/MIPS/x86/X64IRRegCache.cpp +++ b/Core/MIPS/x86/X64IRRegCache.cpp @@ -480,12 +480,52 @@ bool X64IRRegCache::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLo } bool X64IRRegCache::TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg first, int oldlanes) { - // Okay, start by storing if dirty. - // TODO: Consider moving the others to free regs if available? Likely will be wanted later. IRReg oldfirst = nr[nreg].mipsReg; - if (nr[nreg].isDirty) { + + // Is it worth preserving any of the old regs? + int numKept = 0; + for (int i = 0; i < oldlanes; ++i) { + // Skip whichever one this is extracting. + if (oldfirst + i == first) + continue; + // If 0 isn't being transfered, easy to keep in its original reg. + if (i == 0 && dest != nreg) { + numKept++; + continue; + } + + IRNativeReg freeReg = FindFreeReg(MIPSLoc::FREG, MIPSMap::INIT); + if (freeReg != -1 && IsRegRead(MIPSLoc::FREG, oldfirst + i)) { + // If there's one free, use it. Don't modify nreg, though. + u8 shuf = VFPU_SWIZZLE(i, i, i, i); + if (i == 0) { + emit_->MOVAPS(FromNativeReg(freeReg), ::R(FromNativeReg(nreg))); + } else if (cpu_info.bAVX) { + emit_->VPERMILPS(128, FromNativeReg(freeReg), ::R(FromNativeReg(nreg)), shuf); + } else if (i == 2) { + emit_->MOVHLPS(FromNativeReg(freeReg), FromNativeReg(nreg)); + } else { + emit_->MOVAPS(FromNativeReg(freeReg), ::R(FromNativeReg(nreg))); + emit_->SHUFPS(FromNativeReg(freeReg), ::R(FromNativeReg(freeReg)), shuf); + } + + // Update accounting. + nr[freeReg].isDirty = nr[nreg].isDirty; + nr[freeReg].mipsReg = oldfirst + i; + mr[oldfirst + i].lane = -1; + mr[oldfirst + i].nReg = freeReg; + numKept++; + } + } + + // Unless all other lanes were kept, store. + if (nr[nreg].isDirty && numKept < oldlanes - 1) { StoreNativeReg(nreg, oldfirst, oldlanes); - nr[nreg].isDirty = false; + // Set false even for regs that were split out, since they were flushed too. + for (int i = 0; i < oldlanes; ++i) { + if (mr[oldfirst + i].nReg != -1) + nr[mr[oldfirst + i].nReg].isDirty = false; + } } // Next, shuffle the desired element into first place. @@ -494,23 +534,24 @@ bool X64IRRegCache::TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg fir emit_->VPERMILPS(128, FromNativeReg(dest), ::R(FromNativeReg(nreg)), shuf); } else if (mr[first].lane <= 0 && dest != nreg) { emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + } else if (mr[first].lane == 2) { + emit_->MOVHLPS(FromNativeReg(dest), FromNativeReg(nreg)); } else if (mr[first].lane > 0) { - if (mr[first].lane == 2) { - emit_->MOVHLPS(FromNativeReg(dest), FromNativeReg(nreg)); - } else { - if (dest != nreg) - emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); - emit_->SHUFPS(FromNativeReg(dest), ::R(FromNativeReg(dest)), shuf); - } + if (dest != nreg) + emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + emit_->SHUFPS(FromNativeReg(dest), ::R(FromNativeReg(dest)), shuf); } // Now update accounting. for (int i = 0; i < oldlanes; ++i) { auto &mreg = mr[oldfirst + i]; if (oldfirst + i == first) { - mreg.lane = 0; + mreg.lane = -1; mreg.nReg = dest; - } else { + } else if (mreg.nReg == nreg && i == 0 && nreg != dest) { + // Still in the same register, but no longer a vec. + mreg.lane = -1; + } else if (mreg.nReg == nreg) { // No longer in a register. mreg.nReg = -1; mreg.lane = -1; @@ -520,8 +561,10 @@ bool X64IRRegCache::TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg fir if (dest != nreg) { nr[dest].isDirty = nr[nreg].isDirty; - nr[nreg].mipsReg = -1; - nr[nreg].isDirty = false; + if (oldfirst == first) { + nr[nreg].mipsReg = -1; + nr[nreg].isDirty = false; + } } nr[dest].mipsReg = first; From 38e5b33a538db01cf4eaa4d3180916c6f34fc4e8 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Mon, 25 Sep 2023 20:19:11 -0700 Subject: [PATCH 5/5] x86jit: Prefer BLENDPS to INSERTPS. It's faster, this performs better. --- Core/MIPS/x86/X64IRRegCache.cpp | 96 +++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 28 deletions(-) diff --git a/Core/MIPS/x86/X64IRRegCache.cpp b/Core/MIPS/x86/X64IRRegCache.cpp index 6a38303286..cfbb57712e 100644 --- a/Core/MIPS/x86/X64IRRegCache.cpp +++ b/Core/MIPS/x86/X64IRRegCache.cpp @@ -574,7 +574,7 @@ bool X64IRRegCache::TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg fir bool X64IRRegCache::Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg first, int lanes) { X64Reg cur[4]{}; int numInRegs = 0; - int numDirty = 0; + u8 blendMask = 0; for (int i = 0; i < lanes; ++i) { if (mr[first + i].lane != -1 || (i != 0 && mr[first + i].spillLockIRIndex >= irIndex_)) { // Can't do it, either double mapped or overlapping vec. @@ -583,11 +583,10 @@ bool X64IRRegCache::Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg fir if (mr[first + i].nReg == -1) { cur[i] = INVALID_REG; + blendMask |= 1 << i; } else { cur[i] = FromNativeReg(mr[first + i].nReg); numInRegs++; - if (nr[cur[i]].isDirty) - numDirty++; } } @@ -595,49 +594,90 @@ bool X64IRRegCache::Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg fir if (numInRegs == 0) return false; - // If everything's currently in a reg, move it into this reg. - if (lanes == 4) { - if (cur[0] == INVALID_REG) { + // Move things together into a reg. + if (lanes == 4 && cpu_info.bSSE4_1 && numInRegs == 1 && (first & 3) == 0) { + // Use a blend to grab the rest. BLENDPS is pretty good. + if (cpu_info.bAVX && nreg != dest) { + if (cur[0] == INVALID_REG) { + // Broadcast to all lanes, then blend from memory to replace. + emit_->VPERMILPS(128, FromNativeReg(dest), ::R(FromNativeReg(nreg)), 0); + emit_->BLENDPS(FromNativeReg(dest), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } else { + emit_->VBLENDPS(128, FromNativeReg(dest), FromNativeReg(nreg), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } cur[0] = FromNativeReg(dest); - emit_->MOVSS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); - numInRegs++; + } else { + if (cur[0] == INVALID_REG) + emit_->SHUFPS(FromNativeReg(nreg), ::R(FromNativeReg(nreg)), 0); + emit_->BLENDPS(FromNativeReg(nreg), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + // If this is not dest, it'll get moved there later. + cur[0] = FromNativeReg(nreg); } - - // A lot of other methods are possible, but seem to make things slower in practice. - if (numInRegs == 4) { + } else if (lanes == 4) { + if (blendMask == 0) { // y = yw##, x = xz##, x = xyzw. emit_->UNPCKLPS(cur[1], ::R(cur[3])); emit_->UNPCKLPS(cur[0], ::R(cur[2])); emit_->UNPCKLPS(cur[0], ::R(cur[1])); - } else if (numInRegs == 2 && cur[1] != INVALID_REG) { + } else if (blendMask == 0b1100) { // x = xy##, then load zw. emit_->UNPCKLPS(cur[0], ::R(cur[1])); emit_->MOVHPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2))); - } else if (cur[1] != INVALID_REG && cur[2] != INVALID_REG) { - // x = xz##, z=w###, y=yw##, x=xyzw. + } else if (blendMask == 0b1010 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // x = x#z#, x = xyzw. + emit_->SHUFPS(cur[0], ::R(cur[2]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } else if (blendMask == 0b0110 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // x = x##w, x = xyzw. + emit_->SHUFPS(cur[0], ::R(cur[3]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } else if (blendMask == 0b1001 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // y = #yz#, y = xyzw. + emit_->SHUFPS(cur[1], ::R(cur[2]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[1], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + // Will be moved to dest as needed. + cur[0] = cur[1]; + } else if (blendMask == 0b0101 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // y = #y#w, y = xyzw. + emit_->SHUFPS(cur[1], ::R(cur[3]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[1], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + // Will be moved to dest as needed. + cur[0] = cur[1]; + } else if (blendMask == 0b1000) { + // x = xz##, z = w###, y = yw##, x = xyzw. emit_->UNPCKLPS(cur[0], ::R(cur[2])); emit_->MOVSS(cur[2], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 3))); emit_->UNPCKLPS(cur[1], ::R(cur[2])); emit_->UNPCKLPS(cur[0], ::R(cur[1])); - } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[1] != INVALID_REG && cur[3] != INVALID_REG) { - // y = yw##, load z into x[1], x = xyzw. + } else if (blendMask == 0b0100) { + // y = yw##, w = z###, x = xz##, x = xyzw. emit_->UNPCKLPS(cur[1], ::R(cur[3])); - emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2)), 1); + emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2))); + emit_->UNPCKLPS(cur[0], ::R(cur[3])); emit_->UNPCKLPS(cur[0], ::R(cur[1])); - } else if (cpu_info.bSSE4_1 && numDirty != 0 && cur[2] != INVALID_REG && cur[3] != INVALID_REG) { - // load y to x[1], z = zw##, x = xyzw. - emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); + } else if (blendMask == 0b0010) { + // z = zw##, w = y###, x = xy##, x = xyzw. emit_->UNPCKLPS(cur[2], ::R(cur[3])); + emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1))); + emit_->UNPCKLPS(cur[0], ::R(cur[3])); emit_->MOVLHPS(cur[0], cur[2]); - } else if (cpu_info.bSSE4_1) { - // TODO: This might be worse than flushing depending? - for (int i = 1; i < 4; ++i) { - if (cur[i] == INVALID_REG) - emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + i)), i); - else - emit_->INSERTPS(cur[0], ::R(cur[i]), i, 0); - } + } else if (blendMask == 0b0001) { + // y = yw##, w = x###, w = xz##, w = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); + emit_->UNPCKLPS(cur[3], ::R(cur[2])); + emit_->UNPCKLPS(cur[3], ::R(cur[1])); + // Will be moved to dest as needed. + cur[0] = cur[3]; + } else if (blendMask == 0b0011) { + // z = zw##, w = xy##, w = xyzw. + emit_->UNPCKLPS(cur[2], ::R(cur[3])); + emit_->MOVLPS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); + emit_->MOVLHPS(cur[3], cur[2]); + // Will be moved to dest as needed. + cur[0] = cur[3]; } else { + // This must mean no SSE4, and numInRegs <= 2 in trickier cases. return false; } } else if (lanes == 2) {