diff --git a/Common/x64Emitter.cpp b/Common/x64Emitter.cpp index c2a5ba8c4d..814fc7e0d6 100644 --- a/Common/x64Emitter.cpp +++ b/Common/x64Emitter.cpp @@ -1697,7 +1697,6 @@ void XEmitter::MOVMSKPD(X64Reg dest, OpArg arg) {WriteSSEOp(0x66, 0x50, dest, ar void XEmitter::LDDQU(X64Reg dest, OpArg arg) {WriteSSEOp(0xF2, sseLDDQU, dest, arg);} // For integer data only -// THESE TWO ARE UNTESTED. void XEmitter::UNPCKLPS(X64Reg dest, OpArg arg) {WriteSSEOp(0x00, 0x14, dest, arg);} void XEmitter::UNPCKHPS(X64Reg dest, OpArg arg) {WriteSSEOp(0x00, 0x15, dest, arg);} @@ -1892,6 +1891,9 @@ void XEmitter::PTEST(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x3817, dest void XEmitter::PACKUSDW(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x382b, dest, arg);} void XEmitter::DPPS(X64Reg dest, OpArg arg, u8 mask) {WriteSSE41Op(0x66, 0x3A40, dest, arg, 1); Write8(mask);} +void XEmitter::INSERTPS(X64Reg dest, OpArg arg, u8 dstsubreg, u8 srcsubreg, u8 zmask) { WriteSSE41Op(0x66, 0x3A21, dest, arg, 1); Write8((srcsubreg << 6) | (dstsubreg << 4) | zmask); } +void XEmitter::EXTRACTPS(OpArg dest, X64Reg arg, u8 subreg) { WriteSSE41Op(0x66, 0x3A17, arg, dest, 1); Write8(subreg); } + void XEmitter::PMINSB(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x3838, dest, arg);} void XEmitter::PMINSD(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x3839, dest, arg);} void XEmitter::PMINUW(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x383a, dest, arg);} @@ -2084,7 +2086,7 @@ void XEmitter::VCVTTPD2DQ(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, void XEmitter::VCVTTSS2SI(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(0, 0xF3, 0x2C, regOp1, arg, 0, bits == 64 ? 1 : 0); } void XEmitter::VCVTTSD2SI(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(0, 0xF2, 0x2C, regOp1, arg, 0, bits == 64 ? 1 : 0); } void XEmitter::VEXTRACTPS(OpArg arg, X64Reg regOp1, u8 subreg) { WriteAVXOp(0, 0x66, 0x3A17, regOp1, arg, 1); Write8(subreg); } -void XEmitter::VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 subreg) { WriteAVXOp(0, 0x66, 0x3A21, regOp1, regOp2, arg, 1); Write8(subreg); } +void XEmitter::VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 dstsubreg, u8 srcsubreg, u8 zmask) { WriteAVXOp(0, 0x66, 0x3A21, regOp1, regOp2, arg, 1); Write8((srcsubreg << 6) | (dstsubreg << 4) | zmask); } void XEmitter::VLDDQU(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0xF2, sseLDDQU, regOp1, arg); } void XEmitter::VMOVAPS(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0x00, sseMOVAPfromRM, regOp1, arg); } void XEmitter::VMOVAPD(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0x66, sseMOVAPfromRM, regOp1, arg); } diff --git a/Common/x64Emitter.h b/Common/x64Emitter.h index 16f30a35b0..832ed767cb 100644 --- a/Common/x64Emitter.h +++ b/Common/x64Emitter.h @@ -684,12 +684,14 @@ public: // SSE4: Further horizontal operations - dot products. These are weirdly flexible, the arg contains both a read mask and a write "mask". void DPPD(X64Reg dest, OpArg src, u8 arg); - - // These are probably useful for VFPU emulation. - void INSERTPS(X64Reg dest, OpArg src, u8 arg); - void EXTRACTPS(OpArg dest, X64Reg src, u8 arg); #endif + // SSE4: Insert and extract for floats. + // Note: insert from memory or an XMM. + void INSERTPS(X64Reg dest, OpArg arg, u8 dstsubreg, u8 srcsubreg = 0, u8 zmask = 0); + // Extract to memory or GPR. + void EXTRACTPS(OpArg dest, X64Reg arg, u8 subreg); + // SSE3: Horizontal operations in SIMD registers. Very slow! shufps-based code beats it handily on Ivy. void HADDPS(X64Reg dest, OpArg src); @@ -1040,7 +1042,7 @@ public: // Can only extract from the low 128 bits. void VEXTRACTPS(OpArg arg, X64Reg regOp1, u8 subreg); // Can only insert into the low 128 bits, zeros upper bits. Inserts from XMM. - void VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 subreg); + void VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 dstsubreg, u8 srcsubreg = 0, u8 zmask = 0); void VLDDQU(int bits, X64Reg regOp1, OpArg arg); void VMOVAPS(int bits, X64Reg regOp1, OpArg arg); void VMOVAPD(int bits, X64Reg regOp1, OpArg arg); diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.cpp b/Core/MIPS/ARM64/Arm64IRRegCache.cpp index 0420a808ce..f48207fa5f 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.cpp +++ b/Core/MIPS/ARM64/Arm64IRRegCache.cpp @@ -347,7 +347,7 @@ void Arm64IRRegCache::AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) { } } -bool Arm64IRRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) { +bool Arm64IRRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) { // No special flags, skip the check for a little speed. return true; } diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.h b/Core/MIPS/ARM64/Arm64IRRegCache.h index 3a9bf77ab9..9f0b0cbbac 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.h +++ b/Core/MIPS/ARM64/Arm64IRRegCache.h @@ -86,7 +86,7 @@ protected: const int *GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &count, int &base) const override; void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) override; - bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) override; + bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) override; void LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override; diff --git a/Core/MIPS/IR/IRRegCache.cpp b/Core/MIPS/IR/IRRegCache.cpp index b7ce1e265b..38c2fe2bd5 100644 --- a/Core/MIPS/IR/IRRegCache.cpp +++ b/Core/MIPS/IR/IRRegCache.cpp @@ -406,12 +406,12 @@ IRNativeReg IRNativeRegCacheBase::FindFreeReg(MIPSLoc type, MIPSMap flags) const bool IRNativeRegCacheBase::IsGPRClobbered(IRReg gpr) const { _dbg_assert_(IsValidGPR(gpr)); - return IsRegClobbered(MIPSLoc::REG, MIPSMap::INIT, gpr); + return IsRegClobbered(MIPSLoc::REG, gpr); } bool IRNativeRegCacheBase::IsFPRClobbered(IRReg fpr) const { _dbg_assert_(IsValidFPR(fpr)); - return IsRegClobbered(MIPSLoc::FREG, MIPSMap::INIT, fpr + 32); + return IsRegClobbered(MIPSLoc::FREG, fpr + 32); } IRUsage IRNativeRegCacheBase::GetNextRegUsage(const IRSituation &info, MIPSLoc type, IRReg r) const { @@ -423,7 +423,7 @@ IRUsage IRNativeRegCacheBase::GetNextRegUsage(const IRSituation &info, MIPSLoc t return IRUsage::UNKNOWN; } -bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) const { +bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, IRReg r) const { static const int UNUSED_LOOKAHEAD_OPS = 30; IRSituation info; @@ -450,6 +450,21 @@ bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) return false; } +bool IRNativeRegCacheBase::IsRegRead(MIPSLoc type, IRReg first) const { + static const int UNUSED_LOOKAHEAD_OPS = 30; + + IRSituation info; + info.lookaheadCount = UNUSED_LOOKAHEAD_OPS; + // We look starting one ahead, unlike spilling. + info.currentIndex = irIndex_ + 1; + info.instructions = irBlock_->GetInstructions(); + info.numInstructions = irBlock_->GetNumInstructions(); + + // Note: this intentionally doesn't look at the full reg, only the lane. + IRUsage usage = GetNextRegUsage(info, type, first); + return usage == IRUsage::READ; +} + IRNativeReg IRNativeRegCacheBase::FindBestToSpill(MIPSLoc type, MIPSMap flags, bool unusedOnly, bool *clobbered) const { int allocCount = 0, base = 0; const int *allocOrder = GetAllocationOrder(type, flags, allocCount, base); @@ -501,7 +516,7 @@ IRNativeReg IRNativeRegCacheBase::FindBestToSpill(MIPSLoc type, MIPSMap flags, b return -1; } -bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) { +bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) { int allocCount = 0, base = 0; const int *allocOrder = GetAllocationOrder(type, flags, allocCount, base); @@ -514,6 +529,11 @@ bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, return false; } +bool IRNativeRegCacheBase::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) { + // To be overridden if the backend supports transfers. + return false; +} + void IRNativeRegCacheBase::DiscardNativeReg(IRNativeReg nreg) { _assert_msg_(nreg >= 0 && nreg < config_.totalNativeRegs, "DiscardNativeReg on invalid register %d", nreg); if (nr[nreg].mipsReg != IRREG_INVALID) { @@ -930,11 +950,14 @@ IRNativeReg IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRReg first, int la case MIPSLoc::REG: if (type != MIPSLoc::REG) { nreg = AllocateReg(type, flags); - } else if (!IsNativeRegCompatible(nreg, type, flags)) { + } else if (!IsNativeRegCompatible(nreg, type, flags, lanes)) { // If it's not compatible, we'll need to reallocate. - // TODO: Could do a transfer and avoid memory flush. - FlushNativeReg(nreg); - nreg = AllocateReg(type, flags); + if (TransferNativeReg(nreg, -1, type, first, lanes, flags)) { + nreg = mr[first].nReg; + } else { + FlushNativeReg(nreg); + nreg = AllocateReg(type, flags); + } } break; @@ -942,9 +965,13 @@ IRNativeReg IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRReg first, int la case MIPSLoc::VREG: if (type != mr[first].loc) { nreg = AllocateReg(type, flags); - } else if (!IsNativeRegCompatible(nreg, type, flags)) { - FlushNativeReg(nreg); - nreg = AllocateReg(type, flags); + } else if (!IsNativeRegCompatible(nreg, type, flags, lanes)) { + if (TransferNativeReg(nreg, -1, type, first, lanes, flags)) { + nreg = mr[first].nReg; + } else { + FlushNativeReg(nreg); + nreg = AllocateReg(type, flags); + } } break; @@ -981,10 +1008,13 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi _assert_msg_(!mreg.isStatic, "Cannot MapNativeReg a static reg mismatch"); if ((flags & MIPSMap::NOINIT) != MIPSMap::NOINIT) { // If we need init, we have to flush mismatches. - // TODO: Do a shuffle if interior only? - // TODO: We may also be motivated to have multiple read-only "views" or an IRReg. - // For example Vec4Scale v0..v3, v0..v3, v3 - FlushNativeReg(mreg.nReg); + if (!TransferNativeReg(mreg.nReg, nreg, type, first, lanes, flags)) { + // TODO: We may also be motivated to have multiple read-only "views" or an IRReg. + // For example Vec4Scale v0..v3, v0..v3, v3 + FlushNativeReg(mreg.nReg); + } + // The mismatch has been "resolved" now. + mismatch = false; } else if (oldlanes != 1) { // Even if we don't care about the current contents, we can't discard outside. bool extendsBefore = oldlane > i; @@ -1017,6 +1047,9 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi DiscardNativeReg(mreg.nReg); else FlushNativeReg(mreg.nReg); + + // That took care of the mismatch, either by clobber or flush. + mismatch = false; } } } @@ -1027,8 +1060,8 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi if ((flags & MIPSMap::NOINIT) != MIPSMap::NOINIT) { // We better not be trying to map to a different nreg if it's in one now. // This might happen on some sort of transfer... - // TODO: Make a direct transfer, i.e. FREG -> VREG? - FlushNativeReg(mreg.nReg); + if (!TransferNativeReg(mreg.nReg, nreg, type, first, lanes, flags)) + FlushNativeReg(mreg.nReg); } else { DiscardNativeReg(mreg.nReg); } diff --git a/Core/MIPS/IR/IRRegCache.h b/Core/MIPS/IR/IRRegCache.h index c85bb41848..4301886b44 100644 --- a/Core/MIPS/IR/IRRegCache.h +++ b/Core/MIPS/IR/IRRegCache.h @@ -209,13 +209,14 @@ protected: IRNativeReg AllocateReg(MIPSLoc type, MIPSMap flags); IRNativeReg FindFreeReg(MIPSLoc type, MIPSMap flags) const; IRNativeReg FindBestToSpill(MIPSLoc type, MIPSMap flags, bool unusedOnly, bool *clobbered) const; - virtual bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags); + virtual bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes); virtual void DiscardNativeReg(IRNativeReg nreg); virtual void FlushNativeReg(IRNativeReg nreg); virtual void DiscardReg(IRReg mreg); virtual void FlushReg(IRReg mreg); virtual void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state); virtual void MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg first, int lanes, MIPSMap flags); + virtual bool TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags); virtual IRNativeReg MapNativeReg(MIPSLoc type, IRReg first, int lanes, MIPSMap flags); IRNativeReg MapNativeRegAsPointer(IRReg gpr); @@ -238,7 +239,8 @@ protected: void SetSpillLockIRIndex(IRReg reg, int index); int GetMipsRegOffset(IRReg r); - bool IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) const; + bool IsRegClobbered(MIPSLoc type, IRReg r) const; + bool IsRegRead(MIPSLoc type, IRReg r) const; IRUsage GetNextRegUsage(const IRSituation &info, MIPSLoc type, IRReg r) const; bool IsValidGPR(IRReg r) const; diff --git a/Core/MIPS/RiscV/RiscVRegCache.cpp b/Core/MIPS/RiscV/RiscVRegCache.cpp index 7a3e6505cb..25528aa3aa 100644 --- a/Core/MIPS/RiscV/RiscVRegCache.cpp +++ b/Core/MIPS/RiscV/RiscVRegCache.cpp @@ -303,11 +303,11 @@ void RiscVRegCache::AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) { } } -bool RiscVRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) { +bool RiscVRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) { // No special flags except VREG, skip the check for a little speed. if (type != MIPSLoc::VREG) return true; - return IRNativeRegCacheBase::IsNativeRegCompatible(nreg, type, flags); + return IRNativeRegCacheBase::IsNativeRegCompatible(nreg, type, flags, lanes); } void RiscVRegCache::LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) { diff --git a/Core/MIPS/RiscV/RiscVRegCache.h b/Core/MIPS/RiscV/RiscVRegCache.h index facfa52195..e0075f2c61 100644 --- a/Core/MIPS/RiscV/RiscVRegCache.h +++ b/Core/MIPS/RiscV/RiscVRegCache.h @@ -76,7 +76,7 @@ protected: const int *GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &count, int &base) const override; void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) override; - bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) override; + bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) override; void LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override; diff --git a/Core/MIPS/x86/X64IRRegCache.cpp b/Core/MIPS/x86/X64IRRegCache.cpp index ee176546c8..cfbb57712e 100644 --- a/Core/MIPS/x86/X64IRRegCache.cpp +++ b/Core/MIPS/x86/X64IRRegCache.cpp @@ -453,6 +453,275 @@ void X64IRRegCache::StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) { } } +bool X64IRRegCache::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) { + bool allowed = !mr[nr[nreg].mipsReg].isStatic; + // There's currently no support for non-XMMs here. + allowed = allowed && type == MIPSLoc::FREG; + + if (dest == -1) + dest = nreg; + + if (allowed && (flags == MIPSMap::INIT || flags == MIPSMap::DIRTY)) { + // Alright, changing lane count (possibly including lane position.) + IRReg oldfirst = nr[nreg].mipsReg; + int oldlanes = 0; + while (mr[oldfirst + oldlanes].nReg == nreg) + oldlanes++; + _assert_msg_(oldlanes != 0, "TransferNativeReg encountered nreg mismatch"); + _assert_msg_(oldlanes != lanes, "TransferNativeReg transfer to same lanecount, misaligned?"); + + if (lanes == 1 && TransferVecTo1(nreg, dest, first, oldlanes)) + return true; + if (oldlanes == 1 && Transfer1ToVec(nreg, dest, first, lanes)) + return true; + } + + return IRNativeRegCacheBase::TransferNativeReg(nreg, dest, type, first, lanes, flags); +} + +bool X64IRRegCache::TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg first, int oldlanes) { + IRReg oldfirst = nr[nreg].mipsReg; + + // Is it worth preserving any of the old regs? + int numKept = 0; + for (int i = 0; i < oldlanes; ++i) { + // Skip whichever one this is extracting. + if (oldfirst + i == first) + continue; + // If 0 isn't being transfered, easy to keep in its original reg. + if (i == 0 && dest != nreg) { + numKept++; + continue; + } + + IRNativeReg freeReg = FindFreeReg(MIPSLoc::FREG, MIPSMap::INIT); + if (freeReg != -1 && IsRegRead(MIPSLoc::FREG, oldfirst + i)) { + // If there's one free, use it. Don't modify nreg, though. + u8 shuf = VFPU_SWIZZLE(i, i, i, i); + if (i == 0) { + emit_->MOVAPS(FromNativeReg(freeReg), ::R(FromNativeReg(nreg))); + } else if (cpu_info.bAVX) { + emit_->VPERMILPS(128, FromNativeReg(freeReg), ::R(FromNativeReg(nreg)), shuf); + } else if (i == 2) { + emit_->MOVHLPS(FromNativeReg(freeReg), FromNativeReg(nreg)); + } else { + emit_->MOVAPS(FromNativeReg(freeReg), ::R(FromNativeReg(nreg))); + emit_->SHUFPS(FromNativeReg(freeReg), ::R(FromNativeReg(freeReg)), shuf); + } + + // Update accounting. + nr[freeReg].isDirty = nr[nreg].isDirty; + nr[freeReg].mipsReg = oldfirst + i; + mr[oldfirst + i].lane = -1; + mr[oldfirst + i].nReg = freeReg; + numKept++; + } + } + + // Unless all other lanes were kept, store. + if (nr[nreg].isDirty && numKept < oldlanes - 1) { + StoreNativeReg(nreg, oldfirst, oldlanes); + // Set false even for regs that were split out, since they were flushed too. + for (int i = 0; i < oldlanes; ++i) { + if (mr[oldfirst + i].nReg != -1) + nr[mr[oldfirst + i].nReg].isDirty = false; + } + } + + // Next, shuffle the desired element into first place. + u8 shuf = VFPU_SWIZZLE(mr[first].lane, mr[first].lane, mr[first].lane, mr[first].lane); + if (mr[first].lane > 0 && cpu_info.bAVX && dest != nreg) { + emit_->VPERMILPS(128, FromNativeReg(dest), ::R(FromNativeReg(nreg)), shuf); + } else if (mr[first].lane <= 0 && dest != nreg) { + emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + } else if (mr[first].lane == 2) { + emit_->MOVHLPS(FromNativeReg(dest), FromNativeReg(nreg)); + } else if (mr[first].lane > 0) { + if (dest != nreg) + emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg))); + emit_->SHUFPS(FromNativeReg(dest), ::R(FromNativeReg(dest)), shuf); + } + + // Now update accounting. + for (int i = 0; i < oldlanes; ++i) { + auto &mreg = mr[oldfirst + i]; + if (oldfirst + i == first) { + mreg.lane = -1; + mreg.nReg = dest; + } else if (mreg.nReg == nreg && i == 0 && nreg != dest) { + // Still in the same register, but no longer a vec. + mreg.lane = -1; + } else if (mreg.nReg == nreg) { + // No longer in a register. + mreg.nReg = -1; + mreg.lane = -1; + mreg.loc = MIPSLoc::MEM; + } + } + + if (dest != nreg) { + nr[dest].isDirty = nr[nreg].isDirty; + if (oldfirst == first) { + nr[nreg].mipsReg = -1; + nr[nreg].isDirty = false; + } + } + nr[dest].mipsReg = first; + + return true; +} + +bool X64IRRegCache::Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg first, int lanes) { + X64Reg cur[4]{}; + int numInRegs = 0; + u8 blendMask = 0; + for (int i = 0; i < lanes; ++i) { + if (mr[first + i].lane != -1 || (i != 0 && mr[first + i].spillLockIRIndex >= irIndex_)) { + // Can't do it, either double mapped or overlapping vec. + return false; + } + + if (mr[first + i].nReg == -1) { + cur[i] = INVALID_REG; + blendMask |= 1 << i; + } else { + cur[i] = FromNativeReg(mr[first + i].nReg); + numInRegs++; + } + } + + // Shouldn't happen, this should only get called to transfer one in a reg. + if (numInRegs == 0) + return false; + + // Move things together into a reg. + if (lanes == 4 && cpu_info.bSSE4_1 && numInRegs == 1 && (first & 3) == 0) { + // Use a blend to grab the rest. BLENDPS is pretty good. + if (cpu_info.bAVX && nreg != dest) { + if (cur[0] == INVALID_REG) { + // Broadcast to all lanes, then blend from memory to replace. + emit_->VPERMILPS(128, FromNativeReg(dest), ::R(FromNativeReg(nreg)), 0); + emit_->BLENDPS(FromNativeReg(dest), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } else { + emit_->VBLENDPS(128, FromNativeReg(dest), FromNativeReg(nreg), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } + cur[0] = FromNativeReg(dest); + } else { + if (cur[0] == INVALID_REG) + emit_->SHUFPS(FromNativeReg(nreg), ::R(FromNativeReg(nreg)), 0); + emit_->BLENDPS(FromNativeReg(nreg), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + // If this is not dest, it'll get moved there later. + cur[0] = FromNativeReg(nreg); + } + } else if (lanes == 4) { + if (blendMask == 0) { + // y = yw##, x = xz##, x = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->UNPCKLPS(cur[0], ::R(cur[2])); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (blendMask == 0b1100) { + // x = xy##, then load zw. + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + emit_->MOVHPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2))); + } else if (blendMask == 0b1010 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // x = x#z#, x = xyzw. + emit_->SHUFPS(cur[0], ::R(cur[2]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } else if (blendMask == 0b0110 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // x = x##w, x = xyzw. + emit_->SHUFPS(cur[0], ::R(cur[3]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + } else if (blendMask == 0b1001 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // y = #yz#, y = xyzw. + emit_->SHUFPS(cur[1], ::R(cur[2]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[1], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + // Will be moved to dest as needed. + cur[0] = cur[1]; + } else if (blendMask == 0b0101 && cpu_info.bSSE4_1 && (first & 3) == 0) { + // y = #y#w, y = xyzw. + emit_->SHUFPS(cur[1], ::R(cur[3]), VFPU_SWIZZLE(0, 0, 0, 0)); + emit_->BLENDPS(cur[1], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask); + // Will be moved to dest as needed. + cur[0] = cur[1]; + } else if (blendMask == 0b1000) { + // x = xz##, z = w###, y = yw##, x = xyzw. + emit_->UNPCKLPS(cur[0], ::R(cur[2])); + emit_->MOVSS(cur[2], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 3))); + emit_->UNPCKLPS(cur[1], ::R(cur[2])); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (blendMask == 0b0100) { + // y = yw##, w = z###, x = xz##, x = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2))); + emit_->UNPCKLPS(cur[0], ::R(cur[3])); + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (blendMask == 0b0010) { + // z = zw##, w = y###, x = xy##, x = xyzw. + emit_->UNPCKLPS(cur[2], ::R(cur[3])); + emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1))); + emit_->UNPCKLPS(cur[0], ::R(cur[3])); + emit_->MOVLHPS(cur[0], cur[2]); + } else if (blendMask == 0b0001) { + // y = yw##, w = x###, w = xz##, w = xyzw. + emit_->UNPCKLPS(cur[1], ::R(cur[3])); + emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); + emit_->UNPCKLPS(cur[3], ::R(cur[2])); + emit_->UNPCKLPS(cur[3], ::R(cur[1])); + // Will be moved to dest as needed. + cur[0] = cur[3]; + } else if (blendMask == 0b0011) { + // z = zw##, w = xy##, w = xyzw. + emit_->UNPCKLPS(cur[2], ::R(cur[3])); + emit_->MOVLPS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0))); + emit_->MOVLHPS(cur[3], cur[2]); + // Will be moved to dest as needed. + cur[0] = cur[3]; + } else { + // This must mean no SSE4, and numInRegs <= 2 in trickier cases. + return false; + } + } else if (lanes == 2) { + if (cur[0] != INVALID_REG && cur[1] != INVALID_REG) { + emit_->UNPCKLPS(cur[0], ::R(cur[1])); + } else if (cur[0] != INVALID_REG && cpu_info.bSSE4_1) { + emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1); + } else { + return false; + } + } else { + return false; + } + + mr[first].lane = 0; + for (int i = 0; i < lanes; ++i) { + if (mr[first + i].nReg != -1) { + // If this was dirty, the combined reg is now dirty. + if (nr[mr[first + i].nReg].isDirty) + nr[dest].isDirty = true; + + // Throw away the other register we're no longer using. + if (i != 0) + DiscardNativeReg(mr[first + i].nReg); + } + + // And set it as using the new one. + mr[first + i].lane = i; + mr[first + i].loc = MIPSLoc::FREG; + mr[first + i].nReg = dest; + } + + if (cur[0] != FromNativeReg(dest)) + emit_->MOVAPS(FromNativeReg(dest), ::R(cur[0])); + + if (dest != nreg) { + nr[dest].mipsReg = first; + nr[nreg].mipsReg = -1; + nr[nreg].isDirty = false; + } + + return true; +} + void X64IRRegCache::SetNativeRegValue(IRNativeReg nreg, uint32_t imm) { X64Reg r = FromNativeReg(nreg); _dbg_assert_(nreg >= 0 && nreg < NUM_X_REGS); diff --git a/Core/MIPS/x86/X64IRRegCache.h b/Core/MIPS/x86/X64IRRegCache.h index f33e4e8d89..8a21f563d8 100644 --- a/Core/MIPS/x86/X64IRRegCache.h +++ b/Core/MIPS/x86/X64IRRegCache.h @@ -117,8 +117,12 @@ protected: void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override; void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override; void StoreRegValue(IRReg mreg, uint32_t imm) override; + bool TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) override; private: + bool TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg first, int oldlanes); + bool Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg first, int lanes); + IRNativeReg GPRToNativeReg(Gen::X64Reg r) { return (IRNativeReg)r; }