Merge pull request #18234 from unknownbrackets/x86-ir-transfer

x86jit: Perform vector transfers instead of flushing to memory

Common/x64Emitter.cpp

@@ -1697,7 +1697,6 @@ void XEmitter::MOVMSKPD(X64Reg dest, OpArg arg) {WriteSSEOp(0x66, 0x50, dest, ar
 
 void XEmitter::LDDQU(X64Reg dest, OpArg arg)    {WriteSSEOp(0xF2, sseLDDQU, dest, arg);} // For integer data only
 
-// THESE TWO ARE UNTESTED.
 void XEmitter::UNPCKLPS(X64Reg dest, OpArg arg) {WriteSSEOp(0x00, 0x14, dest, arg);}
 void XEmitter::UNPCKHPS(X64Reg dest, OpArg arg) {WriteSSEOp(0x00, 0x15, dest, arg);}
 
@@ -1892,6 +1891,9 @@ void XEmitter::PTEST(X64Reg dest, OpArg arg)    {WriteSSE41Op(0x66, 0x3817, dest
 void XEmitter::PACKUSDW(X64Reg dest, OpArg arg) {WriteSSE41Op(0x66, 0x382b, dest, arg);}
 void XEmitter::DPPS(X64Reg dest, OpArg arg, u8 mask) {WriteSSE41Op(0x66, 0x3A40, dest, arg, 1); Write8(mask);}
 
+void XEmitter::INSERTPS(X64Reg dest, OpArg arg, u8 dstsubreg, u8 srcsubreg, u8 zmask) { WriteSSE41Op(0x66, 0x3A21, dest, arg, 1); Write8((srcsubreg << 6) | (dstsubreg << 4) | zmask); }
+void XEmitter::EXTRACTPS(OpArg dest, X64Reg arg, u8 subreg) { WriteSSE41Op(0x66, 0x3A17, arg, dest, 1); Write8(subreg); }
+
 void XEmitter::PMINSB(X64Reg dest, OpArg arg)   {WriteSSE41Op(0x66, 0x3838, dest, arg);}
 void XEmitter::PMINSD(X64Reg dest, OpArg arg)   {WriteSSE41Op(0x66, 0x3839, dest, arg);}
 void XEmitter::PMINUW(X64Reg dest, OpArg arg)   {WriteSSE41Op(0x66, 0x383a, dest, arg);}
@@ -2084,7 +2086,7 @@ void XEmitter::VCVTTPD2DQ(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits,
 void XEmitter::VCVTTSS2SI(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(0, 0xF3, 0x2C, regOp1, arg, 0, bits == 64 ? 1 : 0); }
 void XEmitter::VCVTTSD2SI(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(0, 0xF2, 0x2C, regOp1, arg, 0, bits == 64 ? 1 : 0); }
 void XEmitter::VEXTRACTPS(OpArg arg, X64Reg regOp1, u8 subreg) { WriteAVXOp(0, 0x66, 0x3A17, regOp1, arg, 1); Write8(subreg); }
-void XEmitter::VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 subreg) { WriteAVXOp(0, 0x66, 0x3A21, regOp1, regOp2, arg, 1); Write8(subreg); }
+void XEmitter::VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 dstsubreg, u8 srcsubreg, u8 zmask) { WriteAVXOp(0, 0x66, 0x3A21, regOp1, regOp2, arg, 1); Write8((srcsubreg << 6) | (dstsubreg << 4) | zmask); }
 void XEmitter::VLDDQU(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0xF2, sseLDDQU, regOp1, arg); }
 void XEmitter::VMOVAPS(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0x00, sseMOVAPfromRM, regOp1, arg); }
 void XEmitter::VMOVAPD(int bits, X64Reg regOp1, OpArg arg) { WriteAVXOp(bits, 0x66, sseMOVAPfromRM, regOp1, arg); }

Common/x64Emitter.h

@@ -684,12 +684,14 @@ public:
 
 	// SSE4: Further horizontal operations - dot products. These are weirdly flexible, the arg contains both a read mask and a write "mask".
 	void DPPD(X64Reg dest, OpArg src, u8 arg);
-
-	// These are probably useful for VFPU emulation.
-	void INSERTPS(X64Reg dest, OpArg src, u8 arg);
-	void EXTRACTPS(OpArg dest, X64Reg src, u8 arg);
 #endif
 
+	// SSE4: Insert and extract for floats.
+	// Note: insert from memory or an XMM.
+	void INSERTPS(X64Reg dest, OpArg arg, u8 dstsubreg, u8 srcsubreg = 0, u8 zmask = 0);
+	// Extract to memory or GPR.
+	void EXTRACTPS(OpArg dest, X64Reg arg, u8 subreg);
+
 	// SSE3: Horizontal operations in SIMD registers. Very slow! shufps-based code beats it handily on Ivy.
 	void HADDPS(X64Reg dest, OpArg src);
 
@@ -1040,7 +1042,7 @@ public:
 	// Can only extract from the low 128 bits.
 	void VEXTRACTPS(OpArg arg, X64Reg regOp1, u8 subreg);
 	// Can only insert into the low 128 bits, zeros upper bits.  Inserts from XMM.
-	void VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 subreg);
+	void VINSERTPS(X64Reg regOp1, X64Reg regOp2, OpArg arg, u8 dstsubreg, u8 srcsubreg = 0, u8 zmask = 0);
 	void VLDDQU(int bits, X64Reg regOp1, OpArg arg);
 	void VMOVAPS(int bits, X64Reg regOp1, OpArg arg);
 	void VMOVAPD(int bits, X64Reg regOp1, OpArg arg);

Core/MIPS/ARM64/Arm64IRRegCache.cpp

@@ -347,7 +347,7 @@ void Arm64IRRegCache::AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) {
 	}
 }
 
-bool Arm64IRRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) {
+bool Arm64IRRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) {
 	// No special flags, skip the check for a little speed.
 	return true;
 }

Core/MIPS/ARM64/Arm64IRRegCache.h

@@ -86,7 +86,7 @@ protected:
 	const int *GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &count, int &base) const override;
 	void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) override;
 
-	bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) override;
+	bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) override;
 	void LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) override;
 	void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override;
 	void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override;

Core/MIPS/IR/IRRegCache.cpp

@@ -406,12 +406,12 @@ IRNativeReg IRNativeRegCacheBase::FindFreeReg(MIPSLoc type, MIPSMap flags) const
 
 bool IRNativeRegCacheBase::IsGPRClobbered(IRReg gpr) const {
 	_dbg_assert_(IsValidGPR(gpr));
-	return IsRegClobbered(MIPSLoc::REG, MIPSMap::INIT, gpr);
+	return IsRegClobbered(MIPSLoc::REG, gpr);
 }
 
 bool IRNativeRegCacheBase::IsFPRClobbered(IRReg fpr) const {
 	_dbg_assert_(IsValidFPR(fpr));
-	return IsRegClobbered(MIPSLoc::FREG, MIPSMap::INIT, fpr + 32);
+	return IsRegClobbered(MIPSLoc::FREG, fpr + 32);
 }
 
 IRUsage IRNativeRegCacheBase::GetNextRegUsage(const IRSituation &info, MIPSLoc type, IRReg r) const {
@@ -423,7 +423,7 @@ IRUsage IRNativeRegCacheBase::GetNextRegUsage(const IRSituation &info, MIPSLoc t
 	return IRUsage::UNKNOWN;
 }
 
-bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) const {
+bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, IRReg r) const {
 	static const int UNUSED_LOOKAHEAD_OPS = 30;
 
 	IRSituation info;
@@ -450,6 +450,21 @@ bool IRNativeRegCacheBase::IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r)
 	return false;
 }
 
+bool IRNativeRegCacheBase::IsRegRead(MIPSLoc type, IRReg first) const {
+	static const int UNUSED_LOOKAHEAD_OPS = 30;
+
+	IRSituation info;
+	info.lookaheadCount = UNUSED_LOOKAHEAD_OPS;
+	// We look starting one ahead, unlike spilling.
+	info.currentIndex = irIndex_ + 1;
+	info.instructions = irBlock_->GetInstructions();
+	info.numInstructions = irBlock_->GetNumInstructions();
+
+	// Note: this intentionally doesn't look at the full reg, only the lane.
+	IRUsage usage = GetNextRegUsage(info, type, first);
+	return usage == IRUsage::READ;
+}
+
 IRNativeReg IRNativeRegCacheBase::FindBestToSpill(MIPSLoc type, MIPSMap flags, bool unusedOnly, bool *clobbered) const {
 	int allocCount = 0, base = 0;
 	const int *allocOrder = GetAllocationOrder(type, flags, allocCount, base);
@@ -501,7 +516,7 @@ IRNativeReg IRNativeRegCacheBase::FindBestToSpill(MIPSLoc type, MIPSMap flags, b
 	return -1;
 }
 
-bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) {
+bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) {
 	int allocCount = 0, base = 0;
 	const int *allocOrder = GetAllocationOrder(type, flags, allocCount, base);
 
@@ -514,6 +529,11 @@ bool IRNativeRegCacheBase::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type,
 	return false;
 }
 
+bool IRNativeRegCacheBase::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) {
+	// To be overridden if the backend supports transfers.
+	return false;
+}
+
 void IRNativeRegCacheBase::DiscardNativeReg(IRNativeReg nreg) {
 	_assert_msg_(nreg >= 0 && nreg < config_.totalNativeRegs, "DiscardNativeReg on invalid register %d", nreg);
 	if (nr[nreg].mipsReg != IRREG_INVALID) {
@@ -930,11 +950,14 @@ IRNativeReg IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRReg first, int la
 		case MIPSLoc::REG:
 			if (type != MIPSLoc::REG) {
 				nreg = AllocateReg(type, flags);
-			} else if (!IsNativeRegCompatible(nreg, type, flags)) {
+			} else if (!IsNativeRegCompatible(nreg, type, flags, lanes)) {
 				// If it's not compatible, we'll need to reallocate.
-				// TODO: Could do a transfer and avoid memory flush.
-				FlushNativeReg(nreg);
-				nreg = AllocateReg(type, flags);
+				if (TransferNativeReg(nreg, -1, type, first, lanes, flags)) {
+					nreg = mr[first].nReg;
+				} else {
+					FlushNativeReg(nreg);
+					nreg = AllocateReg(type, flags);
+				}
 			}
 			break;
 
@@ -942,9 +965,13 @@ IRNativeReg IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRReg first, int la
 		case MIPSLoc::VREG:
 			if (type != mr[first].loc) {
 				nreg = AllocateReg(type, flags);
-			} else if (!IsNativeRegCompatible(nreg, type, flags)) {
-				FlushNativeReg(nreg);
-				nreg = AllocateReg(type, flags);
+			} else if (!IsNativeRegCompatible(nreg, type, flags, lanes)) {
+				if (TransferNativeReg(nreg, -1, type, first, lanes, flags)) {
+					nreg = mr[first].nReg;
+				} else {
+					FlushNativeReg(nreg);
+					nreg = AllocateReg(type, flags);
+				}
 			}
 			break;
 
@@ -981,10 +1008,13 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi
 				_assert_msg_(!mreg.isStatic, "Cannot MapNativeReg a static reg mismatch");
 				if ((flags & MIPSMap::NOINIT) != MIPSMap::NOINIT) {
 					// If we need init, we have to flush mismatches.
-					// TODO: Do a shuffle if interior only?
-					// TODO: We may also be motivated to have multiple read-only "views" or an IRReg.
-					// For example Vec4Scale v0..v3, v0..v3, v3
-					FlushNativeReg(mreg.nReg);
+					if (!TransferNativeReg(mreg.nReg, nreg, type, first, lanes, flags)) {
+						// TODO: We may also be motivated to have multiple read-only "views" or an IRReg.
+						// For example Vec4Scale v0..v3, v0..v3, v3
+						FlushNativeReg(mreg.nReg);
+					}
+					// The mismatch has been "resolved" now.
+					mismatch = false;
 				} else if (oldlanes != 1) {
 					// Even if we don't care about the current contents, we can't discard outside.
 					bool extendsBefore = oldlane > i;
@@ -1017,6 +1047,9 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi
 							DiscardNativeReg(mreg.nReg);
 						else
 							FlushNativeReg(mreg.nReg);
+
+						// That took care of the mismatch, either by clobber or flush.
+						mismatch = false;
 					}
 				}
 			}
@@ -1027,8 +1060,8 @@ void IRNativeRegCacheBase::MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg fi
 				if ((flags & MIPSMap::NOINIT) != MIPSMap::NOINIT) {
 					// We better not be trying to map to a different nreg if it's in one now.
 					// This might happen on some sort of transfer...
-					// TODO: Make a direct transfer, i.e. FREG -> VREG?
-					FlushNativeReg(mreg.nReg);
+					if (!TransferNativeReg(mreg.nReg, nreg, type, first, lanes, flags))
+						FlushNativeReg(mreg.nReg);
 				} else {
 					DiscardNativeReg(mreg.nReg);
 				}

Core/MIPS/IR/IRRegCache.h

@@ -209,13 +209,14 @@ protected:
 	IRNativeReg AllocateReg(MIPSLoc type, MIPSMap flags);
 	IRNativeReg FindFreeReg(MIPSLoc type, MIPSMap flags) const;
 	IRNativeReg FindBestToSpill(MIPSLoc type, MIPSMap flags, bool unusedOnly, bool *clobbered) const;
-	virtual bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags);
+	virtual bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes);
 	virtual void DiscardNativeReg(IRNativeReg nreg);
 	virtual void FlushNativeReg(IRNativeReg nreg);
 	virtual void DiscardReg(IRReg mreg);
 	virtual void FlushReg(IRReg mreg);
 	virtual void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state);
 	virtual void MapNativeReg(MIPSLoc type, IRNativeReg nreg, IRReg first, int lanes, MIPSMap flags);
+	virtual bool TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags);
 	virtual IRNativeReg MapNativeReg(MIPSLoc type, IRReg first, int lanes, MIPSMap flags);
 	IRNativeReg MapNativeRegAsPointer(IRReg gpr);
 
@@ -238,7 +239,8 @@ protected:
 	void SetSpillLockIRIndex(IRReg reg, int index);
 	int GetMipsRegOffset(IRReg r);
 
-	bool IsRegClobbered(MIPSLoc type, MIPSMap flags, IRReg r) const;
+	bool IsRegClobbered(MIPSLoc type, IRReg r) const;
+	bool IsRegRead(MIPSLoc type, IRReg r) const;
 	IRUsage GetNextRegUsage(const IRSituation &info, MIPSLoc type, IRReg r) const;
 
 	bool IsValidGPR(IRReg r) const;

Core/MIPS/RiscV/RiscVRegCache.cpp

@@ -303,11 +303,11 @@ void RiscVRegCache::AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) {
 	}
 }
 
-bool RiscVRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) {
+bool RiscVRegCache::IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) {
 	// No special flags except VREG, skip the check for a little speed.
 	if (type != MIPSLoc::VREG)
 		return true;
-	return IRNativeRegCacheBase::IsNativeRegCompatible(nreg, type, flags);
+	return IRNativeRegCacheBase::IsNativeRegCompatible(nreg, type, flags, lanes);
 }
 
 void RiscVRegCache::LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) {

Core/MIPS/RiscV/RiscVRegCache.h

@@ -76,7 +76,7 @@ protected:
 	const int *GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &count, int &base) const override;
 	void AdjustNativeRegAsPtr(IRNativeReg nreg, bool state) override;
 
-	bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags) override;
+	bool IsNativeRegCompatible(IRNativeReg nreg, MIPSLoc type, MIPSMap flags, int lanes) override;
 	void LoadNativeReg(IRNativeReg nreg, IRReg first, int lanes) override;
 	void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override;
 	void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override;

Core/MIPS/x86/X64IRRegCache.cpp

@@ -453,6 +453,275 @@ void X64IRRegCache::StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) {
 	}
 }
 
+bool X64IRRegCache::TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) {
+	bool allowed = !mr[nr[nreg].mipsReg].isStatic;
+	// There's currently no support for non-XMMs here.
+	allowed = allowed && type == MIPSLoc::FREG;
+
+	if (dest == -1)
+		dest = nreg;
+
+	if (allowed && (flags == MIPSMap::INIT || flags == MIPSMap::DIRTY)) {
+		// Alright, changing lane count (possibly including lane position.)
+		IRReg oldfirst = nr[nreg].mipsReg;
+		int oldlanes = 0;
+		while (mr[oldfirst + oldlanes].nReg == nreg)
+			oldlanes++;
+		_assert_msg_(oldlanes != 0, "TransferNativeReg encountered nreg mismatch");
+		_assert_msg_(oldlanes != lanes, "TransferNativeReg transfer to same lanecount, misaligned?");
+
+		if (lanes == 1 && TransferVecTo1(nreg, dest, first, oldlanes))
+			return true;
+		if (oldlanes == 1 && Transfer1ToVec(nreg, dest, first, lanes))
+			return true;
+	}
+
+	return IRNativeRegCacheBase::TransferNativeReg(nreg, dest, type, first, lanes, flags);
+}
+
+bool X64IRRegCache::TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg first, int oldlanes) {
+	IRReg oldfirst = nr[nreg].mipsReg;
+
+	// Is it worth preserving any of the old regs?
+	int numKept = 0;
+	for (int i = 0; i < oldlanes; ++i) {
+		// Skip whichever one this is extracting.
+		if (oldfirst + i == first)
+			continue;
+		// If 0 isn't being transfered, easy to keep in its original reg.
+		if (i == 0 && dest != nreg) {
+			numKept++;
+			continue;
+		}
+
+		IRNativeReg freeReg = FindFreeReg(MIPSLoc::FREG, MIPSMap::INIT);
+		if (freeReg != -1 && IsRegRead(MIPSLoc::FREG, oldfirst + i)) {
+			// If there's one free, use it.  Don't modify nreg, though.
+			u8 shuf = VFPU_SWIZZLE(i, i, i, i);
+			if (i == 0) {
+				emit_->MOVAPS(FromNativeReg(freeReg), ::R(FromNativeReg(nreg)));
+			} else if (cpu_info.bAVX) {
+				emit_->VPERMILPS(128, FromNativeReg(freeReg), ::R(FromNativeReg(nreg)), shuf);
+			} else if (i == 2) {
+				emit_->MOVHLPS(FromNativeReg(freeReg), FromNativeReg(nreg));
+			} else {
+				emit_->MOVAPS(FromNativeReg(freeReg), ::R(FromNativeReg(nreg)));
+				emit_->SHUFPS(FromNativeReg(freeReg), ::R(FromNativeReg(freeReg)), shuf);
+			}
+
+			// Update accounting.
+			nr[freeReg].isDirty = nr[nreg].isDirty;
+			nr[freeReg].mipsReg = oldfirst + i;
+			mr[oldfirst + i].lane = -1;
+			mr[oldfirst + i].nReg = freeReg;
+			numKept++;
+		}
+	}
+
+	// Unless all other lanes were kept, store.
+	if (nr[nreg].isDirty && numKept < oldlanes - 1) {
+		StoreNativeReg(nreg, oldfirst, oldlanes);
+		// Set false even for regs that were split out, since they were flushed too.
+		for (int i = 0; i < oldlanes; ++i) {
+			if (mr[oldfirst + i].nReg != -1)
+				nr[mr[oldfirst + i].nReg].isDirty = false;
+		}
+	}
+
+	// Next, shuffle the desired element into first place.
+	u8 shuf = VFPU_SWIZZLE(mr[first].lane, mr[first].lane, mr[first].lane, mr[first].lane);
+	if (mr[first].lane > 0 && cpu_info.bAVX && dest != nreg) {
+		emit_->VPERMILPS(128, FromNativeReg(dest), ::R(FromNativeReg(nreg)), shuf);
+	} else if (mr[first].lane <= 0 && dest != nreg) {
+		emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg)));
+	} else if (mr[first].lane == 2) {
+		emit_->MOVHLPS(FromNativeReg(dest), FromNativeReg(nreg));
+	} else if (mr[first].lane > 0) {
+		if (dest != nreg)
+			emit_->MOVAPS(FromNativeReg(dest), ::R(FromNativeReg(nreg)));
+		emit_->SHUFPS(FromNativeReg(dest), ::R(FromNativeReg(dest)), shuf);
+	}
+
+	// Now update accounting.
+	for (int i = 0; i < oldlanes; ++i) {
+		auto &mreg = mr[oldfirst + i];
+		if (oldfirst + i == first) {
+			mreg.lane = -1;
+			mreg.nReg = dest;
+		} else if (mreg.nReg == nreg && i == 0 && nreg != dest) {
+			// Still in the same register, but no longer a vec.
+			mreg.lane = -1;
+		} else if (mreg.nReg == nreg) {
+			// No longer in a register.
+			mreg.nReg = -1;
+			mreg.lane = -1;
+			mreg.loc = MIPSLoc::MEM;
+		}
+	}
+
+	if (dest != nreg) {
+		nr[dest].isDirty = nr[nreg].isDirty;
+		if (oldfirst == first) {
+			nr[nreg].mipsReg = -1;
+			nr[nreg].isDirty = false;
+		}
+	}
+	nr[dest].mipsReg = first;
+
+	return true;
+}
+
+bool X64IRRegCache::Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg first, int lanes) {
+	X64Reg cur[4]{};
+	int numInRegs = 0;
+	u8 blendMask = 0;
+	for (int i = 0; i < lanes; ++i) {
+		if (mr[first + i].lane != -1 || (i != 0 && mr[first + i].spillLockIRIndex >= irIndex_)) {
+			// Can't do it, either double mapped or overlapping vec.
+			return false;
+		}
+
+		if (mr[first + i].nReg == -1) {
+			cur[i] = INVALID_REG;
+			blendMask |= 1 << i;
+		} else {
+			cur[i] = FromNativeReg(mr[first + i].nReg);
+			numInRegs++;
+		}
+	}
+
+	// Shouldn't happen, this should only get called to transfer one in a reg.
+	if (numInRegs == 0)
+		return false;
+
+	// Move things together into a reg.
+	if (lanes == 4 && cpu_info.bSSE4_1 && numInRegs == 1 && (first & 3) == 0) {
+		// Use a blend to grab the rest.  BLENDPS is pretty good.
+		if (cpu_info.bAVX && nreg != dest) {
+			if (cur[0] == INVALID_REG) {
+				// Broadcast to all lanes, then blend from memory to replace.
+				emit_->VPERMILPS(128, FromNativeReg(dest), ::R(FromNativeReg(nreg)), 0);
+				emit_->BLENDPS(FromNativeReg(dest), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask);
+			} else {
+				emit_->VBLENDPS(128, FromNativeReg(dest), FromNativeReg(nreg), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask);
+			}
+			cur[0] = FromNativeReg(dest);
+		} else {
+			if (cur[0] == INVALID_REG)
+				emit_->SHUFPS(FromNativeReg(nreg), ::R(FromNativeReg(nreg)), 0);
+			emit_->BLENDPS(FromNativeReg(nreg), MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask);
+			// If this is not dest, it'll get moved there later.
+			cur[0] = FromNativeReg(nreg);
+		}
+	} else if (lanes == 4) {
+		if (blendMask == 0) {
+			// y = yw##, x = xz##, x = xyzw.
+			emit_->UNPCKLPS(cur[1], ::R(cur[3]));
+			emit_->UNPCKLPS(cur[0], ::R(cur[2]));
+			emit_->UNPCKLPS(cur[0], ::R(cur[1]));
+		} else if (blendMask == 0b1100) {
+			// x = xy##, then load zw.
+			emit_->UNPCKLPS(cur[0], ::R(cur[1]));
+			emit_->MOVHPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2)));
+		} else if (blendMask == 0b1010 && cpu_info.bSSE4_1 && (first & 3) == 0) {
+			// x = x#z#, x = xyzw.
+			emit_->SHUFPS(cur[0], ::R(cur[2]), VFPU_SWIZZLE(0, 0, 0, 0));
+			emit_->BLENDPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask);
+		} else if (blendMask == 0b0110 && cpu_info.bSSE4_1 && (first & 3) == 0) {
+			// x = x##w, x = xyzw.
+			emit_->SHUFPS(cur[0], ::R(cur[3]), VFPU_SWIZZLE(0, 0, 0, 0));
+			emit_->BLENDPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask);
+		} else if (blendMask == 0b1001 && cpu_info.bSSE4_1 && (first & 3) == 0) {
+			// y = #yz#, y = xyzw.
+			emit_->SHUFPS(cur[1], ::R(cur[2]), VFPU_SWIZZLE(0, 0, 0, 0));
+			emit_->BLENDPS(cur[1], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask);
+			// Will be moved to dest as needed.
+			cur[0] = cur[1];
+		} else if (blendMask == 0b0101 && cpu_info.bSSE4_1 && (first & 3) == 0) {
+			// y = #y#w, y = xyzw.
+			emit_->SHUFPS(cur[1], ::R(cur[3]), VFPU_SWIZZLE(0, 0, 0, 0));
+			emit_->BLENDPS(cur[1], MDisp(CTXREG, -128 + GetMipsRegOffset(first)), blendMask);
+			// Will be moved to dest as needed.
+			cur[0] = cur[1];
+		} else if (blendMask == 0b1000) {
+			// x = xz##, z = w###, y = yw##, x = xyzw.
+			emit_->UNPCKLPS(cur[0], ::R(cur[2]));
+			emit_->MOVSS(cur[2], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 3)));
+			emit_->UNPCKLPS(cur[1], ::R(cur[2]));
+			emit_->UNPCKLPS(cur[0], ::R(cur[1]));
+		} else if (blendMask == 0b0100) {
+			// y = yw##, w = z###, x = xz##, x = xyzw.
+			emit_->UNPCKLPS(cur[1], ::R(cur[3]));
+			emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 2)));
+			emit_->UNPCKLPS(cur[0], ::R(cur[3]));
+			emit_->UNPCKLPS(cur[0], ::R(cur[1]));
+		} else if (blendMask == 0b0010) {
+			// z = zw##, w = y###, x = xy##, x = xyzw.
+			emit_->UNPCKLPS(cur[2], ::R(cur[3]));
+			emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)));
+			emit_->UNPCKLPS(cur[0], ::R(cur[3]));
+			emit_->MOVLHPS(cur[0], cur[2]);
+		} else if (blendMask == 0b0001) {
+			// y = yw##, w = x###, w = xz##, w = xyzw.
+			emit_->UNPCKLPS(cur[1], ::R(cur[3]));
+			emit_->MOVSS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0)));
+			emit_->UNPCKLPS(cur[3], ::R(cur[2]));
+			emit_->UNPCKLPS(cur[3], ::R(cur[1]));
+			// Will be moved to dest as needed.
+			cur[0] = cur[3];
+		} else if (blendMask == 0b0011) {
+			// z = zw##, w = xy##, w = xyzw.
+			emit_->UNPCKLPS(cur[2], ::R(cur[3]));
+			emit_->MOVLPS(cur[3], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 0)));
+			emit_->MOVLHPS(cur[3], cur[2]);
+			// Will be moved to dest as needed.
+			cur[0] = cur[3];
+		} else {
+			// This must mean no SSE4, and numInRegs <= 2 in trickier cases.
+			return false;
+		}
+	} else if (lanes == 2) {
+		if (cur[0] != INVALID_REG && cur[1] != INVALID_REG) {
+			emit_->UNPCKLPS(cur[0], ::R(cur[1]));
+		} else if (cur[0] != INVALID_REG && cpu_info.bSSE4_1) {
+			emit_->INSERTPS(cur[0], MDisp(CTXREG, -128 + GetMipsRegOffset(first + 1)), 1);
+		} else {
+			return false;
+		}
+	} else {
+		return false;
+	}
+
+	mr[first].lane = 0;
+	for (int i = 0; i < lanes; ++i) {
+		if (mr[first + i].nReg != -1) {
+			// If this was dirty, the combined reg is now dirty.
+			if (nr[mr[first + i].nReg].isDirty)
+				nr[dest].isDirty = true;
+
+			// Throw away the other register we're no longer using.
+			if (i != 0)
+				DiscardNativeReg(mr[first + i].nReg);
+		}
+
+		// And set it as using the new one.
+		mr[first + i].lane = i;
+		mr[first + i].loc = MIPSLoc::FREG;
+		mr[first + i].nReg = dest;
+	}
+
+	if (cur[0] != FromNativeReg(dest))
+		emit_->MOVAPS(FromNativeReg(dest), ::R(cur[0]));
+
+	if (dest != nreg) {
+		nr[dest].mipsReg = first;
+		nr[nreg].mipsReg = -1;
+		nr[nreg].isDirty = false;
+	}
+
+	return true;
+}
+
 void X64IRRegCache::SetNativeRegValue(IRNativeReg nreg, uint32_t imm) {
 	X64Reg r = FromNativeReg(nreg);
 	_dbg_assert_(nreg >= 0 && nreg < NUM_X_REGS);

Core/MIPS/x86/X64IRRegCache.h

@@ -117,8 +117,12 @@ protected:
 	void StoreNativeReg(IRNativeReg nreg, IRReg first, int lanes) override;
 	void SetNativeRegValue(IRNativeReg nreg, uint32_t imm) override;
 	void StoreRegValue(IRReg mreg, uint32_t imm) override;
+	bool TransferNativeReg(IRNativeReg nreg, IRNativeReg dest, MIPSLoc type, IRReg first, int lanes, MIPSMap flags) override;
 
 private:
+	bool TransferVecTo1(IRNativeReg nreg, IRNativeReg dest, IRReg first, int oldlanes);
+	bool Transfer1ToVec(IRNativeReg nreg, IRNativeReg dest, IRReg first, int lanes);
+
 	IRNativeReg GPRToNativeReg(Gen::X64Reg r) {
 		return (IRNativeReg)r;
 	}