riscv: Add vector load/store ops.

This commit is contained in:
Unknown W. Brackets committed 2023-01-21 19:36:11 -08:00
1 parent 3b388c7692
commit bfd60a67ad
2 files changed
+328 -2

No files matched your search

+205 -2
View File
@@ -53,6 +53,10 @@ static inline bool SupportsZicsr() {
return false;
}
static inline bool SupportsVector() {
return cpu_info.RiscV_V;
}
enum class Opcode32 {
// Note: invalid, just used for FixupBranch.
ZERO = 0b0000000,
@@ -73,6 +77,7 @@ enum class Opcode32 {
FNMSUB = 0b1001011,
FNMADD = 0b1001111,
OP_FP = 0b1010011,
OP_V = 0b1010111,
BRANCH = 0b1100011,
JALR = 0b1100111,
JAL = 0b1101111,
@@ -148,6 +153,12 @@ enum class Funct3 {
CSRRSI = 0b110,
CSRRCI = 0b111,
VSETVL = 0b111,
VLS_8 = 0b000,
VLS_16 = 0b101,
VLS_32 = 0b110,
VLS_64 = 0b111,
C_ADDI4SPN = 0b000,
C_FLD = 0b001,
C_LW = 0b010,
@@ -257,9 +268,24 @@ enum class RiscCReg {
X8, X9, X10, X11, X12, X13, X14, X15,
};
enum class VLSUMop {
ELEMS = 0b00000,
REG = 0b01000,
MASK = 0b01011,
ELEMS_LOAD_FF = 0b10000,
};
enum class VMop {
UNIT = 0b00,
INDEXU = 0b01,
STRIDE = 0b10,
INDEXO = 0b11,
};
static inline RiscVReg DecodeReg(RiscVReg reg) { return (RiscVReg)(reg & 0x1F); }
static inline bool IsGPR(RiscVReg reg) { return reg < 0x20; }
static inline bool IsFPR(RiscVReg reg) { return (reg & 0x20) != 0 && (int)reg < 0x40; }
static inline bool IsGPR(RiscVReg reg) { return (reg & ~0x1F) == 0; }
static inline bool IsFPR(RiscVReg reg) { return (reg & ~0x1F) == 0x20; }
static inline bool IsVPR(RiscVReg reg) { return (reg & ~0x1F) == 0x40; }
static inline bool CanCompress(RiscVReg reg) {
return (DecodeReg(reg) & 0x18) == 0x08;
@@ -579,6 +605,35 @@ static inline int FConvToIntegerBits(FConv c) {
return 0;
}
Funct3 VecBitsToFunct3(int bits) {
int bitsSupported = SupportsVector() ? 64 : 0;
_assert_msg_(bitsSupported >= bits, "Cannot use funct3 width %d, only have %d", bits, bitsSupported);
switch (bits) {
case 8:
return Funct3::VLS_8;
case 16:
return Funct3::VLS_16;
case 32:
return Funct3::VLS_32;
case 64:
return Funct3::VLS_64;
default:
_assert_msg_(false, "Invalid funct3 width %d", bits);
return Funct3::VLS_8;
}
}
static s32 VecLSToSimm12(RiscVReg vrs2, VUseMask vm, VMop mop, int bits, int nf) {
_assert_msg_(nf >= 1 && nf <= 8, "Cannot encode field count %d (must be <= 8)", nf);
int mew = bits >= 128 ? 1 : 0;
int nf3 = nf > 4 ? (0xFFFFFFF8 | (nf - 1)) : (nf - 1);
return (s32)DecodeReg(vrs2) | ((s32)vm << 5) | ((s32)mop << 6) | (mew << 8) | (nf3 << 9);
}
static s32 VecLSToSimm12(VLSUMop lsumop, VUseMask vm, VMop mop, int bits, int nf) {
return VecLSToSimm12((RiscVReg)(int)lsumop, vm, mop, bits, nf);
}
RiscVEmitter::RiscVEmitter(const u8 *ptr, u8 *writePtr) {
SetCodePointer(ptr, writePtr);
}
@@ -1764,6 +1819,154 @@ void RiscVEmitter::CSRRCI(RiscVReg rd, Csr csr, u8 uimm5) {
Write32(EncodeGI(Opcode32::SYSTEM, rd, Funct3::CSRRCI, (RiscVReg)uimm5, (Funct12)csr));
}
void RiscVEmitter::VSETVLI(RiscVReg rd, RiscVReg rs1, VType vtype) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_((vtype.value & ~0xFF) == 0, "%s with invalid vtype", __func__);
_assert_msg_(IsGPR(rd), "%s rd (VL) must be GPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (AVL) must be GPR", __func__);
Write32(EncodeI(Opcode32::OP_V, rd, Funct3::VSETVL, rs1, (s32)vtype.value));
}
void RiscVEmitter::VSETIVLI(RiscVReg rd, u8 uimm5, VType vtype) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_((vtype.value & ~0xFF) == 0, "%s with invalid vtype", __func__);
_assert_msg_(IsGPR(rd), "%s rd (VL) must be GPR", __func__);
_assert_msg_((u32)uimm5 <= 0x1F, "%s (AVL) can only set up to 31", __func__);
s32 simm12 = 0xFFFFFC00 | vtype.value;
Write32(EncodeI(Opcode32::OP_V, rd, Funct3::VSETVL, (RiscVReg)uimm5, (s32)vtype.value));
}
void RiscVEmitter::VSETVL(RiscVReg rd, RiscVReg rs1, RiscVReg rs2) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsGPR(rd), "%s rd (VL) must be GPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (AVL) must be GPR", __func__);
_assert_msg_(IsGPR(rs2), "%s rs2 (vtype) must be GPR", __func__);
Write32(EncodeI(Opcode32::OP_V, rd, Funct3::VSETVL, rs1, rs2));
}
void RiscVEmitter::VLM_V(RiscVReg vd, RiscVReg rs1) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__);
s32 simm12 = VecLSToSimm12(VLSUMop::MASK, VUseMask::NONE, VMop::UNIT, 8, 1);
Write32(EncodeI(Opcode32::LOAD_FP, vd, Funct3::VLS_8, rs1, simm12));
}
void RiscVEmitter::VLSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__);
// Of course, if LMUL > 1, it could still be wrong, but this is a good basic check.
_assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(VLSUMop::ELEMS, vm, VMop::UNIT, dataBits, fields);
Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(dataBits), rs1, simm12));
}
void RiscVEmitter::VLSSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, RiscVReg rs2, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__);
_assert_msg_(IsGPR(rs2), "%s rs2 (stride) must be GPR", __func__);
_assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(rs2, vm, VMop::STRIDE, dataBits, fields);
Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(dataBits), rs1, simm12));
}
void RiscVEmitter::VLUXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__);
_assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__);
_assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXU, indexBits, fields);
Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(indexBits), rs1, simm12));
}
void RiscVEmitter::VLOXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__);
_assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__);
_assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXO, indexBits, fields);
Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(indexBits), rs1, simm12));
}
void RiscVEmitter::VLSEGEFF_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__);
s32 simm12 = VecLSToSimm12(VLSUMop::ELEMS_LOAD_FF, vm, VMop::UNIT, dataBits, fields);
Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(dataBits), rs1, simm12));
}
void RiscVEmitter::VLR_V(int regs, int hintBits, RiscVReg vd, RiscVReg rs1) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__);
_assert_msg_(regs == 1 || regs == 2 || regs == 4 || regs == 8, "%s can only access count=1/2/4/8 at a time, not %d", __func__, regs);
_assert_msg_(((int)DecodeReg(vd) & (regs - 1)) == 0, "%s base reg must align to reg count", __func__);
_assert_msg_((int)DecodeReg(vd) + regs <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(VLSUMop::REG, VUseMask::NONE, VMop::UNIT, hintBits, regs);
Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(hintBits), rs1, simm12));
}
void RiscVEmitter::VSM_V(RiscVReg vs3, RiscVReg rs1) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__);
s32 simm12 = VecLSToSimm12(VLSUMop::MASK, VUseMask::NONE, VMop::UNIT, 8, 1);
Write32(EncodeI(Opcode32::STORE_FP, vs3, Funct3::VLS_8, rs1, simm12));
}
void RiscVEmitter::VSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__);
_assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(VLSUMop::ELEMS, vm, VMop::UNIT, dataBits, fields);
Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(dataBits), rs1, simm12));
}
void RiscVEmitter::VSSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, RiscVReg rs2, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__);
_assert_msg_(IsGPR(rs2), "%s rs2 (stride) must be GPR", __func__);
_assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(rs2, vm, VMop::STRIDE, dataBits, fields);
Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(dataBits), rs1, simm12));
}
void RiscVEmitter::VSUXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__);
_assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__);
_assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXU, indexBits, fields);
Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(indexBits), rs1, simm12));
}
void RiscVEmitter::VSOXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__);
_assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__);
_assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXO, indexBits, fields);
Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(indexBits), rs1, simm12));
}
void RiscVEmitter::VSR_V(int regs, RiscVReg vs3, RiscVReg rs1) {
_assert_msg_(SupportsVector(), "%s instruction not supported", __func__);
_assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__);
_assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__);
_assert_msg_(regs == 1 || regs == 2 || regs == 4 || regs == 8, "%s can only access count=1/2/4/8 at a time, not %d", __func__, regs);
_assert_msg_(((int)DecodeReg(vs3) & (regs - 1)) == 0, "%s base reg must align to reg count", __func__);
_assert_msg_((int)DecodeReg(vs3) + regs <= 32, "%s cannot access beyond V31", __func__);
s32 simm12 = VecLSToSimm12(VLSUMop::REG, VUseMask::NONE, VMop::UNIT, 8, regs);
Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(8), rs1, simm12));
}
bool RiscVEmitter::AutoCompress() const {
return SupportsCompressed() && autoCompress_;
}
+123
View File
@@ -42,6 +42,11 @@ enum RiscVReg {
F8, F9, F10, F11, F12, F13, F14, F15,
F16, F17, F18, F19, F20, F21, F22, F23,
F24, F25, F26, F27, F28, F29, F30, F31,
V0 = 0x40, V1, V2, V3, V4, V5, V6, V7,
V8, V9, V10, V11, V12, V13, V14, V15,
V16, V17, V18, V19, V20, V21, V22, V23,
V24, V25, V26, V27, V28, V29, V30, V31,
};
enum class FixupBranchType {
@@ -99,6 +104,14 @@ enum class Csr {
FRm = 0x002,
FCsr = 0x003,
VStart = 0x008,
VXSat = 0x009,
VXRm = 0x00A,
VCsr = 0x00F,
VL = 0xC20,
VType = 0xC21,
VLenB = 0xC22,
Cycle = 0xC00,
Time = 0xC01,
InstRet = 0xC02,
@@ -107,6 +120,61 @@ enum class Csr {
InstRetH = 0xC82,
};
enum class VLMul {
M1 = 0b000,
M2 = 0b001,
M4 = 0b010,
M8 = 0b011,
MF8 = 0b101,
MF4 = 0b110,
MF2 = 0b111,
};
enum class VSew {
E8 = 0b000,
E16 = 0b001,
E32 = 0b010,
E64 = 0b011,
};
enum class VTail {
U = 0,
A = 1,
};
enum class VMask {
U = 0,
A = 1,
};
struct VType {
constexpr VType(VSew sew, VTail vt, VMask vm)
: value(((uint32_t)sew << 3) | ((uint32_t)vt << 6) | ((uint32_t)vm << 7)) {
}
constexpr VType(VSew sew, VLMul lmul, VTail vt, VMask vm)
: value((uint32_t)lmul | ((uint32_t)sew << 3) | ((uint32_t)vt << 6) | ((uint32_t)vm << 7)) {
}
VType(int bits, VLMul lmul, VTail vt, VMask vm) {
VSew sew = VSew::E8;
switch (bits) {
case 8: sew = VSew::E8; break;
case 16: sew = VSew::E16; break;
case 32: sew = VSew::E32; break;
case 64: sew = VSew::E64; break;
default: _assert_msg_(false, "Invalid vtype width"); break;
}
value = (uint32_t)lmul | ((uint32_t)sew << 3) | ((uint32_t)vt << 6) | ((uint32_t)vm << 7);
}
uint32_t value;
};
enum class VUseMask {
V0_T = 0,
NONE = 1,
};
struct FixupBranch {
FixupBranch() {}
FixupBranch(const u8 *p, FixupBranchType t) : ptr(p), type(t) {}
@@ -347,6 +415,61 @@ public:
FSRMI(R_ZERO, rm);
}
// Vector instructions.
void VSETVLI(RiscVReg rd, RiscVReg rs1, VType vtype);
void VSETIVLI(RiscVReg rd, u8 uimm5, VType vtype);
void VSETVL(RiscVReg rd, RiscVReg rs1, RiscVReg rs2);
// Load contiguous registers, unordered.
void VLE_V(int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE) {
VLSEGE_V(1, dataBits, vd, rs1, vm);
}
// Load registers with stride (note: rs2/stride can be X0/zero to broadcast.)
void VLSE_V(int dataBits, RiscVReg vd, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE) {
VLSSEGE_V(1, dataBits, vd, rs1, rs2, vm);
}
// Load indexed registers (gather), unordered.
void VLUXEI_V(int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) {
VLUXSEGEI_V(1, indexBits, vd, rs1, vs2, vm);
}
// Load indexed registers (gather), ordered.
void VLOXEI_V(int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) {
VLOXSEGEI_V(1, indexBits, vd, rs1, vs2, vm);
}
// Load mask (force 8 bit, EMUL=1, TA)
void VLM_V(RiscVReg vd, RiscVReg rs1);
// Load but ignore faults after first element.
void VLEFF_V(int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE) {
VLSEGEFF_V(1, dataBits, vd, rs1, vm);
}
// Load fields into subsequent registers (destructure.)
void VLSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE);
void VLSSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE);
void VLUXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE);
void VLOXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE);
void VLSEGEFF_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE);
// Load entire registers (implementation dependent size.)
void VLR_V(int regs, int hintBits, RiscVReg vd, RiscVReg rs1);
void VSE_V(int dataBits, RiscVReg vs3, RiscVReg rs1, VUseMask vm = VUseMask::NONE) {
VSSEGE_V(1, dataBits, vs3, rs1, vm);
}
void VSSE_V(int dataBits, RiscVReg vs3, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE) {
VSSSEGE_V(1, dataBits, vs3, rs1, rs2, vm);
}
void VSUXEI_V(int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) {
VSUXSEGEI_V(1, indexBits, vs3, rs1, vs2, vm);
}
void VSOXEI_V(int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) {
VSOXSEGEI_V(1, indexBits, vs3, rs1, vs2, vm);
}
void VSM_V(RiscVReg vs3, RiscVReg rs1);
void VSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, VUseMask vm = VUseMask::NONE);
void VSSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE);
void VSUXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE);
void VSOXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE);
void VSR_V(int regs, RiscVReg vs3, RiscVReg rs1);
// Compressed instructions.
void C_ADDI4SPN(RiscVReg rd, u32 nzuimm10);
void C_FLD(RiscVReg rd, RiscVReg addr, u8 uimm8);