From bfd60a67add698cc22a0e43dafb988e6e6b39b10 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sat, 21 Jan 2023 14:51:52 -0800 Subject: [PATCH] riscv: Add vector load/store ops. --- Common/RiscVEmitter.cpp | 207 +++++++++++++++++++++++++++++++++++++++- Common/RiscVEmitter.h | 123 ++++++++++++++++++++++++ 2 files changed, 328 insertions(+), 2 deletions(-) diff --git a/Common/RiscVEmitter.cpp b/Common/RiscVEmitter.cpp index 0ec05c1c56..fed79d9aa7 100644 --- a/Common/RiscVEmitter.cpp +++ b/Common/RiscVEmitter.cpp @@ -53,6 +53,10 @@ static inline bool SupportsZicsr() { return false; } +static inline bool SupportsVector() { + return cpu_info.RiscV_V; +} + enum class Opcode32 { // Note: invalid, just used for FixupBranch. ZERO = 0b0000000, @@ -73,6 +77,7 @@ enum class Opcode32 { FNMSUB = 0b1001011, FNMADD = 0b1001111, OP_FP = 0b1010011, + OP_V = 0b1010111, BRANCH = 0b1100011, JALR = 0b1100111, JAL = 0b1101111, @@ -148,6 +153,12 @@ enum class Funct3 { CSRRSI = 0b110, CSRRCI = 0b111, + VSETVL = 0b111, + VLS_8 = 0b000, + VLS_16 = 0b101, + VLS_32 = 0b110, + VLS_64 = 0b111, + C_ADDI4SPN = 0b000, C_FLD = 0b001, C_LW = 0b010, @@ -257,9 +268,24 @@ enum class RiscCReg { X8, X9, X10, X11, X12, X13, X14, X15, }; +enum class VLSUMop { + ELEMS = 0b00000, + REG = 0b01000, + MASK = 0b01011, + ELEMS_LOAD_FF = 0b10000, +}; + +enum class VMop { + UNIT = 0b00, + INDEXU = 0b01, + STRIDE = 0b10, + INDEXO = 0b11, +}; + static inline RiscVReg DecodeReg(RiscVReg reg) { return (RiscVReg)(reg & 0x1F); } -static inline bool IsGPR(RiscVReg reg) { return reg < 0x20; } -static inline bool IsFPR(RiscVReg reg) { return (reg & 0x20) != 0 && (int)reg < 0x40; } +static inline bool IsGPR(RiscVReg reg) { return (reg & ~0x1F) == 0; } +static inline bool IsFPR(RiscVReg reg) { return (reg & ~0x1F) == 0x20; } +static inline bool IsVPR(RiscVReg reg) { return (reg & ~0x1F) == 0x40; } static inline bool CanCompress(RiscVReg reg) { return (DecodeReg(reg) & 0x18) == 0x08; @@ -579,6 +605,35 @@ static inline int FConvToIntegerBits(FConv c) { return 0; } +Funct3 VecBitsToFunct3(int bits) { + int bitsSupported = SupportsVector() ? 64 : 0; + _assert_msg_(bitsSupported >= bits, "Cannot use funct3 width %d, only have %d", bits, bitsSupported); + switch (bits) { + case 8: + return Funct3::VLS_8; + case 16: + return Funct3::VLS_16; + case 32: + return Funct3::VLS_32; + case 64: + return Funct3::VLS_64; + default: + _assert_msg_(false, "Invalid funct3 width %d", bits); + return Funct3::VLS_8; + } +} + +static s32 VecLSToSimm12(RiscVReg vrs2, VUseMask vm, VMop mop, int bits, int nf) { + _assert_msg_(nf >= 1 && nf <= 8, "Cannot encode field count %d (must be <= 8)", nf); + int mew = bits >= 128 ? 1 : 0; + int nf3 = nf > 4 ? (0xFFFFFFF8 | (nf - 1)) : (nf - 1); + return (s32)DecodeReg(vrs2) | ((s32)vm << 5) | ((s32)mop << 6) | (mew << 8) | (nf3 << 9); +} + +static s32 VecLSToSimm12(VLSUMop lsumop, VUseMask vm, VMop mop, int bits, int nf) { + return VecLSToSimm12((RiscVReg)(int)lsumop, vm, mop, bits, nf); +} + RiscVEmitter::RiscVEmitter(const u8 *ptr, u8 *writePtr) { SetCodePointer(ptr, writePtr); } @@ -1764,6 +1819,154 @@ void RiscVEmitter::CSRRCI(RiscVReg rd, Csr csr, u8 uimm5) { Write32(EncodeGI(Opcode32::SYSTEM, rd, Funct3::CSRRCI, (RiscVReg)uimm5, (Funct12)csr)); } +void RiscVEmitter::VSETVLI(RiscVReg rd, RiscVReg rs1, VType vtype) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_((vtype.value & ~0xFF) == 0, "%s with invalid vtype", __func__); + _assert_msg_(IsGPR(rd), "%s rd (VL) must be GPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (AVL) must be GPR", __func__); + Write32(EncodeI(Opcode32::OP_V, rd, Funct3::VSETVL, rs1, (s32)vtype.value)); +} + +void RiscVEmitter::VSETIVLI(RiscVReg rd, u8 uimm5, VType vtype) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_((vtype.value & ~0xFF) == 0, "%s with invalid vtype", __func__); + _assert_msg_(IsGPR(rd), "%s rd (VL) must be GPR", __func__); + _assert_msg_((u32)uimm5 <= 0x1F, "%s (AVL) can only set up to 31", __func__); + s32 simm12 = 0xFFFFFC00 | vtype.value; + Write32(EncodeI(Opcode32::OP_V, rd, Funct3::VSETVL, (RiscVReg)uimm5, (s32)vtype.value)); +} + +void RiscVEmitter::VSETVL(RiscVReg rd, RiscVReg rs1, RiscVReg rs2) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsGPR(rd), "%s rd (VL) must be GPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (AVL) must be GPR", __func__); + _assert_msg_(IsGPR(rs2), "%s rs2 (vtype) must be GPR", __func__); + Write32(EncodeI(Opcode32::OP_V, rd, Funct3::VSETVL, rs1, rs2)); +} + +void RiscVEmitter::VLM_V(RiscVReg vd, RiscVReg rs1) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__); + s32 simm12 = VecLSToSimm12(VLSUMop::MASK, VUseMask::NONE, VMop::UNIT, 8, 1); + Write32(EncodeI(Opcode32::LOAD_FP, vd, Funct3::VLS_8, rs1, simm12)); +} + +void RiscVEmitter::VLSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__); + // Of course, if LMUL > 1, it could still be wrong, but this is a good basic check. + _assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(VLSUMop::ELEMS, vm, VMop::UNIT, dataBits, fields); + Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(dataBits), rs1, simm12)); +} + +void RiscVEmitter::VLSSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, RiscVReg rs2, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__); + _assert_msg_(IsGPR(rs2), "%s rs2 (stride) must be GPR", __func__); + _assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(rs2, vm, VMop::STRIDE, dataBits, fields); + Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(dataBits), rs1, simm12)); +} + +void RiscVEmitter::VLUXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__); + _assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__); + _assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXU, indexBits, fields); + Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(indexBits), rs1, simm12)); +} + +void RiscVEmitter::VLOXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__); + _assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__); + _assert_msg_((int)DecodeReg(vd) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXO, indexBits, fields); + Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(indexBits), rs1, simm12)); +} + +void RiscVEmitter::VLSEGEFF_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__); + s32 simm12 = VecLSToSimm12(VLSUMop::ELEMS_LOAD_FF, vm, VMop::UNIT, dataBits, fields); + Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(dataBits), rs1, simm12)); +} + +void RiscVEmitter::VLR_V(int regs, int hintBits, RiscVReg vd, RiscVReg rs1) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vd), "%s vd must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__); + _assert_msg_(regs == 1 || regs == 2 || regs == 4 || regs == 8, "%s can only access count=1/2/4/8 at a time, not %d", __func__, regs); + _assert_msg_(((int)DecodeReg(vd) & (regs - 1)) == 0, "%s base reg must align to reg count", __func__); + _assert_msg_((int)DecodeReg(vd) + regs <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(VLSUMop::REG, VUseMask::NONE, VMop::UNIT, hintBits, regs); + Write32(EncodeI(Opcode32::LOAD_FP, vd, VecBitsToFunct3(hintBits), rs1, simm12)); +} + +void RiscVEmitter::VSM_V(RiscVReg vs3, RiscVReg rs1) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__); + s32 simm12 = VecLSToSimm12(VLSUMop::MASK, VUseMask::NONE, VMop::UNIT, 8, 1); + Write32(EncodeI(Opcode32::STORE_FP, vs3, Funct3::VLS_8, rs1, simm12)); +} + +void RiscVEmitter::VSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__); + _assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(VLSUMop::ELEMS, vm, VMop::UNIT, dataBits, fields); + Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(dataBits), rs1, simm12)); +} + +void RiscVEmitter::VSSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, RiscVReg rs2, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__); + _assert_msg_(IsGPR(rs2), "%s rs2 (stride) must be GPR", __func__); + _assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(rs2, vm, VMop::STRIDE, dataBits, fields); + Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(dataBits), rs1, simm12)); +} + +void RiscVEmitter::VSUXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__); + _assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__); + _assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXU, indexBits, fields); + Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(indexBits), rs1, simm12)); +} + +void RiscVEmitter::VSOXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 (base) must be GPR", __func__); + _assert_msg_(IsVPR(vs2), "%s vs2 (stride) must be VPR", __func__); + _assert_msg_((int)DecodeReg(vs3) + fields <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(vs2, vm, VMop::INDEXO, indexBits, fields); + Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(indexBits), rs1, simm12)); +} + +void RiscVEmitter::VSR_V(int regs, RiscVReg vs3, RiscVReg rs1) { + _assert_msg_(SupportsVector(), "%s instruction not supported", __func__); + _assert_msg_(IsVPR(vs3), "%s vs3 must be VPR", __func__); + _assert_msg_(IsGPR(rs1), "%s rs1 must be GPR", __func__); + _assert_msg_(regs == 1 || regs == 2 || regs == 4 || regs == 8, "%s can only access count=1/2/4/8 at a time, not %d", __func__, regs); + _assert_msg_(((int)DecodeReg(vs3) & (regs - 1)) == 0, "%s base reg must align to reg count", __func__); + _assert_msg_((int)DecodeReg(vs3) + regs <= 32, "%s cannot access beyond V31", __func__); + s32 simm12 = VecLSToSimm12(VLSUMop::REG, VUseMask::NONE, VMop::UNIT, 8, regs); + Write32(EncodeI(Opcode32::STORE_FP, vs3, VecBitsToFunct3(8), rs1, simm12)); +} + bool RiscVEmitter::AutoCompress() const { return SupportsCompressed() && autoCompress_; } diff --git a/Common/RiscVEmitter.h b/Common/RiscVEmitter.h index e1b8266a56..0738d3bf80 100644 --- a/Common/RiscVEmitter.h +++ b/Common/RiscVEmitter.h @@ -42,6 +42,11 @@ enum RiscVReg { F8, F9, F10, F11, F12, F13, F14, F15, F16, F17, F18, F19, F20, F21, F22, F23, F24, F25, F26, F27, F28, F29, F30, F31, + + V0 = 0x40, V1, V2, V3, V4, V5, V6, V7, + V8, V9, V10, V11, V12, V13, V14, V15, + V16, V17, V18, V19, V20, V21, V22, V23, + V24, V25, V26, V27, V28, V29, V30, V31, }; enum class FixupBranchType { @@ -99,6 +104,14 @@ enum class Csr { FRm = 0x002, FCsr = 0x003, + VStart = 0x008, + VXSat = 0x009, + VXRm = 0x00A, + VCsr = 0x00F, + VL = 0xC20, + VType = 0xC21, + VLenB = 0xC22, + Cycle = 0xC00, Time = 0xC01, InstRet = 0xC02, @@ -107,6 +120,61 @@ enum class Csr { InstRetH = 0xC82, }; +enum class VLMul { + M1 = 0b000, + M2 = 0b001, + M4 = 0b010, + M8 = 0b011, + MF8 = 0b101, + MF4 = 0b110, + MF2 = 0b111, +}; + +enum class VSew { + E8 = 0b000, + E16 = 0b001, + E32 = 0b010, + E64 = 0b011, +}; + +enum class VTail { + U = 0, + A = 1, +}; + +enum class VMask { + U = 0, + A = 1, +}; + +struct VType { + constexpr VType(VSew sew, VTail vt, VMask vm) + : value(((uint32_t)sew << 3) | ((uint32_t)vt << 6) | ((uint32_t)vm << 7)) { + } + constexpr VType(VSew sew, VLMul lmul, VTail vt, VMask vm) + : value((uint32_t)lmul | ((uint32_t)sew << 3) | ((uint32_t)vt << 6) | ((uint32_t)vm << 7)) { + } + + VType(int bits, VLMul lmul, VTail vt, VMask vm) { + VSew sew = VSew::E8; + switch (bits) { + case 8: sew = VSew::E8; break; + case 16: sew = VSew::E16; break; + case 32: sew = VSew::E32; break; + case 64: sew = VSew::E64; break; + default: _assert_msg_(false, "Invalid vtype width"); break; + } + value = (uint32_t)lmul | ((uint32_t)sew << 3) | ((uint32_t)vt << 6) | ((uint32_t)vm << 7); + } + + uint32_t value; +}; + +enum class VUseMask { + V0_T = 0, + NONE = 1, +}; + struct FixupBranch { FixupBranch() {} FixupBranch(const u8 *p, FixupBranchType t) : ptr(p), type(t) {} @@ -347,6 +415,61 @@ public: FSRMI(R_ZERO, rm); } + // Vector instructions. + void VSETVLI(RiscVReg rd, RiscVReg rs1, VType vtype); + void VSETIVLI(RiscVReg rd, u8 uimm5, VType vtype); + void VSETVL(RiscVReg rd, RiscVReg rs1, RiscVReg rs2); + + // Load contiguous registers, unordered. + void VLE_V(int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE) { + VLSEGE_V(1, dataBits, vd, rs1, vm); + } + // Load registers with stride (note: rs2/stride can be X0/zero to broadcast.) + void VLSE_V(int dataBits, RiscVReg vd, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE) { + VLSSEGE_V(1, dataBits, vd, rs1, rs2, vm); + } + // Load indexed registers (gather), unordered. + void VLUXEI_V(int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) { + VLUXSEGEI_V(1, indexBits, vd, rs1, vs2, vm); + } + // Load indexed registers (gather), ordered. + void VLOXEI_V(int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) { + VLOXSEGEI_V(1, indexBits, vd, rs1, vs2, vm); + } + // Load mask (force 8 bit, EMUL=1, TA) + void VLM_V(RiscVReg vd, RiscVReg rs1); + // Load but ignore faults after first element. + void VLEFF_V(int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE) { + VLSEGEFF_V(1, dataBits, vd, rs1, vm); + } + // Load fields into subsequent registers (destructure.) + void VLSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE); + void VLSSEGE_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE); + void VLUXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE); + void VLOXSEGEI_V(int fields, int indexBits, RiscVReg vd, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE); + void VLSEGEFF_V(int fields, int dataBits, RiscVReg vd, RiscVReg rs1, VUseMask vm = VUseMask::NONE); + // Load entire registers (implementation dependent size.) + void VLR_V(int regs, int hintBits, RiscVReg vd, RiscVReg rs1); + + void VSE_V(int dataBits, RiscVReg vs3, RiscVReg rs1, VUseMask vm = VUseMask::NONE) { + VSSEGE_V(1, dataBits, vs3, rs1, vm); + } + void VSSE_V(int dataBits, RiscVReg vs3, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE) { + VSSSEGE_V(1, dataBits, vs3, rs1, rs2, vm); + } + void VSUXEI_V(int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) { + VSUXSEGEI_V(1, indexBits, vs3, rs1, vs2, vm); + } + void VSOXEI_V(int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE) { + VSOXSEGEI_V(1, indexBits, vs3, rs1, vs2, vm); + } + void VSM_V(RiscVReg vs3, RiscVReg rs1); + void VSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, VUseMask vm = VUseMask::NONE); + void VSSSEGE_V(int fields, int dataBits, RiscVReg vs3, RiscVReg rs1, RiscVReg rs2, VUseMask vm = VUseMask::NONE); + void VSUXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE); + void VSOXSEGEI_V(int fields, int indexBits, RiscVReg vs3, RiscVReg rs1, RiscVReg vs2, VUseMask vm = VUseMask::NONE); + void VSR_V(int regs, RiscVReg vs3, RiscVReg rs1); + // Compressed instructions. void C_ADDI4SPN(RiscVReg rd, u32 nzuimm10); void C_FLD(RiscVReg rd, RiscVReg addr, u8 uimm8);