From 10d2e5dd40c3f99f975ab2101955128a75847984 Mon Sep 17 00:00:00 2001 From: JordanTheToaster Date: Fri, 27 Feb 2026 14:31:40 +0000 Subject: [PATCH] 3rdparty: Update xbyak to v7.35.2 --- 3rdparty/xbyak/xbyak/xbyak.h | 107 +++- 3rdparty/xbyak/xbyak/xbyak_mnemonic.h | 21 +- 3rdparty/xbyak/xbyak/xbyak_util.h | 861 +++++++++++++++++++++++++- 3 files changed, 943 insertions(+), 46 deletions(-) diff --git a/3rdparty/xbyak/xbyak/xbyak.h b/3rdparty/xbyak/xbyak/xbyak.h index 81f08d2258..623281bd46 100644 --- a/3rdparty/xbyak/xbyak/xbyak.h +++ b/3rdparty/xbyak/xbyak/xbyak.h @@ -37,8 +37,19 @@ #define XBYAK_GNUC_PREREQ(major, minor) 0 #endif +// User defined (must define all 3) +#if defined(XBYAK_STD_UNORDERED_SET) || defined(XBYAK_STD_UNORDERED_MAP) || defined(XBYAK_STD_UNORDERED_MULTIMAP) + #ifndef XBYAK_STD_UNORDERED_SET + #error "Define XBYAK_STD_UNORDERED_SET" + #endif + #ifndef XBYAK_STD_UNORDERED_MAP + #error "Define XBYAK_STD_UNORDERED_MAP" + #endif + #ifndef XBYAK_STD_UNORDERED_MULTIMAP + #error "Define XBYAK_STD_UNORDERED_MULTIMAP" + #endif // This covers -std=(gnu|c)++(0x|11|1y), -stdlib=libc++, and modern Microsoft. -#if ((defined(_MSC_VER) && (_MSC_VER >= 1600)) || defined(_LIBCPP_VERSION) ||\ +#elif ((defined(_MSC_VER) && (_MSC_VER >= 1600)) || defined(_LIBCPP_VERSION) ||\ ((__cplusplus >= 201103) || defined(__GXX_EXPERIMENTAL_CXX0X__))) #include #define XBYAK_STD_UNORDERED_SET std::unordered_set @@ -163,7 +174,7 @@ namespace Xbyak { enum { DEFAULT_MAX_CODE_SIZE = 4096, - VERSION = 0x7300 /* 0xABCD = A.BC(.D) */ + VERSION = 0x7352 /* 0xABCD = A.BC(.D) */ }; #ifndef MIE_INTEGER_TYPE_DEFINED @@ -242,6 +253,8 @@ enum { ERR_INVALID_REG_IDX, ERR_BAD_ENCODING_MODE, ERR_CANT_USE_ABCDH, + ERR_CANT_INIT_CPUTOPOLOGY, + ERR_INVALID_CPUMASK_INDEX, ERR_INTERNAL // Put it at last. }; @@ -302,6 +315,8 @@ inline const char *ConvertErrorToString(int err) "invalid reg index", "bad encoding mode", "can't use [abcd]h with rex", + "can't init CpuTopology", + "invalid cpumask index", "internal error" }; assert(ERR_INTERNAL + 1 == sizeof(errTbl) / sizeof(*errTbl)); @@ -518,7 +533,13 @@ public: } } #endif - void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, mode, fd, 0); + int prot = PROT_READ | PROT_WRITE; +#ifdef PROT_MPROTECT + // Some NetBSD systems have this protection turned on by default + // https://man.netbsd.org/mprotect.2 + prot |= PROT_MPROTECT(PROT_READ | PROT_WRITE | PROT_EXEC); +#endif + void *p = mmap(NULL, size, prot, mode, fd, 0); if (p == MAP_FAILED) { if (fd != -1) close(fd); XBYAK_THROW_RET(ERR_CANT_ALLOC, 0) @@ -984,14 +1005,14 @@ public: #else enum { i32e = 32 }; #endif - XBYAK_CONSTEXPR RegExp() : scale_(0), disp_(0), label_(0), rip_(false), setLabel_(false) { } - XBYAK_CONSTEXPR RegExp(size_t disp) : scale_(0), disp_(disp), label_(0), rip_(false), setLabel_(false) { } + XBYAK_CONSTEXPR RegExp() : scale_(0), disp_(0), label_(0), rip_(false), asPtr_(false) { } + XBYAK_CONSTEXPR RegExp(size_t disp) : scale_(0), disp_(disp), label_(0), rip_(false), asPtr_(false) { } XBYAK_CONSTEXPR RegExp(const Reg& r, int scale = 1) : scale_(scale) , disp_(0) , label_(0) , rip_(false) - , setLabel_(false) + , asPtr_(false) { if (!r.isREG(i32e) && !r.is(Reg::XMM|Reg::YMM|Reg::ZMM|Reg::TMM)) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) if (scale == 0) return; @@ -1004,21 +1025,22 @@ public: } RegExp(Label& label); - RegExp(const void *addr) - : scale_(1) + // can't use constexpr to const void * + explicit RegExp(const void *addr) + : scale_(0) , disp_(size_t(addr)) , label_(0) , rip_(false) - , setLabel_(true) + , asPtr_(true) { } #ifdef XBYAK64 - RegExp(const RegRip& /*rip*/) + XBYAK_CONSTEXPR RegExp(const RegRip& /*rip*/) : scale_(0) , disp_(0) , label_(0) , rip_(true) - , setLabel_(false) + , asPtr_(false) { } #endif @@ -1052,6 +1074,7 @@ public: } } friend RegExp operator+(const RegExp& a, const RegExp& b); + friend RegExp operator+(const RegExp& e, size_t disp); friend RegExp operator-(const RegExp& e, size_t disp); private: /* @@ -1064,7 +1087,7 @@ private: size_t disp_; // absolute address Label *label_; bool rip_; - bool setLabel_; // disp_ contains the address of label + bool asPtr_; // disp_ contains a pointer }; inline RegExp operator+(const RegExp& a, const RegExp& b) @@ -1073,10 +1096,10 @@ inline RegExp operator+(const RegExp& a, const RegExp& b) if (a.label_ && b.label_) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegExp()) if (b.rip_) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegExp()) if (a.rip_ && !b.isOnlyDisp()) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegExp()) - if (a.setLabel_ && b.setLabel_) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegExp()) + if (a.asPtr_ && b.asPtr_) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegExp()) RegExp ret = a; if (ret.label_ == 0) ret.label_ = b.label_; - if (ret.setLabel_ == 0) ret.setLabel_ = b.setLabel_; + if (ret.asPtr_ == 0) ret.asPtr_ = b.asPtr_; if (!ret.index_.getBit()) { ret.index_ = b.index_; ret.scale_ = b.scale_; } if (b.base_.getBit()) { if (ret.base_.getBit()) { @@ -1101,8 +1124,19 @@ inline RegExp operator*(int scale, const Reg& r) { return r * scale; } -// backward compatibility for eax+0 -inline RegExp operator+(const RegExp& a, size_t b) { return a + RegExp(b); } + +// backward compatibility for eax+&x (pointer address) +inline RegExp operator+(const RegExp& a, const void* b) { return a + RegExp(b); } + +// overload for integer literals (e.g. eax+0) to avoid ambiguity with the void* overload +inline RegExp operator+(const RegExp& e, int disp) { return e + size_t(disp); } + +inline RegExp operator+(const RegExp& e, size_t disp) +{ + RegExp ret = e; + ret.disp_ += disp; + return ret; +} inline RegExp operator-(const RegExp& e, size_t disp) { @@ -1351,11 +1385,15 @@ public: class Address : public Operand { public: + XBYAK_CONSTEXPR Address() + : Operand(0, MEM, 0), e_(), label_(NULL), mode_(inner::M_ModRM), immSize(0), + disp8N(0), permitVsib(false), broadcast_(false), optimize_(true) { } XBYAK_CONSTEXPR Address(uint32_t sizeBit, bool broadcast, const RegExp& e) - : Operand(0, MEM, sizeBit), e_(e), label_(e.label_), mode_(), immSize(0), disp8N(0), permitVsib(false), broadcast_(broadcast), optimize_(true) + : Operand(0, MEM, sizeBit), e_(e), label_(e.label_), mode_(), immSize(0), + disp8N(0), permitVsib(false), broadcast_(broadcast), optimize_(true) { if (e.rip_) { - mode_ = (e.label_ || e.setLabel_) ? inner::M_ripAddr : inner::M_rip; + mode_ = (e.label_ || e.asPtr_) ? inner::M_ripAddr : inner::M_rip; } else { #ifdef XBYAK64 uint64_t disp = e.getDisp(); @@ -1435,6 +1473,10 @@ public: { return Address(bit_, broadcast_, e); } + Address operator[](const void *addr) const + { + return operator[](RegExp(addr)); + } }; struct JmpLabel { @@ -1483,7 +1525,7 @@ inline RegExp::RegExp(Label& label) , disp_(0) , label_(0) , rip_(false) - , setLabel_(true) + , asPtr_(true) { const uint8_t *addr = label.getAddress(); if (addr) { @@ -2253,6 +2295,7 @@ private: size_t disp = addr.getDisp(); if (addr.getMode() == inner::M_ripAddr) { if (isAutoGrow()) XBYAK_THROW(ERR_INVALID_RIP_IN_AUTO_GROW) + // compute the relative offset to the pointer address disp -= (size_t)getCurr() + 4 + addr.immSize; } dd(inner::VerifyInInt32(disp)); @@ -3353,11 +3396,14 @@ public: opAVX10ZeroExt(op1, op2, typeTbl, codeTbl, enc, 16|32|64); } /* - use single byte nop if useMultiByteNop = false + useMultiByteNop + = 0: use only single byte nop + = 1: recommended multi-byte + = 2: better for newer CPUs */ - void nop(size_t size = 1, bool useMultiByteNop = true) + void nop(size_t size = 1, int useMultiByteNop = 2) { - if (!useMultiByteNop) { + if (useMultiByteNop == 0) { for (size_t i = 0; i < size; i++) { db(0x90); } @@ -3368,8 +3414,9 @@ public: recommended multi-byte sequence of NOP instruction AMD and Intel seem to agree on the same sequences for up to 9 bytes: https://support.amd.com/TechDocs/55723_SOG_Fam_17h_Processors_3.00.pdf + 10~15 byte nop in Software Optimization Guide for the AMD Zen4 Microarchitecture No. 57647 */ - static const uint8_t nopTbl[9][9] = { + static const uint8_t nopTbl[][15] = { {0x90}, {0x66, 0x90}, {0x0F, 0x1F, 0x00}, @@ -3378,9 +3425,15 @@ public: {0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00}, {0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00}, {0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, - {0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, // 9 + {0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, // 11 + {0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x66, 0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}, }; - const size_t n = sizeof(nopTbl) / sizeof(nopTbl[0]); + const size_t n = useMultiByteNop == 2 ? sizeof(nopTbl) / sizeof(nopTbl[0]) : 9; while (size > 0) { size_t len = (std::min)(n, size); const uint8_t *seq = nopTbl[len - 1]; @@ -3391,9 +3444,9 @@ public: #ifndef XBYAK_DONT_READ_LIST #include "xbyak_mnemonic.h" /* - use single byte nop if useMultiByteNop = false + use single byte nop if useMultiByteNop = 0 */ - void align(size_t x = 16, bool useMultiByteNop = true) + void align(size_t x = 16, int useMultiByteNop = 2) { if (x == 1) return; if (x < 1 || (x & (x - 1))) XBYAK_THROW(ERR_BAD_ALIGN) diff --git a/3rdparty/xbyak/xbyak/xbyak_mnemonic.h b/3rdparty/xbyak/xbyak/xbyak_mnemonic.h index 5f75098f97..961d88f85f 100644 --- a/3rdparty/xbyak/xbyak/xbyak_mnemonic.h +++ b/3rdparty/xbyak/xbyak/xbyak_mnemonic.h @@ -1,4 +1,4 @@ -const char *getVersionString() const { return "7.30"; } +const char *getVersionString() const { return "7.35.2"; } void aadd(const Address& addr, const Reg32e ®) { opMR(addr, reg, T_0F38, 0x0FC, T_APX); } void aand(const Address& addr, const Reg32e ®) { opMR(addr, reg, T_0F38|T_66, 0x0FC, T_APX|T_66); } void adc(const Operand& op, uint32_t imm) { opOI(op, imm, 0x10, 2); } @@ -848,6 +848,7 @@ void por(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0xEB); } void prefetchit0(const Address& addr) { opMR(addr, Reg32(7), T_0F, 0x18); } void prefetchit1(const Address& addr) { opMR(addr, Reg32(6), T_0F, 0x18); } void prefetchnta(const Address& addr) { opMR(addr, Reg32(0), T_0F, 0x18); } +void prefetchrst2(const Address& addr) { opMR(addr, Reg32(4), T_0F, 0x18); } void prefetcht0(const Address& addr) { opMR(addr, Reg32(1), T_0F, 0x18); } void prefetcht1(const Address& addr) { opMR(addr, Reg32(2), T_0F, 0x18); } void prefetcht2(const Address& addr) { opMR(addr, Reg32(3), T_0F, 0x18); } @@ -1874,7 +1875,7 @@ void clui() { db(0xF3); db(0x0F); db(0x01); db(0xEE); } void stui() { db(0xF3); db(0x0F); db(0x01); db(0xEF); } void testui() { db(0xF3); db(0x0F); db(0x01); db(0xED); } void uiret() { db(0xF3); db(0x0F); db(0x01); db(0xEC); } -void cmpxchg16b(const Address& addr) { opMR(addr, Reg64(1), T_0F, 0xC7); } +void cmpxchg16b(const Address& addr) { opMR(addr, Reg64(1), T_0F|T_ALLOW_DIFF_SIZE, 0xC7); } void fxrstor64(const Address& addr) { opMR(addr, Reg64(1), T_0F, 0xAE); } void movq(const Reg64& reg, const Mmx& mmx) { if (mmx.isXMM()) db(0x66); opSSE(mmx, reg, T_0F, 0x7E); } void movq(const Mmx& mmx, const Reg64& reg) { if (mmx.isXMM()) db(0x66); opSSE(mmx, reg, T_0F, 0x6E); } @@ -1937,31 +1938,15 @@ void tdphf8ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, void tmmultf32ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_66|T_0F38|T_W0, 0x48); } void tcmmimfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_66|T_0F38|T_W0, 0x6C); } void tcmmrlfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_0F38|T_W0, 0x6C); } -void tconjtcmmimfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_0F38|T_W0, 0x6B); } -void ttcmmimfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_F2|T_0F38|T_W0, 0x6B); } -void ttcmmrlfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_F3|T_0F38|T_W0, 0x6B); } -void ttdpbf16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_F3|T_0F38|T_W0, 0x6C); } -void ttdpfp16ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_F2|T_0F38|T_W0, 0x6C); } -void ttmmultf32ps(const Tmm& x1, const Tmm& x2, const Tmm& x3) { opVex(x1, &x3, x2, T_0F38|T_W0, 0x48); } void tileloadd(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_F2|T_0F38|T_W0, 0x4B); } void tileloaddt1(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_66|T_0F38|T_W0, 0x4B); } void tileloaddrs(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_F2|T_0F38|T_W0, 0x4A); } void tileloaddrst1(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_66|T_0F38|T_W0, 0x4A); } -void t2rpntlvwz0(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_0F38|T_W0, 0x6E); } -void t2rpntlvwz0t1(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_0F38|T_W0, 0x6F); } -void t2rpntlvwz1(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_66|T_0F38|T_W0, 0x6E); } -void t2rpntlvwz1t1(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_66|T_0F38|T_W0, 0x6F); } -void t2rpntlvwz0rs(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_MAP5|T_W0, 0xF8); } -void t2rpntlvwz0rst1(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_MAP5|T_W0, 0xF9); } -void t2rpntlvwz1rs(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_66|T_MAP5|T_W0, 0xF8); } -void t2rpntlvwz1rst1(const Tmm& tm, const Address& addr) { opAMX(tm, addr, T_66|T_MAP5|T_W0, 0xF9); } void ldtilecfg(const Address& addr) { opAMX(tmm0, addr, T_0F38|T_W0, 0x49); } void sttilecfg(const Address& addr) { opAMX(tmm0, addr, T_66|T_0F38|T_W0, 0x49); } void tilestored(const Address& addr, const Tmm& tm) { opAMX(tm, addr, T_F3|T_0F38|T_W0, 0x4B); } void tilerelease() { db(0xc4); db(0xe2); db(0x78); db(0x49); db(0xc0); } void tilezero(const Tmm& t) { opVex(t, &tmm0, tmm0, T_F2|T_0F38|T_W0, 0x49); } -void tconjtfp16(const Tmm& t1, const Tmm& t2) { opVex(t1, 0, t2, T_66|T_0F38|T_W0, 0x6B); } -void ttransposed(const Tmm& t1, const Tmm& t2) { opVex(t1, 0, t2, T_F3|T_0F38|T_W0, 0x5F); } #else void jcxz(std::string label) { db(0x67); opJmp(label, T_SHORT, 0xe3, 0, 0); } void jcxz(const Label& label) { db(0x67); opJmp(label, T_SHORT, 0xe3, 0, 0); } diff --git a/3rdparty/xbyak/xbyak/xbyak_util.h b/3rdparty/xbyak/xbyak/xbyak_util.h index 3cee6c1395..cb6365c951 100644 --- a/3rdparty/xbyak/xbyak/xbyak_util.h +++ b/3rdparty/xbyak/xbyak/xbyak_util.h @@ -15,9 +15,11 @@ #else #define XBYAK_CONSTEXPR #endif +#define XBYAK_CPUMASK_COMPACT 0 #endif #else #include +#include /** utility class and functions for Xbyak @@ -86,6 +88,32 @@ #define XBYAK_USE_PERF #endif +#ifndef XBYAK_CPU_CACHE + #define XBYAK_CPU_CACHE 1 +#endif +#if XBYAK_CPU_CACHE == 1 +#include +#ifndef XBYAK_CPUMASK_COMPACT + #define XBYAK_CPUMASK_COMPACT 1 +#endif +#if XBYAK_CPUMASK_COMPACT == 0 + #include +#endif +#ifdef _WIN32 +#include +#endif +namespace Xbyak { namespace util { +class CpuTopology; +class Cpu; +namespace impl { + +bool initCpuTopology(CpuTopology& cpuTopo); + +} // Xbyak::util::impl +} } // Xbyak::util +#endif // XBYAK_CPU_CACHE + + namespace Xbyak { namespace util { typedef enum { @@ -421,7 +449,7 @@ public: static inline void getCpuidEx(uint32_t eaxIn, uint32_t ecxIn, uint32_t data[4]) { #ifdef XBYAK_INTEL_CPU_SPECIFIC - #ifdef _WIN32 + #ifdef _MSC_VER __cpuidex(reinterpret_cast(data), eaxIn, ecxIn); #else __cpuid_count(eaxIn, ecxIn, data[0], data[1], data[2], data[3]); @@ -558,6 +586,7 @@ public: XBYAK_DEFINE_TYPE(94, tAMX_MOVRS); XBYAK_DEFINE_TYPE(95, tAMX_FP8); XBYAK_DEFINE_TYPE(96, tMOVRS); + XBYAK_DEFINE_TYPE(97, tHYBRID); #undef XBYAK_SPLIT_ID #undef XBYAK_DEFINE_TYPE @@ -695,6 +724,7 @@ public: if (ecx & (1U << 28)) type_ |= tMOVDIR64B; if (edx & (1U << 5)) type_ |= tUINTR; if (edx & (1U << 14)) type_ |= tSERIALIZE; + if (edx & (1U << 15)) type_ |= tHYBRID; if (edx & (1U << 16)) type_ |= tTSXLDTRK; if (edx & (1U << 22)) type_ |= tAMX_BF16; if (edx & (1U << 24)) type_ |= tAMX_TILE; @@ -761,6 +791,820 @@ public: #endif #ifndef XBYAK_ONLY_CLASS_CPU +#if XBYAK_CPU_CACHE == 1 + +enum CoreType { + Unknown, + Performance, // P-core (Intel) + Efficient, // E-core (Intel) + Standard // Non-hybrid +}; + +inline const char *getCoreTypeStr(int coreType) +{ + switch (coreType) { + case Performance: return "P-core"; + case Efficient: return "E-core"; + case Standard: return "Standard"; + default: return "Unknown"; + } +} + +enum CacheType { + L1i, + L1d, + L2, + L3, + CACHE_UNKNOWN, + CACHE_TYPE_NUM = CACHE_UNKNOWN +}; + +inline const char* getCacheTypeStr(int type) +{ + switch (type) { + case L1i: return "L1i"; + case L1d: return "L1d"; + case L2: return "L2"; + case L3: return "L3"; + default: return "Unknown"; + } +} + +namespace impl { + +inline void appendStr(std::string& s, uint32_t v) +{ +#if __cplusplus >= 201103L + s += std::to_string(v); +#else + char buf[16]; + snprintf(buf, sizeof(buf), "%u", v); + s += buf; +#endif +} + +// str = "(int|range)[,(int|range)]*" +// range = int-int +// e.g. "1,3,5", "0-3,5-7", "" +template +bool setStr(T& x, const char *str) +{ + const char *p = str; + while (*p) { + if (p != str) { + if (*p != ',') return false; + p++; + } + char *endp; + uint32_t v = uint32_t(strtoul(p, &endp, 10)); + if (endp == p) return false; + if (*endp == '-') { + const char *rangeStart = endp + 1; + uint32_t next = uint32_t(strtoul(rangeStart, &endp, 10)); + if (endp == rangeStart) return false; + if (!x.appendRange(v, next)) return false; + } else { + if (!x.append(v)) return false; + } + if (*endp == '\0') return true; + p = endp; + } + return true; +} + +} // impl + +#ifndef XBYAK_CPUMASK_N +#define XBYAK_CPUMASK_N 6 +#endif +#ifndef XBYAK_CPUMASK_BITN +#define XBYAK_CPUMASK_BITN 10 // max number of logical cpu = 1024 +#endif +#if XBYAK_CPUMASK_COMPACT == 1 +/* + a_ is treated as an array of N elements, each being bitN bits + a_ = 1<> (idx*bitN)) & mask; + } +#ifndef NDEBUG + // Return true if the idx-th value exists + bool hasNext(uint32_t idx) const + { + if (empty()) return false; + if (!range_) return idx <= n_; + uint32_t n = 0; + for (uint32_t i = 1; i <= n_; i += 2) { + n += get_a(i) + 1; + if (idx < n) return true; + } + return false; + } +#endif +public: + CpuMask() { clear(); } + class ConstIterator { + const CpuMask& parent_; + uint32_t idx_; + uint32_t size_; + friend class CpuMask; + public: + ConstIterator(const CpuMask& parent) + : parent_(parent), idx_(0), size_(uint32_t(parent.size())) {} + uint32_t operator*() const { return parent_.get(idx_); } + ConstIterator& operator++() { idx_++; return *this; } + bool operator==(const ConstIterator& rhs) const { return idx_ == rhs.idx_; } + bool operator!=(const ConstIterator& rhs) const { return !operator==(rhs); } + }; + ConstIterator begin() const { return ConstIterator(*this); } + ConstIterator end() const { + ConstIterator it(*this); + it.idx_ = uint32_t(size()); + return it; + } + typedef ConstIterator iterator; + typedef ConstIterator const_iterator; + void clear() { a_ = 1 << bitN; n_ = 0; range_ = 0; } + bool empty() const + { + return a_ == 1 << bitN && n_ == 0 && range_ == 0; + } + uint64_t to_u64() const { return a_ | (uint64_t(n_) << (N * bitN)) | (uint64_t(range_) << (N * bitN + 3)); } + bool operator<(const CpuMask& rhs) const { return to_u64() < rhs.to_u64(); } + bool operator>(const CpuMask& rhs) const { return to_u64() > rhs.to_u64(); } + bool operator>=(const CpuMask& rhs) const { return !operator<(rhs); } + bool operator<=(const CpuMask& rhs) const { return !operator>(rhs); } + bool operator==(const CpuMask& rhs) const { return to_u64() == rhs.to_u64(); } + bool operator!=(const CpuMask& rhs) const { return !operator==(rhs); } + // Add element v + // v should be monotonically increasing + bool append(uint32_t v) + { + uint32_t prev = 0, n = 0; + if (v > mask) goto ERR; + // When adding for the first time, treat as discrete value + if (empty()) { + a_ = v; + n_ = 0; + return true; + } + if (!range_) { + prev = get_a(n_); + if (v <= prev) goto ERR; + // If there's one discrete value and it forms an interval with the new value, switch to interval mode + if (n_ == 0 && prev + 1 == v) { + set_a(1, 1); + range_ = 1; + n_ = 1; + return true; + } + if (n_ >= N - 1) goto ERR; + // Add discrete value + n_++; + set_a(n_, v); + return true; + } + // If the value to add is 1 greater than the end of the current interval + n = get_a(n_); + prev = get_a(n_ - 1) + n; + if (prev >= v) goto ERR; + if (prev + 1 == v) { + // Increase the interval length by one + set_a(n_, n + 1); + return true; + } else { + if (n_ >= N - 1) goto ERR; + // If not continuous with the previous interval + // Add a new interval [v] + set_a(n_ + 1, v); + n_ += 2; + return true; + } + ERR: + XBYAK_THROW_RET(ERR_INVALID_CPUMASK_INDEX, false) + } + // add range [a, b] which means a, a+1, ..., b + bool appendRange(uint32_t a, uint32_t b) + { + if ((empty() || (range_ && n_ < N - 1)) && (a <= b && b <= mask)) { + range_ = true; + n_ += n_ == 0 ? 1 : 2; + set_a(n_ - 1, a); + set_a(n_, b - a); + return true; + } + return false; + } + // str = "(int|range)[,(int|range)]*" + // range = int-int + bool setStr(const char *str) + { + return impl::setStr(*this, str); + } + bool setStr(const std::string& str) { return setStr(str.c_str()); } + std::string getStr() const + { + std::string s; + if (empty()) return s; + if (!range_) { + for (uint32_t i = 0; i <= n_; i++) { + if (!s.empty()) s += ","; + impl::appendStr(s, get_a(i)); + } + return s; + } + for (uint32_t i = 0; i <= n_; i += 2) { + uint32_t v = get_a(i); + uint32_t len = get_a(i + 1); + if (!s.empty()) s += ","; + impl::appendStr(s, v); + if (len > 0) { + s += "-"; + impl::appendStr(s, v + len); + } + } + return s; + } + size_t size() const + { + if (empty()) return 0; + if (!range_) return n_ + 1; + size_t n = 0; + for (uint32_t i = 1; i <= n_; i += 2) { + n += get_a(i) + 1; + } + return n; + } + + uint32_t get(uint32_t idx) const + { + assert(hasNext(idx)); + if (!range_) return get_a(idx); + uint32_t n = 0; + for (uint32_t i = 1; i <= n_; i += 2) { + uint32_t range = get_a(i) + 1; + if (idx < n + range) { + return get_a(i - 1) + (idx - n); + } + n += range; + } + return false; + } + void dump() const + { + printf("a_:"); + for (int i = int(N) - 1; i >= 0; i--) { + printf("%u ", uint32_t((a_ >> (i * bitN)) & mask)); + } + printf("\n"); + printf("n_: %u\n", (uint32_t)n_); + printf("range_: %u\n", (uint32_t)range_); + } + void put(const char *label = NULL) const + { + if (label) printf("%s: ", label); + printf("%s\n", getStr().c_str()); + } +}; +#else +class CpuMask { + typedef std::set IntSet; + IntSet indices_; +public: + CpuMask() : indices_() {} + typedef IntSet::const_iterator const_iterator; + typedef const_iterator iterator; + const_iterator begin() const { return indices_.begin(); } + const_iterator end() const { return indices_.end(); } + + void clear() { indices_.clear(); } + bool empty() const { return indices_.empty(); } + bool operator<(const CpuMask& rhs) const { return indices_ < rhs.indices_; } + bool operator>(const CpuMask& rhs) const { return indices_ > rhs.indices_; } + bool operator>=(const CpuMask& rhs) const { return !operator<(rhs); } + bool operator<=(const CpuMask& rhs) const { return !operator>(rhs); } + bool operator==(const CpuMask& rhs) const { return indices_ == rhs.indices_; } + bool operator!=(const CpuMask& rhs) const { return !operator==(rhs); } + // idx should be monotonically increasing + bool append(uint32_t idx) + { + if (idx >= (1u << XBYAK_CPUMASK_BITN)) return false; + if (!indices_.empty() && *indices_.rbegin() >= idx) return false; + indices_.insert(idx); + return true; + } + // add range [a, b] which means a, a+1, ..., b + bool appendRange(uint32_t a, uint32_t b) + { + if (a > b) return false; + while (a <= b) { + if (!append(a)) return false; + a++; + } + return true; + } + bool setStr(const char *str) + { + return impl::setStr(*this, str); + } + bool setStr(const std::string& str) { return setStr(str.c_str()); } + std::string getStr() const + { + std::string s; + bool inRange = false; + uint32_t prev = 0x80000000; + for (const_iterator i = indices_.begin(); i != indices_.end(); ++i) { + uint32_t v = *i; + if (inRange) { + if (prev + 1 != v) { + impl::appendStr(s, prev); + inRange = false; + s += ','; + impl::appendStr(s, v); + } + } else { + if (prev + 1 == v) { + // start range + s += '-'; + inRange = true; + } else { + if (!s.empty()) s += ','; + impl::appendStr(s, v); + } + } + prev = v; + } + if (inRange) { + impl::appendStr(s, prev); + } + return s; + } + size_t size() const { return indices_.size(); } + uint32_t get(uint32_t idx) const + { + assert(idx < size()); + const_iterator it = indices_.begin(); + std::advance(it, idx); + return *it; + } + void put(const char *label = NULL) const + { + if (label) printf("%s: ", label); + printf("%s\n", getStr().c_str()); + } +}; +#endif + +class CpuCache { +public: + CpuCache() : size(0), associativity(0) {} + + // Cache size in bytes + uint32_t size; + + // number of ways of associativity + uint32_t associativity; + + // Set of logical CPU indices sharing this cache + CpuMask sharedCpuIndices; + + // Whether this is a shared cache + bool isShared() const { return sharedCpuIndices.size() > 1; } + + // Number of logical CPUs sharing this cache + size_t getSharedCpuNum() const { return sharedCpuIndices.size(); } + + void put(const char *label = NULL) const + { + if (label) printf("%s: ", label); + printf("%u KiB, assoc. %u, shared ", size / 1024, associativity); + sharedCpuIndices.put(); + } +}; + +struct LogicalCpu { + LogicalCpu() + : coreId(0) + , coreType(Unknown) + , cache() + { + } + uint32_t coreId; // index of physical core + CoreType coreType; // for hybrid systems + CpuCache cache[CACHE_TYPE_NUM]; + const CpuMask& getSiblings() const { return cache[L1i].sharedCpuIndices; } + + void put(const char *label = NULL) const + { + if (label) printf("%s: ", label); + printf("coreId %u, type %s\n", coreId, getCoreTypeStr(coreType)); + for (int i = 0; i < CACHE_TYPE_NUM; i++) { + cache[i].put(getCacheTypeStr(i)); + } + } +}; + +class CpuTopology { +public: + explicit CpuTopology(const Cpu& cpu) + : logicalCpus_() + , physicalCoreNum_(0) + , lineSize_(0) + , isHybrid_(cpu.has(cpu.tHYBRID)) + { + if (!impl::initCpuTopology(*this)) { + XBYAK_THROW(ERR_CANT_INIT_CPUTOPOLOGY); + } + } + + // Number of logical CPUs + size_t getLogicalCpuNum() const { return logicalCpus_.size(); } + + // Number of physical cores + size_t getPhysicalCoreNum() const { return physicalCoreNum_; } + + // Cache line size in bytes + uint32_t getLineSize() const { return lineSize_; } + + // Get logical CPU information + const LogicalCpu& getLogicalCpu(size_t cpuIdx) const + { + return logicalCpus_[cpuIdx]; + } + + // Get cache information for a specific logical CPU + const CpuCache& getCache(size_t cpuIdx, CacheType type) const + { + return logicalCpus_[cpuIdx].cache[type]; + } + + // Whether this is a hybrid system + bool isHybrid() const { return isHybrid_; } +private: + friend bool impl::initCpuTopology(CpuTopology&); + std::vector logicalCpus_; + size_t physicalCoreNum_; + uint32_t lineSize_; + bool isHybrid_; +}; + +namespace impl { + +inline uint32_t popcnt(uint64_t mask) +{ +#if defined(_M_X64) || defined(_M_AMD64) + return (int)__popcnt64(mask); +#elif defined(__GNUC__) || defined(__clang__) + return __builtin_popcountll(mask); +#else + uint32_t count = 0; + while (mask) { + count += (mask & 1); + mask >>= 1; + } + return count; +#endif +} + +#ifdef _WIN32 + +typedef std::vector U32Vec; +typedef SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX ProcInfo; + +// return total logical cpus if sucessful, 0 if failed +inline uint32_t getGroupAcc(U32Vec& v) +{ + DWORD len = 0; + GetLogicalProcessorInformationEx(RelationGroup, NULL, &len); + std::vector buf(len); + if (!GetLogicalProcessorInformationEx(RelationGroup, reinterpret_cast(buf.data()), &len)) { + return 0; + } + const auto& entry = *reinterpret_cast(buf.data()); + const GROUP_RELATIONSHIP& gr = entry.Group; + + const uint32_t n = gr.ActiveGroupCount; + if (n == 0) return 0; + + v.resize(n); + + uint32_t acc = 0; + for (uint32_t g = 0; g < n; g++) { + v[g] = acc; + acc += gr.GroupInfo[g].ActiveProcessorCount; + } + return acc; +} + +// return number of physical cores if successful, 0 if failed +static inline uint32_t getCores(std::vector& cpus, bool isHybrid, const U32Vec& groupAcc) { + DWORD len = 0; + GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &len); + std::vector buf(len); + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast(buf.data()), &len)) return 0; + + // get core indices + const char *p = buf.data(); + const char *end = p + len; + uint32_t coreIdx = 0; + + while (p < end) { + const auto& entry = *reinterpret_cast(p); + if (entry.Relationship == RelationProcessorCore) { + const PROCESSOR_RELATIONSHIP& core = entry.Processor; + LogicalCpu cpu; + cpu.coreId = coreIdx++; + if (!isHybrid) { + cpu.coreType = Standard; + } else if (core.EfficiencyClass > 0) { + cpu.coreType = Performance; + } else { + cpu.coreType = Efficient; + } + + const GROUP_AFFINITY* masks = core.GroupMask; + for (WORD i = 0; i < core.GroupCount; i++) { + const WORD group = masks[i].Group; + const KAFFINITY m = masks[i].Mask; + const uint32_t base = groupAcc[group]; + + for (uint32_t b = 0; b < sizeof(KAFFINITY) * 8; b++) { + if (m & (KAFFINITY(1) << b)) { + const uint32_t idx = base + b; + if (idx >= cpus.size()) return 0; + cpus[idx] = cpu; + } + } + } + } + p += entry.Size; + } + return coreIdx; +} + +inline bool convertMask(CpuMask& mask, const U32Vec& groupAcc, const CACHE_RELATIONSHIP& cache) +{ + const GROUP_AFFINITY* masks = cache.GroupMasks; + + for (WORD i = 0; i < cache.GroupCount; i++) { + const WORD group = masks[i].Group; + const KAFFINITY m = masks[i].Mask; + const uint32_t base = groupAcc[group]; + + for (uint32_t b = 0; b < sizeof(KAFFINITY) * 8; b++) { + if (m & (KAFFINITY(1) << b)) { + if (!mask.append(base + b)) return false; + } + } + } + return true; +} + +inline bool initCpuTopology(CpuTopology& cpuTopo) +{ + U32Vec groupAcc; + const uint32_t logicalCpuNum = getGroupAcc(groupAcc); + if (logicalCpuNum == 0) return false; + if (logicalCpuNum >= (1u << XBYAK_CPUMASK_BITN)) return false; + + cpuTopo.logicalCpus_.resize(logicalCpuNum); + cpuTopo.physicalCoreNum_ = getCores(cpuTopo.logicalCpus_, cpuTopo.isHybrid(), groupAcc); + if (cpuTopo.physicalCoreNum_ == 0) return false; + + DWORD len = 0; + GetLogicalProcessorInformationEx(RelationCache, NULL, &len); + std::vector buf(len); + if (!GetLogicalProcessorInformationEx(RelationCache, reinterpret_cast(buf.data()), &len)) return false; + + const char *p = buf.data(); + const char *end = p + len; + + while (p < end) { + const auto& entry = *reinterpret_cast(p); + if (entry.Relationship == RelationCache) { + const CACHE_RELATIONSHIP& cache = entry.Cache; + uint32_t type = CACHE_UNKNOWN; + if (cache.Level == 1) { + if (cache.Type == CacheInstruction) { + type = L1i; + } else if (cache.Type == CacheData) { + type = L1d; + } + } else if (cache.Level == 2) { + type = L2; + } else if (cache.Level == 3) { + type = L3; + } + if (type != CACHE_UNKNOWN) { + CpuMask mask; + if (!convertMask(mask, groupAcc, cache)) return false; + for (const auto& i : mask) { + if (i >= cpuTopo.logicalCpus_.size()) return false; + cpuTopo.logicalCpus_[i].cache[type].size = cache.CacheSize; + if (cpuTopo.lineSize_ == 0) cpuTopo.lineSize_ = cache.LineSize; + cpuTopo.logicalCpus_[i].cache[type].associativity = cache.Associativity; + cpuTopo.logicalCpus_[i].cache[type].sharedCpuIndices = mask; + } + } + } + p += entry.Size; + } + return true; +} + +#elif defined(__linux__) // Linux + +struct WrapFILE { + FILE *f; + explicit WrapFILE(const char *name) + : f(fopen(name, "r")) + { + } + ~WrapFILE() { if (f) fclose(f); } +}; + +inline uint32_t readIntFromFile(const char* path) { + WrapFILE wf(path); + if (!wf.f) return 0; + uint32_t val = 0; + int n = fscanf(wf.f, "%u", &val); + return (n == 1) ? val : 0; +} + +inline bool parseCpuList(CpuMask& mask, const char* path) { + WrapFILE wf(path); + if (!wf.f) return false; + char buf[1024]; + if (!fgets(buf, sizeof(buf), wf.f)) return false; + size_t n = strlen(buf); + if (n > 0 && buf[n - 1] == '\n') buf[n - 1] = '\0'; + return setStr(mask, buf); +} + +inline bool initCpuTopology(CpuTopology& cpuTopo) +{ + const uint32_t logicalCpuNum = sysconf(_SC_NPROCESSORS_ONLN); + + if (logicalCpuNum == 0) return false; + if (logicalCpuNum >= (1u << XBYAK_CPUMASK_BITN)) return false; + + cpuTopo.logicalCpus_.resize(logicalCpuNum); + uint32_t maxPhisicalIdx = 0; + + for (uint32_t cpuIdx = 0; cpuIdx < logicalCpuNum; cpuIdx++) { + char path[256]; + LogicalCpu& logCpu = cpuTopo.logicalCpus_[cpuIdx]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/topology/core_id", cpuIdx); + logCpu.coreId = readIntFromFile(path); + maxPhisicalIdx = (std::max)(maxPhisicalIdx, logCpu.coreId); + + logCpu.coreType = Standard; + + for (uint32_t cacheIdx = 0; cacheIdx < CACHE_TYPE_NUM; cacheIdx++) { + CacheType cacheType = CACHE_UNKNOWN; + + // Map cache index to cache type + { + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/cache/index%u/type", cpuIdx, cacheIdx); + char typeStr[32]; + WrapFILE wf(path); + + if (wf.f && fgets(typeStr, sizeof(typeStr), wf.f)) { + if (strncmp(typeStr, "Instruction", 11) == 0) { + cacheType = L1i; + } else if (strncmp(typeStr, "Data", 4) == 0) { + // Determine level + char path[256]; + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/cache/index%u/level", cpuIdx, cacheIdx); + switch (readIntFromFile(path)) { + case 1: cacheType = L1d; break; + case 2: cacheType = L2; break; + case 3: cacheType = L3; break; + default: break;; + } + } else if (strncmp(typeStr, "Unified", 7) == 0) { + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/cache/index%u/level", cpuIdx, cacheIdx); + switch (readIntFromFile(path)) { + case 2: cacheType = L2; break; + case 3: cacheType = L3; break; + default: break;; + } + } + } + } + if (cacheType == CACHE_UNKNOWN) continue; + CpuCache& cache = logCpu.cache[cacheType]; + + // Read cache size + { + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/cache/index%u/size", cpuIdx, cacheIdx); + char sizeStr[32]; + WrapFILE wf(path); + if (wf.f && fgets(sizeStr, sizeof(sizeStr), wf.f)) { + char *endp; + uint32_t size = (uint32_t)strtoul(sizeStr, &endp, 10); + switch (*endp) { + case '\0': case '\n': cache.size = size; break; + case 'K': case 'k': cache.size = size * 1024; break; + case 'M': case 'm': cache.size = size * 1024 * 1024; break; + default: break; + } + } + } + + // Read ways of associativity + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/cache/index%u/ways_of_associativity", cpuIdx, cacheIdx); + cache.associativity = readIntFromFile(path); + + // Read shared CPU list + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/cache/index%u/shared_cpu_list", cpuIdx, cacheIdx); + parseCpuList(cache.sharedCpuIndices, path); + + } + } + + // Assign core types for hybrid architectures + const bool isHybrid = cpuTopo.isHybrid(); + if (isHybrid) { + // For hybrid systems, read P-core and E-core lists from sysfs + CpuMask pCoreMask; + if (parseCpuList(pCoreMask, "/sys/devices/cpu_core/cpus")) { + // Set Performance core types + for (CpuMask::const_iterator it = pCoreMask.begin(); it != pCoreMask.end(); ++it) { + uint32_t cpuIdx = *it; + if (cpuIdx < logicalCpuNum) { + cpuTopo.logicalCpus_[cpuIdx].coreType = Performance; + } + } + } + CpuMask eCoreMask; + if (parseCpuList(eCoreMask, "/sys/devices/cpu_atom/cpus")) { + // Set Efficient core types + for (CpuMask::const_iterator it = eCoreMask.begin(); it != eCoreMask.end(); ++it) { + uint32_t cpuIdx = *it; + if (cpuIdx < logicalCpuNum) { + cpuTopo.logicalCpus_[cpuIdx].coreType = Efficient; + } + } + } + } + + // Read coherency line size + cpuTopo.lineSize_ = readIntFromFile("/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size"); + + cpuTopo.physicalCoreNum_ = maxPhisicalIdx + 1; + return true; +} +#else // Other OS (e.g., macOS) +inline bool initCpuTopology(CpuTopology& cpuTopo) +{ + // CPU topology detection not yet implemented + (void)cpuTopo; + return false; +} +#endif // _WIN32 / __linux__ / other OS + +} // namespace impl +#endif // XBYAK_CPU_CACHE + class Clock { public: static inline uint64_t getRdtsc() @@ -1170,4 +2014,19 @@ public: } } // end of util +#if XBYAK_CPUMASK_COMPACT == 1 && __cplusplus >= 201103 + +namespace std { + +template<> +struct hash { + size_t operator()(const Xbyak::util::CpuMask& m) const noexcept { + return std::hash{}(m.to_u64()); + } +}; + +} // std + +#endif + #endif