USB: Additional commands for MSAC-US1

DEV9: Isolate raw PCAP packet IO
Qt: Fix crash when interacting with per-game achievement settings
2026-07-11 09:44:17 +02:00 · 2026-05-13 12:17:55 -04:00 · 2026-05-13 12:17:27 -04:00 · 2026-05-13 12:17:02 -04:00 · 2026-05-13 17:03:02 +02:00 · 2026-05-12 12:14:40 -04:00
120 changed files with 26858 additions and 11204 deletions
@@ -24,7 +24,7 @@ QTAPNG=1.3.0
 FFMPEG=8.1
 LIBBACKTRACE=ad106d5fdd5d960bd33fae1c48a351af567fd075
 LIBJPEGTURBO=3.1.4.1
-LIBPNG=1.6.56
+LIBPNG=1.6.58
 LIBWEBP=1.6.0
 NVENC=13.0.19.0
 SDL=SDL3-3.4.8
@@ -34,12 +34,12 @@ ZSTD=1.5.7
 KDDOCKWIDGETS=2.4.0
 PLUTOVG=1.3.2
 PLUTOSVG=0.0.7
-RAPIDYAML=0.11.1
+RAPIDYAML=0.12.1

-SHADERC=2026.1
-SHADERC_GLSLANG=f0bd0257c308b9a26562c1a30c4748a0219cc951
-SHADERC_SPIRVHEADERS=04f10f650d514df88b76d25e83db360142c7b174
-SHADERC_SPIRVTOOLS=fbe4f3ad913c44fe8700545f8ffe35d1382b7093
+SHADERC=2026.2
+SHADERC_GLSLANG=275822a6261ee689aadb1da5f09a0ec2f058685c
+SHADERC_SPIRVHEADERS=58006c901d1d5c37dece6b6610e9af87fa951375
+SHADERC_SPIRVTOOLS=6337eb62cadd7d124ac6789bf39c0f71148f0a73

 mkdir -p deps-build
 cd deps-build
@@ -58,10 +58,10 @@ f1d3be3489f758efe1a8f12118a212febbe611aa670af32e0159fa3c1feab2a6  QtApng-$QTAPNG
 b072aed6871998cce9b36e7774033105ca29e33632be5b6347f3206898e0756a  ffmpeg-$FFMPEG.tar.xz
 96e5c2d7f2c482a60d5804da48a2eb9a0db0719b2c65dcc169fbfdcf37f3a45d  libbacktrace-$LIBBACKTRACE.tar.gz
 ecae8008e2cc9ade2f2c1bb9d5e6d4fb73e7c433866a056bd82980741571a022  libjpeg-turbo-$LIBJPEGTURBO.tar.gz
-f7d8bf1601b7804f583a254ab343a6549ca6cf27d255c302c47af2d9d36a6f18  libpng-$LIBPNG.tar.xz
+28eb403f51f0f7405249132cecfe82ea5c0ef97f1b32c5a65828814ae0d34775  libpng-$LIBPNG.tar.xz
 e4ab7009bf0629fd11982d4c2aa83964cf244cffba7347ecd39019a9e38c4564  libwebp-$LIBWEBP.tar.gz
 e9fff7467fb60f037e6708da18b25560649e4c63edc2a69bb871b960d9cbfbba  $SDL.tar.gz
-9ce32d4a2763a2ac5f258726ba2f49e9011327c1ee8c30862a32d0f30889fbe8  libpng-$LIBPNG-apng.patch.gz
+eee7dea22ed502868017971c86c63c4ed1e6085de0baebfdcc3d3322f00f3eb0  libpng-$LIBPNG-apng.patch.gz
 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b  lz4-$LZ4.tar.gz
 13da39edb3a40ed9713ae390ca89faa2f1202c9dda869ef306a8d4383e242bee  nv-codec-headers-$NVENC.tar.gz
 c465aa56757e7746ac707f582b6e2d51546569a4a2488c1172fb543aa5fdfc2c  vulkan-sdk-$VULKAN.tar.gz
@@ -69,12 +69,12 @@ eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3  zstd-$ZSTD.tar
 51dbf24fe72e43dd7cb9a289d3cab47112010f1a2ed69b6fc8ac0dff31991ed2  KDDockWidgets-$KDDOCKWIDGETS.tar.gz
 7bd4e79ce18b1d47517e7e91fbb7cf19d4f01942804a519bc7c0bf32b6325dd5  plutovg-$PLUTOVG.tar.gz
 78561b571ac224030cdc450ca2986b4de915c2ba7616004a6d71a379bffd15f3  plutosvg-$PLUTOSVG.tar.gz
-9d9938269adc25e9a9b84650338b87d130cf469d82685fffc028c325279619c1  rapidyaml-$RAPIDYAML-src.tgz
+e9efcdd17f86287748793cf21d106e461fcad8d103a3e5a23632afe93828660d  rapidyaml-$RAPIDYAML-src.tgz

-245002feccbe7f8361b223545a5654cea69780745886872d7efff50a38d96c66  shaderc-$SHADERC.tar.gz
-bd58dca4dac67dcf7640292d7d63e0416274d40ee2200f7301878cec11ac6647  shaderc-glslang-$SHADERC_GLSLANG.tar.gz
-1b220e3eec1714f0451b0e3652979bd280edf10893f617837b88e6359a804ded  shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz
-cabb35f4eef0da3ef72ad9edd596af4191d7507a8f35c05df526d2d5ff889f59  shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz
+f924178e75e3293082481b25ed64d5e48a795b479dac3bd3c83d23070855df42  shaderc-$SHADERC.tar.gz
+971848a1cc639ce8dc244e778b17efe0f690e32ac398a75e31d1c67ad06d3e0a  shaderc-glslang-$SHADERC_GLSLANG.tar.gz
+a6cb1b300bb8171795e116457e858e555334749f9cacaed8068ae0ef8681110c  shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz
+e156be0bd81c8812f1bff8e520422bfa9df61b3045587b9eb483185f1074a7b2  shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz
 EOF

 if ! shasum -sa 256 --check SHASUMS 2> /dev/null; then
@@ -24,7 +24,7 @@ LZ4=1.10.0
 ZSTD=1.5.7
 PLUTOVG=1.3.2
 PLUTOSVG=0.0.7
-RAPIDYAML=0.11.1
+RAPIDYAML=0.12.1

 SHADERC=2025.4
 SHADERC_GLSLANG=7a47e2531cb334982b2a2dd8513dca0a3de4373d
@@ -50,7 +50,7 @@ c693867f10a7760ef1bcf85419d51783586768cc2c601d03841bc6a8b2554b9c  shaderc-spirv-
 06b0a042f2e121e954badb4fd78c9e2d4bc7ed6087eceb26ab559c23cf94334f  shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz
 7bd4e79ce18b1d47517e7e91fbb7cf19d4f01942804a519bc7c0bf32b6325dd5  plutovg-$PLUTOVG.tar.gz
 78561b571ac224030cdc450ca2986b4de915c2ba7616004a6d71a379bffd15f3  plutosvg-$PLUTOSVG.tar.gz
-9d9938269adc25e9a9b84650338b87d130cf469d82685fffc028c325279619c1  rapidyaml-$RAPIDYAML-src.tgz
+e9efcdd17f86287748793cf21d106e461fcad8d103a3e5a23632afe93828660d  rapidyaml-$RAPIDYAML-src.tgz
 EOF

 if ! shasum -sa 256 --check SHASUMS 2> /dev/null; then
@@ -15,24 +15,24 @@
    {
      "type": "git",
      "url": "https://github.com/google/shaderc.git",
-      "commit": "301b4ede53d59b68bf55f95bb26412d9233c8187"
+      "commit": "d5f08ae5c5a9a45165578445cbd0f9adf0223448"
    },
    {
      "type": "archive",
-      "url": "https://github.com/KhronosGroup/glslang/archive/f0bd0257c308b9a26562c1a30c4748a0219cc951.tar.gz",
-      "sha256": "bd58dca4dac67dcf7640292d7d63e0416274d40ee2200f7301878cec11ac6647",
+      "url": "https://github.com/KhronosGroup/glslang/archive/275822a6261ee689aadb1da5f09a0ec2f058685c.tar.gz",
+      "sha256": "971848a1cc639ce8dc244e778b17efe0f690e32ac398a75e31d1c67ad06d3e0a",
      "dest": "third_party/glslang"
    },
    {
      "type": "archive",
-      "url": "https://github.com/KhronosGroup/SPIRV-Headers/archive/04f10f650d514df88b76d25e83db360142c7b174.tar.gz",
-      "sha256": "1b220e3eec1714f0451b0e3652979bd280edf10893f617837b88e6359a804ded",
+      "url": "https://github.com/KhronosGroup/SPIRV-Headers/archive/58006c901d1d5c37dece6b6610e9af87fa951375.tar.gz",
+      "sha256": "a6cb1b300bb8171795e116457e858e555334749f9cacaed8068ae0ef8681110c",
      "dest": "third_party/spirv-headers"
    },
    {
      "type": "archive",
-      "url": "https://github.com/KhronosGroup/SPIRV-Tools/archive/fbe4f3ad913c44fe8700545f8ffe35d1382b7093.tar.gz",
-      "sha256": "cabb35f4eef0da3ef72ad9edd596af4191d7507a8f35c05df526d2d5ff889f59",
+      "url": "https://github.com/KhronosGroup/SPIRV-Tools/archive/6337eb62cadd7d124ac6789bf39c0f71148f0a73.tar.gz",
+      "sha256": "e156be0bd81c8812f1bff8e520422bfa9df61b3045587b9eb483185f1074a7b2",
      "dest": "third_party/spirv-tools"
    },
    {
@@ -14,21 +14,21 @@
  "sources": [
    {
      "type": "archive",
-      "url": "https://downloads.sourceforge.net/project/libpng/libpng16/1.6.56/libpng-1.6.56.tar.xz",
-      "sha256": "f7d8bf1601b7804f583a254ab343a6549ca6cf27d255c302c47af2d9d36a6f18"
+      "url": "https://downloads.sourceforge.net/project/libpng/libpng16/1.6.58/libpng-1.6.58.tar.xz",
+      "sha256": "28eb403f51f0f7405249132cecfe82ea5c0ef97f1b32c5a65828814ae0d34775"
    },
    {
      "type": "file",
-      "url": "https://download.sourceforge.net/libpng-apng/libpng-1.6.56-apng.patch.gz",
-      "dest-filename": "libpng-1.6.56-apng.patch.gz",
-      "sha256": "9ce32d4a2763a2ac5f258726ba2f49e9011327c1ee8c30862a32d0f30889fbe8"
+      "url": "https://download.sourceforge.net/libpng-apng/libpng-1.6.58-apng.patch.gz",
+      "dest-filename": "libpng-1.6.58-apng.patch.gz",
+      "sha256": "eee7dea22ed502868017971c86c63c4ed1e6085de0baebfdcc3d3322f00f3eb0"
    },
    {
      "type": "shell",
      "commands":
      [
-          "gunzip -f libpng-1.6.56-apng.patch.gz",
-          "patch -p1  < \"libpng-1.6.56-apng.patch\""
+          "gunzip -f libpng-1.6.58-apng.patch.gz",
+          "patch -p1  < \"libpng-1.6.58-apng.patch\""
      ]
    }
  ],
@@ -12,8 +12,8 @@
    {
      "type": "git",
      "url": "https://github.com/biojppm/rapidyaml.git",
-      "tag": "v0.11.1",
-      "commit": "119b6042064d3828819e428e32e5a0f3035d5643"
+      "tag": "v0.12.1",
+      "commit": "b56567b0bd24e9ce7beb08d6950a5732f62f6e74"
    }
  ],
  "cleanup": [
@@ -48,10 +48,10 @@ QTAPNG=1.3.0

 FREETYPE=2.14.3
 SDL=SDL3-3.4.8
-HARFBUZZ=14.0.0
+HARFBUZZ=14.2.0
 ZSTD=1.5.7
 LZ4=1.10.0
-LIBPNG=1.6.56
+LIBPNG=1.6.58
 LIBJPEGTURBO=3.1.4.1
 LIBWEBP=1.6.0
 FFMPEG=8.1
@@ -59,12 +59,12 @@ MOLTENVK=1.4.1
 KDDOCKWIDGETS=2.4.0
 PLUTOVG=1.3.2
 PLUTOSVG=0.0.7
-RAPIDYAML=0.11.1
+RAPIDYAML=0.12.1

-SHADERC=2026.1
-SHADERC_GLSLANG=f0bd0257c308b9a26562c1a30c4748a0219cc951
-SHADERC_SPIRVHEADERS=04f10f650d514df88b76d25e83db360142c7b174
-SHADERC_SPIRVTOOLS=fbe4f3ad913c44fe8700545f8ffe35d1382b7093
+SHADERC=2026.2
+SHADERC_GLSLANG=275822a6261ee689aadb1da5f09a0ec2f058685c
+SHADERC_SPIRVHEADERS=58006c901d1d5c37dece6b6610e9af87fa951375
+SHADERC_SPIRVTOOLS=6337eb62cadd7d124ac6789bf39c0f71148f0a73

 mkdir -p deps-build
 cd deps-build
@@ -94,24 +94,24 @@ f1d3be3489f758efe1a8f12118a212febbe611aa670af32e0159fa3c1feab2a6  QtApng-$QTAPNG

 36bc4f1cc413335368ee656c42afca65c5a3987e8768cc28cf11ba775e785a5f  freetype-$FREETYPE.tar.xz
 e9fff7467fb60f037e6708da18b25560649e4c63edc2a69bb871b960d9cbfbba  $SDL.tar.gz
-f29db9470e0ca5cef484e04e27baeec233aa428e8fdabe9e51b0f706c0809d24  harfbuzz-$HARFBUZZ.tar.gz
+c652d5d94971031654ab3989891a490a895d3e3f2b71171c62692b28e94b1b93  harfbuzz-$HARFBUZZ.tar.gz
 eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3  zstd-$ZSTD.tar.gz
 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b  lz4-$LZ4.tar.gz
-f7d8bf1601b7804f583a254ab343a6549ca6cf27d255c302c47af2d9d36a6f18  libpng-$LIBPNG.tar.xz
+28eb403f51f0f7405249132cecfe82ea5c0ef97f1b32c5a65828814ae0d34775  libpng-$LIBPNG.tar.xz
 e4ab7009bf0629fd11982d4c2aa83964cf244cffba7347ecd39019a9e38c4564  libwebp-$LIBWEBP.tar.gz
-9ce32d4a2763a2ac5f258726ba2f49e9011327c1ee8c30862a32d0f30889fbe8  libpng-$LIBPNG-apng.patch.gz
+eee7dea22ed502868017971c86c63c4ed1e6085de0baebfdcc3d3322f00f3eb0  libpng-$LIBPNG-apng.patch.gz
 ecae8008e2cc9ade2f2c1bb9d5e6d4fb73e7c433866a056bd82980741571a022  libjpeg-turbo-$LIBJPEGTURBO.tar.gz
 b072aed6871998cce9b36e7774033105ca29e33632be5b6347f3206898e0756a  ffmpeg-$FFMPEG.tar.xz
 9985f141902a17de818e264d17c1ce334b748e499ee02fcb4703e4dc0038f89c  MoltenVK-$MOLTENVK.tar.gz
 51dbf24fe72e43dd7cb9a289d3cab47112010f1a2ed69b6fc8ac0dff31991ed2  KDDockWidgets-$KDDOCKWIDGETS.tar.gz
 7bd4e79ce18b1d47517e7e91fbb7cf19d4f01942804a519bc7c0bf32b6325dd5  plutovg-$PLUTOVG.tar.gz
 78561b571ac224030cdc450ca2986b4de915c2ba7616004a6d71a379bffd15f3  plutosvg-$PLUTOSVG.tar.gz
-9d9938269adc25e9a9b84650338b87d130cf469d82685fffc028c325279619c1  rapidyaml-$RAPIDYAML-src.tgz
+e9efcdd17f86287748793cf21d106e461fcad8d103a3e5a23632afe93828660d  rapidyaml-$RAPIDYAML-src.tgz

-245002feccbe7f8361b223545a5654cea69780745886872d7efff50a38d96c66  shaderc-$SHADERC.tar.gz
-bd58dca4dac67dcf7640292d7d63e0416274d40ee2200f7301878cec11ac6647  shaderc-glslang-$SHADERC_GLSLANG.tar.gz
-1b220e3eec1714f0451b0e3652979bd280edf10893f617837b88e6359a804ded  shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz
-cabb35f4eef0da3ef72ad9edd596af4191d7507a8f35c05df526d2d5ff889f59  shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz
+f924178e75e3293082481b25ed64d5e48a795b479dac3bd3c83d23070855df42  shaderc-$SHADERC.tar.gz
+971848a1cc639ce8dc244e778b17efe0f690e32ac398a75e31d1c67ad06d3e0a  shaderc-glslang-$SHADERC_GLSLANG.tar.gz
+a6cb1b300bb8171795e116457e858e555334749f9cacaed8068ae0ef8681110c  shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz
+e156be0bd81c8812f1bff8e520422bfa9df61b3045587b9eb483185f1074a7b2  shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz
 EOF

 if ! shasum -sa 256 --check SHASUMS 2> /dev/null; then
@@ -25,10 +25,10 @@ QTAPNG=1.3.0

 FREETYPE=2.14.3
 SDL=SDL3-3.4.8
-HARFBUZZ=14.0.0
+HARFBUZZ=14.2.0
 ZSTD=1.5.7
 LZ4=1.10.0
-LIBPNG=1.6.56
+LIBPNG=1.6.58
 LIBJPEGTURBO=3.1.4.1
 LIBWEBP=1.6.0
 FFMPEG=8.1
@@ -36,12 +36,12 @@ MOLTENVK=1.4.1
 KDDOCKWIDGETS=2.4.0
 PLUTOVG=1.3.2
 PLUTOSVG=0.0.7
-RAPIDYAML=0.11.1
+RAPIDYAML=0.12.1

-SHADERC=2026.1
-SHADERC_GLSLANG=f0bd0257c308b9a26562c1a30c4748a0219cc951
-SHADERC_SPIRVHEADERS=04f10f650d514df88b76d25e83db360142c7b174
-SHADERC_SPIRVTOOLS=fbe4f3ad913c44fe8700545f8ffe35d1382b7093
+SHADERC=2026.2
+SHADERC_GLSLANG=275822a6261ee689aadb1da5f09a0ec2f058685c
+SHADERC_SPIRVHEADERS=58006c901d1d5c37dece6b6610e9af87fa951375
+SHADERC_SPIRVTOOLS=6337eb62cadd7d124ac6789bf39c0f71148f0a73

 mkdir -p deps-build
 cd deps-build
@@ -70,24 +70,24 @@ f1d3be3489f758efe1a8f12118a212febbe611aa670af32e0159fa3c1feab2a6  QtApng-$QTAPNG

 36bc4f1cc413335368ee656c42afca65c5a3987e8768cc28cf11ba775e785a5f  freetype-$FREETYPE.tar.xz
 e9fff7467fb60f037e6708da18b25560649e4c63edc2a69bb871b960d9cbfbba  $SDL.tar.gz
-f29db9470e0ca5cef484e04e27baeec233aa428e8fdabe9e51b0f706c0809d24  harfbuzz-$HARFBUZZ.tar.gz
+c652d5d94971031654ab3989891a490a895d3e3f2b71171c62692b28e94b1b93  harfbuzz-$HARFBUZZ.tar.gz
 eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3  zstd-$ZSTD.tar.gz
 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b  lz4-$LZ4.tar.gz
-f7d8bf1601b7804f583a254ab343a6549ca6cf27d255c302c47af2d9d36a6f18  libpng-$LIBPNG.tar.xz
+28eb403f51f0f7405249132cecfe82ea5c0ef97f1b32c5a65828814ae0d34775  libpng-$LIBPNG.tar.xz
 e4ab7009bf0629fd11982d4c2aa83964cf244cffba7347ecd39019a9e38c4564  libwebp-$LIBWEBP.tar.gz
-9ce32d4a2763a2ac5f258726ba2f49e9011327c1ee8c30862a32d0f30889fbe8  libpng-$LIBPNG-apng.patch.gz
+eee7dea22ed502868017971c86c63c4ed1e6085de0baebfdcc3d3322f00f3eb0  libpng-$LIBPNG-apng.patch.gz
 ecae8008e2cc9ade2f2c1bb9d5e6d4fb73e7c433866a056bd82980741571a022  libjpeg-turbo-$LIBJPEGTURBO.tar.gz
 b072aed6871998cce9b36e7774033105ca29e33632be5b6347f3206898e0756a  ffmpeg-$FFMPEG.tar.xz
 9985f141902a17de818e264d17c1ce334b748e499ee02fcb4703e4dc0038f89c  v$MOLTENVK.tar.gz
 51dbf24fe72e43dd7cb9a289d3cab47112010f1a2ed69b6fc8ac0dff31991ed2  KDDockWidgets-$KDDOCKWIDGETS.tar.gz
 7bd4e79ce18b1d47517e7e91fbb7cf19d4f01942804a519bc7c0bf32b6325dd5  plutovg-$PLUTOVG.tar.gz
 78561b571ac224030cdc450ca2986b4de915c2ba7616004a6d71a379bffd15f3  plutosvg-$PLUTOSVG.tar.gz
-9d9938269adc25e9a9b84650338b87d130cf469d82685fffc028c325279619c1  rapidyaml-$RAPIDYAML-src.tgz
+e9efcdd17f86287748793cf21d106e461fcad8d103a3e5a23632afe93828660d  rapidyaml-$RAPIDYAML-src.tgz

-245002feccbe7f8361b223545a5654cea69780745886872d7efff50a38d96c66  shaderc-$SHADERC.tar.gz
-bd58dca4dac67dcf7640292d7d63e0416274d40ee2200f7301878cec11ac6647  shaderc-glslang-$SHADERC_GLSLANG.tar.gz
-1b220e3eec1714f0451b0e3652979bd280edf10893f617837b88e6359a804ded  shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz
-cabb35f4eef0da3ef72ad9edd596af4191d7507a8f35c05df526d2d5ff889f59  shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz
+f924178e75e3293082481b25ed64d5e48a795b479dac3bd3c83d23070855df42  shaderc-$SHADERC.tar.gz
+971848a1cc639ce8dc244e778b17efe0f690e32ac398a75e31d1c67ad06d3e0a  shaderc-glslang-$SHADERC_GLSLANG.tar.gz
+a6cb1b300bb8171795e116457e858e555334749f9cacaed8068ae0ef8681110c  shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz
+e156be0bd81c8812f1bff8e520422bfa9df61b3045587b9eb483185f1074a7b2  shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz
 EOF

 if ! shasum -sa 256 --check SHASUMS 2> /dev/null; then
@@ -59,11 +59,11 @@ set QTMINOR=6.11
 set QTAPNG=1.3.0

 set FREETYPE=2.14.3
-set HARFBUZZ=14.0.0
+set HARFBUZZ=14.2.0
 set SDL=SDL3-3.4.8
 set LIBJPEGTURBO=3.1.4.1
-set LIBPNG=1656
-set LIBPNGLONG=1.6.56
+set LIBPNG=1658
+set LIBPNGLONG=1.6.58
 set LZ4=1.10.0
 set WEBP=1.6.0
 set ZLIB=1.3.2
@@ -72,14 +72,14 @@ set ZSTD=1.5.7
 set KDDOCKWIDGETS=2.4.0
 set PLUTOVG=1.3.2
 set PLUTOSVG=0.0.7
-set RAPIDYAML=0.11.1
+set RAPIDYAML=0.12.1

-set SHADERC=2026.1
-set SHADERC_GLSLANG=f0bd0257c308b9a26562c1a30c4748a0219cc951
-set SHADERC_SPIRVHEADERS=04f10f650d514df88b76d25e83db360142c7b174
-set SHADERC_SPIRVTOOLS=fbe4f3ad913c44fe8700545f8ffe35d1382b7093
+set SHADERC=2026.2
+set SHADERC_GLSLANG=275822a6261ee689aadb1da5f09a0ec2f058685c
+set SHADERC_SPIRVHEADERS=58006c901d1d5c37dece6b6610e9af87fa951375
+set SHADERC_SPIRVTOOLS=6337eb62cadd7d124ac6789bf39c0f71148f0a73

-set AGILITYSDK=1.619.1
+set AGILITYSDK=1.619.2

 call :downloadfile "qtbase-everywhere-src-%QT%.zip" "https://download.qt.io/official_releases/qt/%QTMINOR%/%QT%/submodules/qtbase-everywhere-src-%QT%.zip" 590d5ae246c85fa14d6458a36ff75a11236acfe8987c2475090aab1770acbdf8 || goto error
 call :downloadfile "qtimageformats-everywhere-src-%QT%.zip" "https://download.qt.io/official_releases/qt/%QTMINOR%/%QT%/submodules/qtimageformats-everywhere-src-%QT%.zip" 5dfb3c0cb84d2c935c1716b3b86358ca496fb9216676e7e28fee1357fb4a050e || goto error
@@ -89,9 +89,9 @@ call :downloadfile "qttranslations-everywhere-src-%QT%.zip" "https://download.qt
 call :downloadfile "QtApng-%QTAPNG%.zip" "https://github.com/jurplel/QtApng/archive/refs/tags/%QTAPNG%.zip" 5176082cdd468047a7eb1ec1f106b032f57df207aa318d559b29606b00d159ac || goto error

 call :downloadfile "freetype-%FREETYPE%.tar.gz" https://sourceforge.net/projects/freetype/files/freetype2/%FREETYPE%/freetype-%FREETYPE%.tar.gz/download e61b31ab26358b946e767ed7eb7f4bb2e507da1cfefeb7a8861ace7fd5c899a1 || goto error
-call :downloadfile "harfbuzz-%HARFBUZZ%.zip" https://github.com/harfbuzz/harfbuzz/archive/refs/tags/%HARFBUZZ%.zip f4b86f4d107bc4f0b005c97d6cde43e548a125ee2810fa00d41844b46b6fe16a || goto error
-call :downloadfile "lpng%LIBPNG%.zip" https://download.sourceforge.net/libpng/lpng1656.zip 1ccf023c5f4ee1a7b75c5624f53acede5066f79b2fb14ddffaa28369adf9baac || goto error
-call :downloadfile "lpng%LIBPNG%-apng.patch.gz" https://download.sourceforge.net/libpng-apng/libpng-%LIBPNGLONG%-apng.patch.gz 9ce32d4a2763a2ac5f258726ba2f49e9011327c1ee8c30862a32d0f30889fbe8 || goto error
+call :downloadfile "harfbuzz-%HARFBUZZ%.zip" https://github.com/harfbuzz/harfbuzz/archive/refs/tags/%HARFBUZZ%.zip bb2f83255706b1c92d731541c7cefaf98bb5b93e8f76d16f6deda05225ff20ee || goto error
+call :downloadfile "lpng%LIBPNG%.zip" https://download.sourceforge.net/libpng/lpng1658.zip b32f170855dbbe3e6d9e645af40b538137041773672c3ba3e02db5816c82d376 || goto error
+call :downloadfile "lpng%LIBPNG%-apng.patch.gz" https://download.sourceforge.net/libpng-apng/libpng-%LIBPNGLONG%-apng.patch.gz eee7dea22ed502868017971c86c63c4ed1e6085de0baebfdcc3d3322f00f3eb0 || goto error
 call :downloadfile "libjpeg-turbo-%LIBJPEGTURBO%.tar.gz" "https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/%LIBJPEGTURBO%/libjpeg-turbo-%LIBJPEGTURBO%.tar.gz" ecae8008e2cc9ade2f2c1bb9d5e6d4fb73e7c433866a056bd82980741571a022 || goto error
 call :downloadfile "libwebp-%WEBP%.tar.gz" "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-%WEBP%.tar.gz" e4ab7009bf0629fd11982d4c2aa83964cf244cffba7347ecd39019a9e38c4564 || goto error
 call :downloadfile "%SDL%.zip" "https://libsdl.org/release/%SDL%.zip" 506206e02f90c1f37e048eacaf9e8a3a7dc682fd27783eb0ff15a7d2dcc9c2af || goto error
@@ -101,13 +101,13 @@ call :downloadfile "zstd-%ZSTD%.zip" "https://github.com/facebook/zstd/archive/r
 call :downloadfile "KDDockWidgets-%KDDOCKWIDGETS%.zip" "https://github.com/KDAB/KDDockWidgets/archive/v%KDDOCKWIDGETS%.zip" 47ddb48197872055f0adf8e90a7235f8a3b795ca1ee3a28ac2c504c673ae3806 || goto error
 call :downloadfile "plutovg-%PLUTOVG%.zip" "https://github.com/sammycage/plutovg/archive/v%PLUTOVG%.zip" 4fe4e48f28aa80171b2166d45c0976ab0f21eecedb52cd4c3ef73b5afb48fac9 || goto error
 call :downloadfile "plutosvg-%PLUTOSVG%.zip" "https://github.com/sammycage/plutosvg/archive/v%PLUTOSVG%.zip" 82dee2c57ad712bdd6d6d81d3e76249d89caa4b5a4214353660fd5adff12201a || goto error
-call :downloadfile "agility-sdk-%AGILITYSDK%.nupkg" "https://www.nuget.org/api/v2/package/Microsoft.Direct3D.D3D12/%AGILITYSDK%" 9073a264301e93183844026239c9081717d52d13fb71aae68a0555c576b8de44 || goto error
-call :downloadfile "rapidyaml-%RAPIDYAML%-src.zip" "https://github.com/biojppm/rapidyaml/releases/download/v%RAPIDYAML%/rapidyaml-%RAPIDYAML%-src.zip" 30054b74abdf0ba35bf2cb435b6e49fcb6d62a8e78a240a018c36aa60dba765f || goto error
+call :downloadfile "agility-sdk-%AGILITYSDK%.nupkg" "https://www.nuget.org/api/v2/package/Microsoft.Direct3D.D3D12/%AGILITYSDK%" eb92d90bb23b2ec23410c41d791e41dbdbec942ab946924d1fdcb31eac6f0735 || goto error
+call :downloadfile "rapidyaml-%RAPIDYAML%-src.zip" "https://github.com/biojppm/rapidyaml/releases/download/v%RAPIDYAML%/rapidyaml-%RAPIDYAML%-src.zip" 96276f55b9fa7837ac8f3f72fd52965879cbb5d5d2e6af548c69a177fb078304 || goto error

-call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/google/shaderc/archive/refs/tags/v%SHADERC%.zip" 3ac59c8216d367ab7858684d39c8faf872a64150aeb139335f4e083c5f79dde0 || goto error
-call :downloadfile "shaderc-glslang-%SHADERC_GLSLANG%.zip" "https://github.com/KhronosGroup/glslang/archive/%SHADERC_GLSLANG%.zip" 42a30acca4a35955370ed8ff6e54b823b4d4a5a86571baec1203d3fce87da447 || goto error
-call :downloadfile "shaderc-spirv-headers-%SHADERC_SPIRVHEADERS%.zip" "https://github.com/KhronosGroup/SPIRV-Headers/archive/%SHADERC_SPIRVHEADERS%.zip" 00ecd73dcaaa956cf2221ce899ce096e9535ba20695483c5277adede462d1bde || goto error
-call :downloadfile "shaderc-spirv-tools-%SHADERC_SPIRVTOOLS%.zip" "https://github.com/KhronosGroup/SPIRV-Tools/archive/%SHADERC_SPIRVTOOLS%.zip" 65b23ace0ff0c64daf51f7741ebb6448899fa4aceefc72403e56f5b95607ca8e || goto error
+call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/google/shaderc/archive/refs/tags/v%SHADERC%.zip" f9401cc5cb36c276cd1e072b6595dbd728148e8dba389e50f7339e2d388dbc08 || goto error
+call :downloadfile "shaderc-glslang-%SHADERC_GLSLANG%.zip" "https://github.com/KhronosGroup/glslang/archive/%SHADERC_GLSLANG%.zip" 2b63189efad0348d88d410a5e12ec550a612e0b6ceef64624b8f45491269fb9c || goto error
+call :downloadfile "shaderc-spirv-headers-%SHADERC_SPIRVHEADERS%.zip" "https://github.com/KhronosGroup/SPIRV-Headers/archive/%SHADERC_SPIRVHEADERS%.zip" d2f071e94c081f5a4606559770ebf1f7d1eac92a1def0c3e10609844aa8b69b2 || goto error
+call :downloadfile "shaderc-spirv-tools-%SHADERC_SPIRVTOOLS%.zip" "https://github.com/KhronosGroup/SPIRV-Tools/archive/%SHADERC_SPIRVTOOLS%.zip" 4011be89aa73e3461c9deef73936a62c79a3097590c5135d058041cc9fb99c6f || goto error

 if %DEBUG%==1 (
  echo Building debug and release libraries...
@@ -76,11 +76,11 @@ set LIBSVTAV1=4.0.1
 set LIBX264=b35605ace3ddf7c1a5d67a2eb553f034aef41d55

 set FREETYPE=2.14.3
-set HARFBUZZ=14.0.0
+set HARFBUZZ=14.2.0
 set SDL=SDL3-3.4.8
 set LIBJPEGTURBO=3.1.4.1
-set LIBPNG=1656
-set LIBPNGLONG=1.6.56
+set LIBPNG=1658
+set LIBPNGLONG=1.6.58
 set LZ4=1.10.0
 set WEBP=1.6.0
 set ZLIB=1.3.2
@@ -89,14 +89,14 @@ set ZSTD=1.5.7
 set KDDOCKWIDGETS=2.4.0
 set PLUTOVG=1.3.2
 set PLUTOSVG=0.0.7
-set RAPIDYAML=0.11.1
+set RAPIDYAML=0.12.1

-set SHADERC=2026.1
-set SHADERC_GLSLANG=f0bd0257c308b9a26562c1a30c4748a0219cc951
-set SHADERC_SPIRVHEADERS=04f10f650d514df88b76d25e83db360142c7b174
-set SHADERC_SPIRVTOOLS=fbe4f3ad913c44fe8700545f8ffe35d1382b7093
+set SHADERC=2026.2
+set SHADERC_GLSLANG=275822a6261ee689aadb1da5f09a0ec2f058685c
+set SHADERC_SPIRVHEADERS=58006c901d1d5c37dece6b6610e9af87fa951375
+set SHADERC_SPIRVTOOLS=6337eb62cadd7d124ac6789bf39c0f71148f0a73

-set AGILITYSDK=1.619.1
+set AGILITYSDK=1.619.2

 call :downloadfile "qtbase-everywhere-src-%QT%.zip" "https://download.qt.io/official_releases/qt/%QTMINOR%/%QT%/submodules/qtbase-everywhere-src-%QT%.zip" 590d5ae246c85fa14d6458a36ff75a11236acfe8987c2475090aab1770acbdf8 || goto error
 call :downloadfile "qtimageformats-everywhere-src-%QT%.zip" "https://download.qt.io/official_releases/qt/%QTMINOR%/%QT%/submodules/qtimageformats-everywhere-src-%QT%.zip" 5dfb3c0cb84d2c935c1716b3b86358ca496fb9216676e7e28fee1357fb4a050e || goto error
@@ -116,9 +116,9 @@ call :downloadfile "SVT-AV1-v%LIBSVTAV1%.zip" "https://gitlab.com/AOMediaCodec/S
 call :downloadfile "x264-%LIBX264%.zip" "https://code.videolan.org/videolan/x264/-/archive/%LIBX264%.zip" d95d059eff81cc565165cd058b66e208f0cc9874106a8fe94a811a66cf8a85a2 || goto error

 call :downloadfile "freetype-%FREETYPE%.tar.gz" https://sourceforge.net/projects/freetype/files/freetype2/%FREETYPE%/freetype-%FREETYPE%.tar.gz/download e61b31ab26358b946e767ed7eb7f4bb2e507da1cfefeb7a8861ace7fd5c899a1 || goto error
-call :downloadfile "harfbuzz-%HARFBUZZ%.zip" https://github.com/harfbuzz/harfbuzz/archive/refs/tags/%HARFBUZZ%.zip f4b86f4d107bc4f0b005c97d6cde43e548a125ee2810fa00d41844b46b6fe16a || goto error
-call :downloadfile "lpng%LIBPNG%.zip" https://download.sourceforge.net/libpng/lpng1656.zip 1ccf023c5f4ee1a7b75c5624f53acede5066f79b2fb14ddffaa28369adf9baac || goto error
-call :downloadfile "lpng%LIBPNG%-apng.patch.gz" https://download.sourceforge.net/libpng-apng/libpng-%LIBPNGLONG%-apng.patch.gz 9ce32d4a2763a2ac5f258726ba2f49e9011327c1ee8c30862a32d0f30889fbe8 || goto error
+call :downloadfile "harfbuzz-%HARFBUZZ%.zip" https://github.com/harfbuzz/harfbuzz/archive/refs/tags/%HARFBUZZ%.zip bb2f83255706b1c92d731541c7cefaf98bb5b93e8f76d16f6deda05225ff20ee || goto error
+call :downloadfile "lpng%LIBPNG%.zip" https://download.sourceforge.net/libpng/lpng1658.zip b32f170855dbbe3e6d9e645af40b538137041773672c3ba3e02db5816c82d376 || goto error
+call :downloadfile "lpng%LIBPNG%-apng.patch.gz" https://download.sourceforge.net/libpng-apng/libpng-%LIBPNGLONG%-apng.patch.gz eee7dea22ed502868017971c86c63c4ed1e6085de0baebfdcc3d3322f00f3eb0 || goto error
 call :downloadfile "libjpeg-turbo-%LIBJPEGTURBO%.tar.gz" "https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/%LIBJPEGTURBO%/libjpeg-turbo-%LIBJPEGTURBO%.tar.gz" ecae8008e2cc9ade2f2c1bb9d5e6d4fb73e7c433866a056bd82980741571a022 || goto error
 call :downloadfile "libwebp-%WEBP%.tar.gz" "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-%WEBP%.tar.gz" e4ab7009bf0629fd11982d4c2aa83964cf244cffba7347ecd39019a9e38c4564 || goto error
 call :downloadfile "%SDL%.zip" "https://libsdl.org/release/%SDL%.zip" 506206e02f90c1f37e048eacaf9e8a3a7dc682fd27783eb0ff15a7d2dcc9c2af || goto error
@@ -128,13 +128,13 @@ call :downloadfile "zstd-%ZSTD%.zip" "https://github.com/facebook/zstd/archive/r
 call :downloadfile "KDDockWidgets-%KDDOCKWIDGETS%.zip" "https://github.com/KDAB/KDDockWidgets/archive/v%KDDOCKWIDGETS%.zip" 47ddb48197872055f0adf8e90a7235f8a3b795ca1ee3a28ac2c504c673ae3806 || goto error
 call :downloadfile "plutovg-%PLUTOVG%.zip" "https://github.com/sammycage/plutovg/archive/v%PLUTOVG%.zip" 4fe4e48f28aa80171b2166d45c0976ab0f21eecedb52cd4c3ef73b5afb48fac9 || goto error
 call :downloadfile "plutosvg-%PLUTOSVG%.zip" "https://github.com/sammycage/plutosvg/archive/v%PLUTOSVG%.zip" 82dee2c57ad712bdd6d6d81d3e76249d89caa4b5a4214353660fd5adff12201a || goto error
-call :downloadfile "agility-sdk-%AGILITYSDK%.nupkg" "https://www.nuget.org/api/v2/package/Microsoft.Direct3D.D3D12/%AGILITYSDK%" 9073a264301e93183844026239c9081717d52d13fb71aae68a0555c576b8de44 || goto error
-call :downloadfile "rapidyaml-%RAPIDYAML%-src.zip" "https://github.com/biojppm/rapidyaml/releases/download/v%RAPIDYAML%/rapidyaml-%RAPIDYAML%-src.zip" 30054b74abdf0ba35bf2cb435b6e49fcb6d62a8e78a240a018c36aa60dba765f || goto error
+call :downloadfile "agility-sdk-%AGILITYSDK%.nupkg" "https://www.nuget.org/api/v2/package/Microsoft.Direct3D.D3D12/%AGILITYSDK%" eb92d90bb23b2ec23410c41d791e41dbdbec942ab946924d1fdcb31eac6f0735 || goto error
+call :downloadfile "rapidyaml-%RAPIDYAML%-src.zip" "https://github.com/biojppm/rapidyaml/releases/download/v%RAPIDYAML%/rapidyaml-%RAPIDYAML%-src.zip" 96276f55b9fa7837ac8f3f72fd52965879cbb5d5d2e6af548c69a177fb078304 || goto error

-call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/google/shaderc/archive/refs/tags/v%SHADERC%.zip" 3ac59c8216d367ab7858684d39c8faf872a64150aeb139335f4e083c5f79dde0 || goto error
-call :downloadfile "shaderc-glslang-%SHADERC_GLSLANG%.zip" "https://github.com/KhronosGroup/glslang/archive/%SHADERC_GLSLANG%.zip" 42a30acca4a35955370ed8ff6e54b823b4d4a5a86571baec1203d3fce87da447 || goto error
-call :downloadfile "shaderc-spirv-headers-%SHADERC_SPIRVHEADERS%.zip" "https://github.com/KhronosGroup/SPIRV-Headers/archive/%SHADERC_SPIRVHEADERS%.zip" 00ecd73dcaaa956cf2221ce899ce096e9535ba20695483c5277adede462d1bde || goto error
-call :downloadfile "shaderc-spirv-tools-%SHADERC_SPIRVTOOLS%.zip" "https://github.com/KhronosGroup/SPIRV-Tools/archive/%SHADERC_SPIRVTOOLS%.zip" 65b23ace0ff0c64daf51f7741ebb6448899fa4aceefc72403e56f5b95607ca8e || goto error
+call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/google/shaderc/archive/refs/tags/v%SHADERC%.zip" f9401cc5cb36c276cd1e072b6595dbd728148e8dba389e50f7339e2d388dbc08 || goto error
+call :downloadfile "shaderc-glslang-%SHADERC_GLSLANG%.zip" "https://github.com/KhronosGroup/glslang/archive/%SHADERC_GLSLANG%.zip" 2b63189efad0348d88d410a5e12ec550a612e0b6ceef64624b8f45491269fb9c || goto error
+call :downloadfile "shaderc-spirv-headers-%SHADERC_SPIRVHEADERS%.zip" "https://github.com/KhronosGroup/SPIRV-Headers/archive/%SHADERC_SPIRVHEADERS%.zip" d2f071e94c081f5a4606559770ebf1f7d1eac92a1def0c3e10609844aa8b69b2 || goto error
+call :downloadfile "shaderc-spirv-tools-%SHADERC_SPIRVTOOLS%.zip" "https://github.com/KhronosGroup/SPIRV-Tools/archive/%SHADERC_SPIRVTOOLS%.zip" 4011be89aa73e3461c9deef73936a62c79a3097590c5135d058041cc9fb99c6f || goto error

 if %DEBUG%==1 (
  echo Building debug and release libraries...
@@ -11,6 +11,10 @@ obj/
 *.pyc
 *.pyo

+# Bazel-related files
+bazel-*
+MODULE.bazel.lock
+
 # System files
 .DS_Store
 .DS_Store?
@@ -42,7 +42,7 @@ ENDMACRO()

 MACRO(CPUINFO_TARGET_ENABLE_CXX11 target)
  SET_TARGET_PROPERTIES(${target} PROPERTIES
-    CXX_STANDARD 14
+    CXX_STANDARD 17
    CXX_EXTENSIONS NO)
 ENDMACRO()

@@ -1,11 +1,11 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12 FATAL_ERROR)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.18 FATAL_ERROR)

 PROJECT(googletest-download NONE)

 INCLUDE(ExternalProject)
 ExternalProject_Add(googletest
-	URL https://github.com/google/googletest/archive/release-1.8.0.zip
-	URL_HASH SHA256=f3ed3b58511efd272eb074a3a6d6fb79d7c2e6a0e374323d1e6bcbcc1ef141bf
+	URL https://github.com/google/googletest/archive/refs/tags/v1.17.0.zip
+	URL_HASH SHA256=40d4ec942217dcc84a9ebe2a68584ada7d4a33a8ee958755763278ea1c5e18ff
 	SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/googletest"
 	BINARY_DIR "${CONFU_DEPENDENCIES_BINARY_DIR}/googletest"
 	CONFIGURE_COMMAND ""
@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 3.1 FATAL_ERROR)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.10 FATAL_ERROR)

 INCLUDE(GNUInstallDirs)

@@ -355,6 +355,12 @@ enum cpuinfo_uarch {
 	cpuinfo_uarch_sunny_cove = 0x0010020C,
 	/** Intel Willow Cove microarchitecture (10 nm, Tiger Lake). */
 	cpuinfo_uarch_willow_cove = 0x0010020D,
+	/** Intel Golden Cove microarchitecture (Sapphire Rapids). */
+	cpuinfo_uarch_golden_cove = 0x0010020E,
+	/** Intel Raptor Cove microarchitecture (Emerald Rapids). */
+	cpuinfo_uarch_raptor_cove = 0x0010020F,
+	/** Intel Redwood Cove microarchitecture (Granite Rapids). */
+	cpuinfo_uarch_redwood_cove = 0x00100210,

 	/** Pentium 4 with Willamette, Northwood, or Foster cores. */
 	cpuinfo_uarch_willamette = 0x00100300,
@@ -519,6 +525,8 @@ enum cpuinfo_uarch {
 	cpuinfo_uarch_cortex_a510 = 0x00300551,
 	/** ARM Cortex-A520. */
 	cpuinfo_uarch_cortex_a520 = 0x00300552,
+	/** ARM Cortex-A320. */
+	cpuinfo_uarch_cortex_a320 = 0x00300553,
 	/** ARM Cortex-A710. */
 	cpuinfo_uarch_cortex_a710 = 0x00300571,
 	/** ARM Cortex-A715. */
@@ -2226,6 +2234,12 @@ struct cpuinfo_riscv_isa {
 	bool c;
 	/* Vector Extension. */
 	bool v;
+
+	/* ISA Extensions */
+	/* Half-Precision Floating-Point Extension. */
+	bool zfh;
+	/* Half-Precision Floating-Point Vector Extension. */
+	bool zvfh;
 };

 extern struct cpuinfo_riscv_isa cpuinfo_isa;
@@ -2301,6 +2315,22 @@ static inline bool cpuinfo_has_riscv_v(void) {
 #endif
 }

+static inline bool cpuinfo_has_riscv_zfh(void) {
+#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64
+	return cpuinfo_isa.zfh;
+#else
+	return false;
+#endif
+}
+
+static inline bool cpuinfo_has_riscv_zvfh(void) {
+#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64
+	return cpuinfo_isa.zvfh;
+#else
+	return false;
+#endif
+}
+
 const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processors(void);
 const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_cores(void);
 const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_clusters(void);
@@ -13,6 +13,7 @@ enum cpuinfo_android_chipset_property {
 	cpuinfo_android_chipset_property_ro_arch,
 	cpuinfo_android_chipset_property_ro_chipname,
 	cpuinfo_android_chipset_property_ro_hardware_chipname,
+	cpuinfo_android_chipset_property_ro_soc_model,
 	cpuinfo_android_chipset_property_max,
 };

@@ -63,4 +63,7 @@ void cpuinfo_arm_android_parse_properties(struct cpuinfo_android_properties prop
 		cpuinfo_android_property_get("ro.hardware.chipname", properties->ro_hardware_chipname);
 	cpuinfo_log_debug(
 		"read ro.hardware.chipname = \"%.*s\"", ro_hardware_chipname_length, properties->ro_hardware_chipname);
+
+	const int ro_soc_model_length = cpuinfo_android_property_get("ro.soc.model", properties->ro_soc_model);
+	cpuinfo_log_debug("read ro.soc.model = \"%.*s\"", ro_soc_model_length, properties->ro_soc_model);
 }
@@ -45,6 +45,7 @@ enum cpuinfo_arm_chipset_series {
 	cpuinfo_arm_chipset_series_qualcomm_msm,
 	cpuinfo_arm_chipset_series_qualcomm_apq,
 	cpuinfo_arm_chipset_series_qualcomm_snapdragon,
+	cpuinfo_arm_chipset_series_qualcomm_sm,
 	cpuinfo_arm_chipset_series_mediatek_mt,
 	cpuinfo_arm_chipset_series_samsung_exynos,
 	cpuinfo_arm_chipset_series_hisilicon_k3v,
@@ -29,6 +29,7 @@ struct cpuinfo_android_properties {
 	char ro_arch[CPUINFO_BUILD_PROP_VALUE_MAX];
 	char ro_chipname[CPUINFO_BUILD_PROP_VALUE_MAX];
 	char ro_hardware_chipname[CPUINFO_BUILD_PROP_VALUE_MAX];
+	char ro_soc_model[CPUINFO_BUILD_PROP_VALUE_MAX];
 };
 #endif

@@ -364,6 +365,8 @@ CPUINFO_INTERNAL struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_f
 	const char ro_chipname[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]);
 CPUINFO_INTERNAL struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_hardware_chipname(
 	const char ro_hardware_chipname[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]);
+CPUINFO_INTERNAL struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_soc_model(
+	const char ro_soc_model[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]);
 #else
 CPUINFO_INTERNAL struct cpuinfo_arm_chipset cpuinfo_arm_linux_decode_chipset_from_proc_cpuinfo_revision(
 	const char proc_cpuinfo_revision[restrict static CPUINFO_REVISION_VALUE_MAX]);
@@ -264,7 +264,7 @@ static bool match_sm(const char* start, const char* end, struct cpuinfo_arm_chip
 	/* Return parsed chipset. */
 	*chipset = (struct cpuinfo_arm_chipset){
 		.vendor = cpuinfo_arm_chipset_vendor_qualcomm,
-		.series = cpuinfo_arm_chipset_series_qualcomm_snapdragon,
+		.series = cpuinfo_arm_chipset_series_qualcomm_sm,
 		.model = model,
 	};
 	return true;
@@ -3491,6 +3491,36 @@ struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_chipname(
 		.series = cpuinfo_arm_chipset_series_unknown,
 	};
 }
+
+struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset_from_ro_soc_model(
+	const char soc_model[restrict static CPUINFO_BUILD_PROP_VALUE_MAX]) {
+	struct cpuinfo_arm_chipset chipset;
+	const size_t soc_model_length = strnlen(soc_model, CPUINFO_BUILD_PROP_VALUE_MAX);
+	const char* soc_model_end = soc_model + soc_model_length;
+
+	/* Check Qualcomm SMxxxx signature */
+	if (match_sm(soc_model, soc_model_end, &chipset)) {
+		cpuinfo_log_debug(
+			"matched Qualcomm SM signature in ro.soc.model string \"%.*s\"",
+			(int)soc_model_length,
+			soc_model);
+		return chipset;
+	}
+
+	/* Check Qualcomm MSM/APQ signatures */
+	if (match_msm_apq(soc_model, soc_model_end, &chipset)) {
+		cpuinfo_log_debug(
+			"matched Qualcomm MSM/APQ signature in ro.soc.model string \"%.*s\"",
+			(int)soc_model_length,
+			soc_model);
+		return chipset;
+	}
+
+	return (struct cpuinfo_arm_chipset){
+		.vendor = cpuinfo_arm_chipset_vendor_unknown,
+		.series = cpuinfo_arm_chipset_series_unknown,
+	};
+}
 #endif /* __ANDROID__ */

 /*
@@ -3837,6 +3867,7 @@ static const char* chipset_series_string[cpuinfo_arm_chipset_series_max] = {
 	[cpuinfo_arm_chipset_series_qualcomm_msm] = "MSM",
 	[cpuinfo_arm_chipset_series_qualcomm_apq] = "APQ",
 	[cpuinfo_arm_chipset_series_qualcomm_snapdragon] = "Snapdragon ",
+	[cpuinfo_arm_chipset_series_qualcomm_sm] = "SM",
 	[cpuinfo_arm_chipset_series_mediatek_mt] = "MT",
 	[cpuinfo_arm_chipset_series_samsung_exynos] = "Exynos ",
 	[cpuinfo_arm_chipset_series_hisilicon_k3v] = "K3V",
@@ -4074,6 +4105,8 @@ struct cpuinfo_arm_chipset cpuinfo_arm_android_decode_chipset(
 			cpuinfo_arm_android_decode_chipset_from_ro_chipname(properties->ro_chipname),
 		[cpuinfo_android_chipset_property_ro_hardware_chipname] =
 			cpuinfo_arm_android_decode_chipset_from_ro_chipname(properties->ro_hardware_chipname),
+		[cpuinfo_android_chipset_property_ro_soc_model] =
+			cpuinfo_arm_android_decode_chipset_from_ro_soc_model(properties->ro_soc_model),
 	};
 	enum cpuinfo_arm_chipset_vendor vendor = cpuinfo_arm_chipset_vendor_unknown;
 	for (size_t i = 0; i < cpuinfo_android_chipset_property_max; i++) {
@@ -1,5 +1,7 @@
+#include <ctype.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

@@ -14,6 +16,92 @@
 #include <cpuinfo/log.h>
 #include <linux/api.h>

+/* Parse a uint32 from sysfs file content */
+static bool uint32_parser(const char* filename, const char* text_start, const char* text_end, void* context) {
+	uint32_t* value_ptr = (uint32_t*)context;
+	if (text_start == text_end) {
+		return false;
+	}
+	uint32_t value = 0;
+	for (const char* p = text_start; p < text_end && *p >= '0' && *p <= '9'; p++) {
+		value = value * 10 + (*p - '0');
+	}
+	*value_ptr = value;
+	return value > 0;
+}
+
+/* Parse cache size with K/M suffix from /sys/devices/system/cpu/cpuN/cache/indexN/size (e.g. "2048K", "1M") */
+static bool cache_size_parser(const char* filename, const char* text_start, const char* text_end, void* context) {
+	uint32_t* size_ptr = (uint32_t*)context;
+	if (text_start == text_end) {
+		return false;
+	}
+	uint32_t value = 0;
+	const char* p = text_start;
+	while (p < text_end && *p >= '0' && *p <= '9') {
+		value = value * 10 + (*p - '0');
+		p++;
+	}
+	if (p == text_start || value == 0) {
+		return false;
+	}
+	uint32_t multiplier = 1024;
+	if (p < text_end && toupper(*p) == 'M') {
+		multiplier = 1024 * 1024;
+	}
+	*size_ptr = value * multiplier;
+	return true;
+}
+
+/* Check if /sys/devices/system/cpu/cpuN/cache/index2/shared_cpu_list indicates a single CPU (per-core cache) */
+static bool shared_cpu_list_parser(const char* filename, const char* text_start, const char* text_end, void* context) {
+	bool* is_per_core = (bool*)context;
+	for (const char* p = text_start; p < text_end; p++) {
+		if (*p == ',' || *p == '-') {
+			*is_per_core = false;
+			return true;
+		}
+	}
+	*is_per_core = (text_start != text_end);
+	return *is_per_core;
+}
+
+/*
+ * Read cache size from sysfs.
+ *
+ * ARM CPUs lack a CPUID-equivalent instruction to query cache properties,
+ * so the library relies on hardcoded values that are often incorrect.
+ * Linux exposes cache topology via sysfs (existed before 2008, documented 2014):
+ * https://raw.githubusercontent.com/torvalds/linux/master/Documentation/ABI/testing/sysfs-devices-system-cpu
+ *
+ */
+static uint32_t cpuinfo_linux_read_sysfs_cache_size(uint32_t cpu_id, uint32_t cache_level) {
+	char path[256];
+
+	/* Verify the index corresponds to the requested cache level */
+	snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u/cache/index%u/level", cpu_id, cache_level);
+	uint32_t actual_level = 0;
+	if (!cpuinfo_linux_parse_small_file(path, 16, uint32_parser, &actual_level) || actual_level != cache_level) {
+		return 0;
+	}
+
+	snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u/cache/index%u/size", cpu_id, cache_level);
+	uint32_t size = 0;
+	if (!cpuinfo_linux_parse_small_file(path, 32, cache_size_parser, &size)) {
+		return 0;
+	}
+	return size;
+}
+
+/* Check if L2 cache is per-core by reading sysfs shared_cpu_list */
+static bool cpuinfo_linux_is_l2_per_core(uint32_t cpu_id) {
+	char path[256];
+	snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u/cache/index2/shared_cpu_list", cpu_id);
+	bool is_per_core = false;
+	cpuinfo_linux_parse_small_file(path, 128, shared_cpu_list_parser, &is_per_core);
+	return is_per_core;
+}
+
 struct cpuinfo_arm_isa cpuinfo_isa = {0};

 static struct cpuinfo_package package = {{0}};
@@ -719,11 +807,21 @@ void cpuinfo_arm_linux_init(void) {
 			 */
 			shared_l3 = false;
 			if (temp_l2.size != 0) {
-				/* Assume L2 is shared by cores in the same
-				 * cluster */
-				if (arm_linux_processors[i].package_leader_id ==
-				    arm_linux_processors[i].system_processor_id) {
+				/* Check if L2 is per-core using sysfs */
+				uint32_t sysfs_l2_size = cpuinfo_linux_read_sysfs_cache_size(
+					arm_linux_processors[i].system_processor_id, 2);
+				bool l2_is_per_core = (sysfs_l2_size > 0) &&
+					cpuinfo_linux_is_l2_per_core(arm_linux_processors[i].system_processor_id);
+
+				if (l2_is_per_core) {
+					/* L2 is private to each core */
 					l2_count += 1;
+				} else {
+					/* Assume L2 is shared by cores in the same cluster */
+					if (arm_linux_processors[i].package_leader_id ==
+					    arm_linux_processors[i].system_processor_id) {
+						l2_count += 1;
+					}
 				}
 			}
 		}
@@ -771,10 +869,27 @@ void cpuinfo_arm_linux_init(void) {
 			&temp_l2,
 			&temp_l3);

-		if (temp_l3.size != 0) {
+		/* Try to read L2 cache size from sysfs (more accurate) */
+		uint32_t sysfs_l2_size =
+			cpuinfo_linux_read_sysfs_cache_size(arm_linux_processors[i].system_processor_id, 2);
+		if (sysfs_l2_size > 0) {
+			temp_l2.size = sysfs_l2_size;
+			/* Recalculate sets to maintain consistency: size = associativity * sets * partitions *
+			 * line_size */
+			if (temp_l2.associativity > 0 && temp_l2.line_size > 0 && temp_l2.partitions > 0) {
+				temp_l2.sets = sysfs_l2_size /
+					(temp_l2.associativity * temp_l2.partitions * temp_l2.line_size);
+			}
+		}
+
+		/* Check if L2 is per-core by reading sysfs */
+		bool l2_is_per_core = (sysfs_l2_size > 0) &&
+			cpuinfo_linux_is_l2_per_core(arm_linux_processors[i].system_processor_id);
+
+		if (temp_l3.size != 0 || l2_is_per_core) {
 			/*
 			 * Assumptions:
-			 * - L2 is private to each core
+			 * - L2 is private to each core (either has L3, or sysfs confirms per-core L2)
 			 * - L3 is shared by cores in the same cluster
 			 * - If cores in different clusters report the same L3,
 			 * it is shared between all cores.
@@ -141,17 +141,20 @@ void cpuinfo_arm_decode_vendor_uarch(
 				case 0xD87: /* Cortex-A725 */
 					*uarch = cpuinfo_uarch_cortex_a725;
 					break;
-				case 0xD8C:
-					*uarch = cpuinfo_uarch_lumex_c1_ultra;
-					break;
-				case 0xD90:
-					*uarch = cpuinfo_uarch_lumex_c1_premium;
+				case 0xD8A:
+					*uarch = cpuinfo_uarch_lumex_c1_nano;
 					break;
 				case 0xD8B:
 					*uarch = cpuinfo_uarch_lumex_c1_pro;
 					break;
-				case 0xD8A:
-					*uarch = cpuinfo_uarch_lumex_c1_nano;
+				case 0xD8C:
+					*uarch = cpuinfo_uarch_lumex_c1_ultra;
+					break;
+				case 0xD8F: /* Cortex-A320 */
+					*uarch = cpuinfo_uarch_cortex_a320;
+					break;
+				case 0xD90:
+					*uarch = cpuinfo_uarch_lumex_c1_premium;
 					break;
 				default:
 					switch (midr_get_part(midr) >> 8) {
@@ -61,11 +61,13 @@ CPUINFO_INTERNAL void cpuinfo_riscv_linux_decode_isa_from_hwcap(struct cpuinfo_r
 * @param[processor] - The Linux ID of the target processor.
 * @param[vendor] - Reference to the cpuinfo_vendor to populate.
 * @param[uarch] - Reference to the cpuinfo_uarch to populate.
+ * @param[isa] - Reference to the cpuinfo_riscv_isa to populate isa extensions.
 */
 CPUINFO_INTERNAL void cpuinfo_riscv_linux_decode_vendor_uarch_from_hwprobe(
 	uint32_t processor,
 	enum cpuinfo_vendor vendor[restrict static 1],
-	enum cpuinfo_uarch uarch[restrict static 1]);
+	enum cpuinfo_uarch uarch[restrict static 1],
+	struct cpuinfo_riscv_isa isa[restrict static 1]);

 /* Used to determine which uarch is associated with the current thread. */
 extern CPUINFO_INTERNAL const uint32_t* cpuinfo_linux_cpu_to_uarch_index_map;
@@ -6,7 +6,7 @@
 #include <riscv/linux/api.h>

 /* ISA structure to hold supported extensions. */
-struct cpuinfo_riscv_isa cpuinfo_isa;
+struct cpuinfo_riscv_isa cpuinfo_isa = {0};

 /* Helper function to bitmask flags and ensure operator precedence. */
 static inline bool bitmask_all(uint32_t flags, uint32_t mask) {
@@ -320,7 +320,8 @@ void cpuinfo_riscv_linux_init(void) {
 		cpuinfo_riscv_linux_decode_vendor_uarch_from_hwprobe(
 			processor,
 			&riscv_linux_processors[processor].core.vendor,
-			&riscv_linux_processors[processor].core.uarch);
+			&riscv_linux_processors[processor].core.uarch,
+			&cpuinfo_isa);

 		/* Populate frequency information of this core. */
 		uint32_t frequency = cpuinfo_linux_get_processor_cur_frequency(processor);
@@ -53,6 +53,30 @@ struct riscv_hwprobe {
 #define RISCV_HWPROBE_EXT_ZBB (1 << 4)
 #define RISCV_HWPROBE_EXT_ZBS (1 << 5)
 #define RISCV_HWPROBE_EXT_ZICBOZ (1 << 6)
+#define RISCV_HWPROBE_EXT_ZBC (1 << 7)
+#define RISCV_HWPROBE_EXT_ZBKB (1 << 8)
+#define RISCV_HWPROBE_EXT_ZBKC (1 << 9)
+#define RISCV_HWPROBE_EXT_ZBKX (1 << 10)
+#define RISCV_HWPROBE_EXT_ZKND (1 << 11)
+#define RISCV_HWPROBE_EXT_ZKNE (1 << 12)
+#define RISCV_HWPROBE_EXT_ZKNH (1 << 13)
+#define RISCV_HWPROBE_EXT_ZKSED (1 << 14)
+#define RISCV_HWPROBE_EXT_ZKSH (1 << 15)
+#define RISCV_HWPROBE_EXT_ZKT (1 << 16)
+#define RISCV_HWPROBE_EXT_ZVBB (1 << 17)
+#define RISCV_HWPROBE_EXT_ZVBC (1 << 18)
+#define RISCV_HWPROBE_EXT_ZVKB (1 << 19)
+#define RISCV_HWPROBE_EXT_ZVKG (1 << 20)
+#define RISCV_HWPROBE_EXT_ZVKNED (1 << 21)
+#define RISCV_HWPROBE_EXT_ZVKNHA (1 << 22)
+#define RISCV_HWPROBE_EXT_ZVKNHB (1 << 23)
+#define RISCV_HWPROBE_EXT_ZVKSED (1 << 24)
+#define RISCV_HWPROBE_EXT_ZVKSH (1 << 25)
+#define RISCV_HWPROBE_EXT_ZVKT (1 << 26)
+#define RISCV_HWPROBE_EXT_ZFH (1 << 27)
+#define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28)
+#define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29)
+#define RISCV_HWPROBE_EXT_ZVFH (1 << 30)
 #define RISCV_HWPROBE_KEY_CPUPERF_0 5
 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
 #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
@@ -72,7 +96,8 @@ struct riscv_hwprobe {
 void cpuinfo_riscv_linux_decode_vendor_uarch_from_hwprobe(
 	uint32_t processor,
 	enum cpuinfo_vendor vendor[restrict static 1],
-	enum cpuinfo_uarch uarch[restrict static 1]) {
+	enum cpuinfo_uarch uarch[restrict static 1],
+	struct cpuinfo_riscv_isa isa[restrict static 1]) {
 	struct riscv_hwprobe pairs[] = {
 		{
 			.key = RISCV_HWPROBE_KEY_MVENDORID,
@@ -83,6 +108,9 @@ void cpuinfo_riscv_linux_decode_vendor_uarch_from_hwprobe(
 		{
 			.key = RISCV_HWPROBE_KEY_MIMPID,
 		},
+		{
+			.key = RISCV_HWPROBE_KEY_IMA_EXT_0,
+		},
 	};
 	const size_t pairs_count = sizeof(pairs) / sizeof(struct riscv_hwprobe);

@@ -128,6 +156,7 @@ void cpuinfo_riscv_linux_decode_vendor_uarch_from_hwprobe(
 	uint32_t vendor_id = 0;
 	uint32_t arch_id = 0;
 	uint32_t imp_id = 0;
+	uint64_t ima_ext_0 = 0;
 	for (size_t pair = 0; pair < pairs_count; pair++) {
 		switch (pairs[pair].key) {
 			case RISCV_HWPROBE_KEY_MVENDORID:
@@ -139,6 +168,9 @@ void cpuinfo_riscv_linux_decode_vendor_uarch_from_hwprobe(
 			case RISCV_HWPROBE_KEY_MIMPID:
 				imp_id = pairs[pair].value;
 				break;
+			case RISCV_HWPROBE_KEY_IMA_EXT_0:
+				ima_ext_0 = pairs[pair].value;
+				break;
 			default:
 				/* The key value may be -1 if unsupported. */
 				break;
@@ -146,6 +178,16 @@ void cpuinfo_riscv_linux_decode_vendor_uarch_from_hwprobe(
 	}
 	cpuinfo_riscv_decode_vendor_uarch(vendor_id, arch_id, imp_id, vendor, uarch);

+	/* Parse ISA extensions retrieved. */
+	if (ima_ext_0 != 0) {
+		if (ima_ext_0 & RISCV_HWPROBE_EXT_ZFH) {
+			isa->zfh = true;
+		}
+		if (ima_ext_0 & RISCV_HWPROBE_EXT_ZVFH) {
+			isa->zvfh = true;
+		}
+	}
+
 cleanup:
 	CPU_FREE(cpu_set);
 }
@@ -171,6 +171,12 @@ enum cpuinfo_uarch cpuinfo_x86_decode_uarch(
 						case 0x8C: // Tiger U
 						case 0x8D: // Tiger H
 							return cpuinfo_uarch_willow_cove;
+						case 0x8F: // Sapphire Rapids
+							return cpuinfo_uarch_golden_cove;
+						case 0xCF: // Emerald Rapids
+							return cpuinfo_uarch_raptor_cove;
+						case 0xAD: // Granite Rapids
+							return cpuinfo_uarch_redwood_cove;
 						/* Low-power cores */
 						case 0x1C: // Diamondville,
 							   // Silverthorne,
@@ -585,8 +585,9 @@ cubeb_destroy(cubeb * context);
                                NULL if this stream is input only. When input
                                and output stream parameters are supplied, their
                                rate has to be the same.
-    @param latency_frames Stream latency in frames.  Valid range
-                          is [1, 96000].
+    @param latency_frames Requested stream latency in frames. Valid range is
+                          [1, 96000]. The actual latency may differ depending
+                          on the backend, platform, and hardware.
    @param data_callback Will be called to preroll data before playback is
                         started by cubeb_stream_start.
    @param state_callback A pointer to a state callback.
@@ -294,6 +294,9 @@ set_timeout(struct timeval * timeout, unsigned int ms)
 static void
 stream_buffer_decrement(cubeb_stream * stm, long count)
 {
+  if (count < 0 || (snd_pcm_uframes_t)count > stm->bufframes) {
+    count = stm->bufframes;
+  }
  char * bufremains =
      stm->buffer + WRAP(snd_pcm_frames_to_bytes)(stm->pcm, count);
  memmove(stm->buffer, bufremains,
@@ -405,12 +405,15 @@ cbjack_process(jack_nframes_t nframes, void * arg)

  for (int j = 0; j < MAX_STREAMS; j++) {
    cubeb_stream * stm = &ctx->streams[j];
-    float * bufs_out[stm->out_params.channels];
-    float * bufs_in[stm->in_params.channels];

    if (!stm->in_use)
      continue;

+    float * bufs_out[MAX_CHANNELS] = {};
+    float * bufs_in[MAX_CHANNELS] = {};
+    XASSERT(stm->out_params.channels <= MAX_CHANNELS);
+    XASSERT(stm->in_params.channels <= MAX_CHANNELS);
+
    // handle xruns by skipping audio that should have been played
    stm->position += t_jack_xruns * ctx->fragment_size * stm->ratio;

@@ -851,6 +854,14 @@ cbjack_stream_init(cubeb * context, cubeb_stream ** stream,
    return CUBEB_ERROR_INVALID_FORMAT;
  }

+  if ((output_stream_params &&
+       (output_stream_params->channels < 1 ||
+        output_stream_params->channels > MAX_CHANNELS)) ||
+      (input_stream_params && (input_stream_params->channels < 1 ||
+                               input_stream_params->channels > MAX_CHANNELS))) {
+    return CUBEB_ERROR_INVALID_FORMAT;
+  }
+
  if ((input_device && input_device != JACK_DEFAULT_IN) ||
      (output_device && output_device != JACK_DEFAULT_OUT)) {
    return CUBEB_ERROR_NOT_SUPPORTED;
@@ -14,6 +14,7 @@
 #include <algorithm>
 #include <atomic>
 #include <audioclient.h>
+#include <audiopolicy.h>
 #include <avrt.h>
 #include <cmath>
 #include <devicetopology.h>
@@ -317,6 +318,7 @@ struct cubeb {
 };

 class wasapi_endpoint_notification_client;
+class wasapi_session_notification_client;

 /* We have three possible callbacks we can use with a stream:
 * - input only
@@ -415,6 +417,10 @@ struct cubeb_stream {
     audio device changes and route the audio to the new default audio output
     device */
  com_ptr<wasapi_endpoint_notification_client> notification_client;
+  /* Session notification client, to be notified when the audio session is
+     disconnected (e.g. when an audio device is removed from the system). */
+  com_ptr<IAudioSessionControl> session_control;
+  com_ptr<wasapi_session_notification_client> session_notification_client;
  /* Main andle to the WASAPI capture stream. */
  com_ptr<IAudioClient> input_client;
  /* Interface to use the event driven capture interface */
@@ -431,6 +437,10 @@ struct cubeb_stream {
   * practice, we read from the input stream in the output callback, so
   * this is not used, but it is necessary to start getting input data. */
  HANDLE input_available_event = 0;
+  /* Signaled by stream_start/stream_stop when the active state changes,
+     so the render thread can promote/demote its MMCSS task accordingly
+     instead of remaining promoted while idle. */
+  HANDLE mmcss_event = 0;
  /* Each cubeb_stream has its own thread. */
  HANDLE thread = 0;
  /* The lock protects all members that are touched by the render thread or
@@ -834,6 +844,89 @@ private:
  DWORD last_device_change;
 };

+class wasapi_session_notification_client : public IAudioSessionEvents {
+public:
+  ULONG STDMETHODCALLTYPE AddRef() { return InterlockedIncrement(&ref_count); }
+
+  ULONG STDMETHODCALLTYPE Release()
+  {
+    ULONG ulRef = InterlockedDecrement(&ref_count);
+    if (0 == ulRef) {
+      delete this;
+    }
+    return ulRef;
+  }
+
+  HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, VOID ** ppvInterface)
+  {
+    if (__uuidof(IUnknown) == riid) {
+      AddRef();
+      *ppvInterface = (IUnknown *)this;
+    } else if (__uuidof(IAudioSessionEvents) == riid) {
+      AddRef();
+      *ppvInterface = (IAudioSessionEvents *)this;
+    } else {
+      *ppvInterface = NULL;
+      return E_NOINTERFACE;
+    }
+    return S_OK;
+  }
+
+  wasapi_session_notification_client(HANDLE event)
+      : ref_count(1), reconfigure_event(event)
+  {
+  }
+
+  virtual ~wasapi_session_notification_client() {}
+
+  HRESULT STDMETHODCALLTYPE
+  OnSessionDisconnected(AudioSessionDisconnectReason reason)
+  {
+    LOG("session: Audio session disconnected, reason: %d", reason);
+    BOOL ok = SetEvent(reconfigure_event);
+    if (!ok) {
+      LOG("session: SetEvent on reconfigure_event failed: %lx", GetLastError());
+    }
+    return S_OK;
+  }
+
+  HRESULT STDMETHODCALLTYPE OnDisplayNameChanged(LPCWSTR value,
+                                                 LPCGUID event_context)
+  {
+    return S_OK;
+  }
+  HRESULT STDMETHODCALLTYPE OnIconPathChanged(LPCWSTR value,
+                                              LPCGUID event_context)
+  {
+    return S_OK;
+  }
+  HRESULT STDMETHODCALLTYPE OnSimpleVolumeChanged(float volume, BOOL mute,
+                                                  LPCGUID event_context)
+  {
+    return S_OK;
+  }
+  HRESULT STDMETHODCALLTYPE OnChannelVolumeChanged(DWORD channel_count,
+                                                   float volumes[],
+                                                   DWORD changed_channel,
+                                                   LPCGUID event_context)
+  {
+    return S_OK;
+  }
+  HRESULT STDMETHODCALLTYPE OnGroupingParamChanged(LPCGUID grouping_param,
+                                                   LPCGUID event_context)
+  {
+    return S_OK;
+  }
+  HRESULT STDMETHODCALLTYPE OnStateChanged(AudioSessionState state)
+  {
+    return S_OK;
+  }
+
+private:
+  LONG ref_count;
+  HANDLE reconfigure_event;
+};
+
 namespace {

 long
@@ -1414,8 +1507,9 @@ static unsigned int __stdcall wasapi_stream_render_loop(LPVOID stream)
  } com;

  bool is_playing = true;
-  HANDLE wait_array[4] = {stm->shutdown_event, stm->reconfigure_event,
-                          stm->refill_event, stm->input_available_event};
+  HANDLE wait_array[5] = {stm->shutdown_event, stm->mmcss_event,
+                          stm->reconfigure_event, stm->refill_event,
+                          stm->input_available_event};
  HANDLE mmcss_handle = NULL;
  HRESULT hr = 0;
  DWORD mmcss_task_index = 0;
@@ -1428,15 +1522,6 @@ static unsigned int __stdcall wasapi_stream_render_loop(LPVOID stream)
    return 0;
  }

-  /* We could consider using "Pro Audio" here for WebAudio and
-     maybe WebRTC. */
-  mmcss_handle = AvSetMmThreadCharacteristicsA("Audio", &mmcss_task_index);
-  if (!mmcss_handle) {
-    /* This is not fatal, but we might glitch under heavy load. */
-    LOG("Unable to use mmcss to bump the render thread priority: %lx",
-        GetLastError());
-  }
-
  while (is_playing) {
    DWORD waitResult = WaitForMultipleObjects(ARRAY_LENGTH(wait_array),
                                              wait_array, FALSE, INFINITE);
@@ -1450,7 +1535,30 @@ static unsigned int __stdcall wasapi_stream_render_loop(LPVOID stream)
      }
      continue;
    }
-    case WAIT_OBJECT_0 + 1: { /* reconfigure */
+    case WAIT_OBJECT_0 + 1: { /* mmcss: active state changed */
+      /* stm->active was set by wasapi_stream_start/_stop before signaling,
+         and SetEvent provides the necessary memory barrier. */
+      if (stm->active && !mmcss_handle) {
+        /* We could consider using "Pro Audio" here for WebAudio and
+           maybe WebRTC. */
+        mmcss_handle =
+            AvSetMmThreadCharacteristicsA("Audio", &mmcss_task_index);
+        if (!mmcss_handle) {
+          /* This is not fatal, but we might glitch under heavy load. */
+          LOG("Unable to use mmcss to bump the render thread priority: %lx",
+              GetLastError());
+        } else {
+          LOG("MMCSS render thread promoted (task index %lu)",
+              mmcss_task_index);
+        }
+      } else if (!stm->active && mmcss_handle) {
+        AvRevertMmThreadCharacteristics(mmcss_handle);
+        mmcss_handle = NULL;
+        LOG("MMCSS render thread demoted");
+      }
+      continue;
+    }
+    case WAIT_OBJECT_0 + 2: { /* reconfigure */
      auto_lock lock(stm->stream_reset_lock);
      if (!stm->active) {
        /* Avoid reconfiguring, stream start will handle it. */
@@ -1460,13 +1568,12 @@ static unsigned int __stdcall wasapi_stream_render_loop(LPVOID stream)
      XASSERT(stm->output_client || stm->input_client);
      LOG("Reconfiguring the stream");
      /* Close the stream */
-      bool was_running = false;
      if (stm->output_client) {
-        was_running = stm->output_client->Stop() == S_OK;
+        stm->output_client->Stop();
        LOG("Output stopped.");
      }
      if (stm->input_client) {
-        was_running = stm->input_client->Stop() == S_OK;
+        stm->input_client->Stop();
        LOG("Input stopped.");
      }
      close_wasapi_stream(stm);
@@ -1484,7 +1591,7 @@ static unsigned int __stdcall wasapi_stream_render_loop(LPVOID stream)
      }
      LOG("Stream setup successfuly.");
      XASSERT(stm->output_client || stm->input_client);
-      if (was_running && stm->output_client) {
+      if (stm->output_client) {
        hr = stm->output_client->Start();
        if (FAILED(hr)) {
          LOG("Error starting output after reconfigure, error: %lx", hr);
@@ -1493,7 +1600,7 @@ static unsigned int __stdcall wasapi_stream_render_loop(LPVOID stream)
        }
        LOG("Output started after reconfigure.");
      }
-      if (was_running && stm->input_client) {
+      if (stm->input_client) {
        hr = stm->input_client->Start();
        if (FAILED(hr)) {
          LOG("Error starting input after reconfiguring, error: %lx", hr);
@@ -1504,12 +1611,12 @@ static unsigned int __stdcall wasapi_stream_render_loop(LPVOID stream)
      }
      break;
    }
-    case WAIT_OBJECT_0 + 2: /* refill */
+    case WAIT_OBJECT_0 + 3: /* refill */
      XASSERT((has_input(stm) && has_output(stm)) ||
              (!has_input(stm) && has_output(stm)));
      is_playing = stm->refill_callback(stm);
      break;
-    case WAIT_OBJECT_0 + 3: { /* input available */
+    case WAIT_OBJECT_0 + 4: { /* input available */
      bool rv = get_input_buffer(stm);
      if (!rv) {
        is_playing = false;
@@ -2643,6 +2750,22 @@ setup_wasapi_stream(cubeb_stream * stm)
      return CUBEB_ERROR;
    }

+    hr = stm->output_client->GetService(__uuidof(IAudioSessionControl),
+                                        stm->session_control.receive_vpp());
+    if (SUCCEEDED(hr)) {
+      stm->session_notification_client.reset(
+          new wasapi_session_notification_client(stm->reconfigure_event));
+      hr = stm->session_control->RegisterAudioSessionNotification(
+          stm->session_notification_client.get());
+      if (FAILED(hr)) {
+        LOG("Could not register session notification client: %lx", hr);
+        stm->session_notification_client = nullptr;
+        stm->session_control = nullptr;
+      }
+    } else {
+      LOG("Could not get the IAudioSessionControl: %lx", hr);
+    }
+
 #ifdef CUBEB_WASAPI_USE_IAUDIOSTREAMVOLUME
    /* Restore the stream volume over a device change. */
    if (stream_set_volume(stm, stm->volume) != CUBEB_OK) {
@@ -2842,6 +2965,12 @@ wasapi_stream_init(cubeb * context, cubeb_stream ** stream,
    return CUBEB_ERROR;
  }

+  stm->mmcss_event = CreateEvent(NULL, 0, 0, NULL);
+  if (!stm->mmcss_event) {
+    LOG("Can't create the mmcss event, error: %lx", GetLastError());
+    return CUBEB_ERROR;
+  }
+
  stm->thread_ready_event = CreateEvent(NULL, 0, 0, NULL);
  if (!stm->thread_ready_event) {
    LOG("Can't create the thread ready event, error: %lx", GetLastError());
@@ -2906,6 +3035,13 @@ close_wasapi_stream(cubeb_stream * stm)

  stm->stream_reset_lock.assert_current_thread_owns();

+  if (stm->session_control && stm->session_notification_client) {
+    stm->session_control->UnregisterAudioSessionNotification(
+        stm->session_notification_client.get());
+    stm->session_notification_client = nullptr;
+    stm->session_control = nullptr;
+  }
+
 #ifdef CUBEB_WASAPI_USE_IAUDIOSTREAMVOLUME
  stm->audio_stream_volume = nullptr;
 #endif
@@ -2960,6 +3096,7 @@ wasapi_stream_release(cubeb_stream * stm)
    CloseHandle(stm->reconfigure_event);
    CloseHandle(stm->refill_event);
    CloseHandle(stm->input_available_event);
+    CloseHandle(stm->mmcss_event);

    CloseHandle(stm->thread);

@@ -3056,6 +3193,11 @@ wasapi_stream_start(cubeb_stream * stm)

  stm->active = true;

+  if (!SetEvent(stm->mmcss_event)) {
+    LOG("wasapi_stream_start: SetEvent(mmcss_event) failed: %lx",
+        GetLastError());
+  }
+
  stm->state_callback(stm, stm->user_ptr, CUBEB_STATE_STARTED);

  return CUBEB_OK;
@@ -3088,6 +3230,11 @@ wasapi_stream_stop(cubeb_stream * stm)

    stm->active = false;

+    if (!SetEvent(stm->mmcss_event)) {
+      LOG("wasapi_stream_stop: SetEvent(mmcss_event) failed: %lx",
+          GetLastError());
+    }
+
    wasapi_state_callback(stm, stm->user_ptr, CUBEB_STATE_STOPPED);
  }

@@ -3563,11 +3710,13 @@ wasapi_register_device_collection_changed(
  }

  if (collection_changed_callback) {
-    // Make sure it has been unregistered first.
-    XASSERT(((devtype & CUBEB_DEVICE_TYPE_INPUT) &&
-             !context->input_collection_changed_callback) ||
-            ((devtype & CUBEB_DEVICE_TYPE_OUTPUT) &&
-             !context->output_collection_changed_callback));
+    if (((devtype & CUBEB_DEVICE_TYPE_INPUT) &&
+         context->input_collection_changed_callback) ||
+        ((devtype & CUBEB_DEVICE_TYPE_OUTPUT) &&
+         context->output_collection_changed_callback)) {
+      LOG("register_device_collection_changed: callback already registered");
+      return CUBEB_ERROR_INVALID_PARAMETER;
+    }

    // Stop the notification client. Notifications arrive on
    // a separate thread. We stop them here to avoid
@@ -3,6 +3,9 @@

 [![Ubuntu 22.04 CI (GCC 11)](https://github.com/fastfloat/fast_float/actions/workflows/ubuntu22.yml/badge.svg)](https://github.com/fastfloat/fast_float/actions/workflows/ubuntu22.yml)

+*Note: This library is for C++ users. C programmers should consider [ffc.h](https://github.com/kolemannix/ffc.h). It is a high-performance port of fast_float to C.*
+
+
 The fast_float library provides fast header-only implementations for the C++
 from_chars functions for `float` and `double` types as well as integer types.
 These functions convert ASCII strings representing decimal values (e.g.,
@@ -10,6 +13,7 @@ These functions convert ASCII strings representing decimal values (e.g.,
 even). In our experience, these `fast_float` functions many times faster than
 comparable number-parsing functions from existing C++ standard libraries.

+
 Specifically, `fast_float` provides the following two functions to parse
 floating-point numbers with a C++17-like syntax (the library itself only
 requires C++11):
@@ -18,9 +22,9 @@ requires C++11):
 from_chars_result from_chars(char const *first, char const *last, float &value, ...);
 from_chars_result from_chars(char const *first, char const *last, double &value, ...);
 ```
+If they are available on your system, we also support fixed-width floating-point types such as `std::float64_t`, `std::float32_t`, `std::float16_t`, and `std::bfloat16_t`.

-You can also parse integer types:
-
+You can also parse integer types such as `char`, `short`, `long`, `long long`,  `unsigned char`, `unsigned short`, `unsigned long`, `unsigned long long`, `bool` (0/1),  `int8_t`, `int16_t`, `int32_t`, `int64_t`, `uint8_t`, `uint16_t`, `uint32_t`, `uint64_t`.
 ```C++
 from_chars_result from_chars(char const *first, char const *last, int &value, ...);
 from_chars_result from_chars(char const *first, char const *last, unsigned &value, ...);
@@ -69,7 +73,7 @@ int main() {
 }
 ```

-Though the C++17 standard has you do a comparison with `std::errc()` to check whether the conversion worked, you can avoid it by casting the result to a `bool` like so:
+Prior to C++26, checking for a successful `std::from_chars` conversion requires comparing the `from_chars_result::ec` member to `std::errc()`. As an extension `fast_float::from_chars` supports the improved C++26 API that allows checking the result by converting it to `bool`, like so:

 ```cpp
 #include "fast_float/fast_float.h"
@@ -83,7 +87,7 @@ int main() {
    std::cout << "parsed the number " << result << std::endl;
    return EXIT_SUCCESS;
  }
-  std::cerr << "failed to parse " << result << std::endl;
+  std::cerr << "failed to parse " << input << std::endl;
  return EXIT_FAILURE;
 }
 ```
@@ -141,9 +145,12 @@ Furthermore, we have the following restrictions:
  fixed-width floating-point types such as `std::float64_t`, `std::float32_t`,
  `std::float16_t`, and `std::bfloat16_t`.
 * We only support the decimal format: we do not support hexadecimal strings.
-* For values that are either very large or very small (e.g., `1e9999`), we
-  represent it using the infinity or negative infinity value and the returned
+* For values that are very large positives or negatives (e.g., `1e9999`), we
+  represent them using a positive or negative infinity and the returned
  `ec` is set to `std::errc::result_out_of_range`.
+* For values that are very close to zero (e.g., `1e-9999`), we represent them
+  using a positive or negative zero and the returned `ec` is set to
+  `std::errc::result_out_of_range`.

 We support Visual Studio, macOS, Linux, freeBSD. We support big and little
 endian. We support 32-bit and 64-bit systems.
@@ -401,6 +408,23 @@ except `fast_float::integer_times_pow10()` does not report out-of-range errors,
 underflows to zero or overflows to infinity when the resulting value is
 out of range.

+You can use template overloads to get the result converted to different
+supported floating-point types: `float`, `double`, etc.
+For example, to get result as `float` use
+`fast_float::integer_times_pow10<float>()` specialization:
+```C++
+const uint64_t W = 12345678;
+const int Q = 23;
+const float result = fast_float::integer_times_pow10<float>(W, Q);
+std::cout.precision(9);
+std::cout << "float: " << W << " * 10^" << Q << " = " << result << " ("
+          << (result == 12345678e23f ? "==" : "!=") << "expected)\n";
+```
+outputs
+```
+float: 12345678 * 10^23 = 1.23456782e+30 (==expected)
+```
+
 Overloads of `fast_float::integer_times_pow10()` are provided for
 signed and unsigned integer types: `int64_t`, `uint64_t`, etc.

@@ -443,7 +467,7 @@ framework](https://github.com/microsoft/LightGBM).
 Packages
 ------

-[![Packaging status](https://repology.org/badge/vertical-allrepos/fastfloat.svg)](https://repology.org/project/fastfloat/versions)
+[![Packaging status](https://repology.org/badge/vertical-allrepos/fast-float.svg)](https://repology.org/project/fast-float/versions)


 ## References
@@ -468,6 +492,7 @@ Packages
  [Jackson](https://github.com/FasterXML/jackson-core).
 * [There is a C# port of the fast_float
  library](https://github.com/CarlVerret/csFastFloat) called `csFastFloat`.
+* [There is a plain C port of the fast_float library](https://github.com/kolemannix/ffc.h) called ffc.h

 ## How fast is it?

@@ -516,7 +541,7 @@ sufficiently recent version of CMake (3.11 or better at least):
 FetchContent_Declare(
  fast_float
  GIT_REPOSITORY https://github.com/fastfloat/fast_float.git
-  GIT_TAG tags/v8.1.0
+  GIT_TAG tags/v8.2.5
  GIT_SHALLOW TRUE)

 FetchContent_MakeAvailable(fast_float)
@@ -532,7 +557,7 @@ You may also use [CPM](https://github.com/cpm-cmake/CPM.cmake), like so:
 CPMAddPackage(
  NAME fast_float
  GITHUB_REPOSITORY "fastfloat/fast_float"
-  GIT_TAG v8.1.0)
+  GIT_TAG v8.2.5)
 ```

 ## Using as single header
@@ -544,7 +569,7 @@ if desired as described in the command line help.

 You may directly download automatically generated single-header files:

-<https://github.com/fastfloat/fast_float/releases/download/v8.1.0/fast_float.h>
+<https://github.com/fastfloat/fast_float/releases/download/v8.2.5/fast_float.h>

 ## Benchmarking

@@ -598,6 +623,11 @@ long digits.
 The library includes code adapted from Google Wuffs (written by Nigel Tao) which
 was originally published under the Apache 2.0 license.

+## Stars
+
+
+[![Star History Chart](https://api.star-history.com/svg?repos=fastfloat/fast_float&type=Date)](https://www.star-history.com/#fastfloat/fast_float&Date)
+
 ## License

 <sup>
@@ -32,7 +32,7 @@ template <typename UC> fastfloat_really_inline constexpr bool has_simd_opt() {
 // able to optimize it well.
 template <typename UC>
 fastfloat_really_inline constexpr bool is_integer(UC c) noexcept {
-  return !(c > UC('9') || c < UC('0'));
+  return (unsigned)(c - UC('0')) <= 9u;
 }

 fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) {
@@ -42,6 +42,11 @@ fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) {
         (val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56;
 }

+fastfloat_really_inline constexpr uint32_t byteswap_32(uint32_t val) {
+  return (val >> 24) | ((val >> 8) & 0x0000FF00u) | ((val << 8) & 0x00FF0000u) |
+         (val << 24);
+}
+
 // Read 8 UC into a u64. Truncates UC if not char.
 template <typename UC>
 fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
@@ -63,6 +68,25 @@ read8_to_u64(UC const *chars) {
  return val;
 }

+// Read 4 UC into a u32. Truncates UC if not char.
+template <typename UC>
+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint32_t
+read4_to_u32(UC const *chars) {
+  if (cpp20_and_in_constexpr() || !std::is_same<UC, char>::value) {
+    uint32_t val = 0;
+    for (int i = 0; i < 4; ++i) {
+      val |= uint32_t(uint8_t(*chars)) << (i * 8);
+      ++chars;
+    }
+    return val;
+  }
+  uint32_t val;
+  ::memcpy(&val, chars, sizeof(uint32_t));
+#if FASTFLOAT_IS_BIG_ENDIAN == 1
+  val = byteswap_32(val);
+#endif
+  return val;
+}
 #ifdef FASTFLOAT_SSE2

 fastfloat_really_inline uint64_t simd_read8_to_u64(__m128i const data) {
@@ -144,6 +168,18 @@ is_made_of_eight_digits_fast(uint64_t val) noexcept {
            0x8080808080808080));
 }

+fastfloat_really_inline constexpr bool
+is_made_of_four_digits_fast(uint32_t val) noexcept {
+  return !((((val + 0x46464646) | (val - 0x30303030)) & 0x80808080));
+}
+
+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint32_t
+parse_four_digits_unrolled(uint32_t val) noexcept {
+  val -= 0x30303030;
+  val = (val * 10) + (val >> 8);
+  return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF;
+}
+
 #ifdef FASTFLOAT_HAS_SIMD

 // Call this if chars might not be 8 digits.
@@ -509,6 +545,148 @@ parse_int_string(UC const *p, UC const *pend, T &value,

  UC const *const start_digits = p;

+  FASTFLOAT_IF_CONSTEXPR17((std::is_same<T, std::uint8_t>::value)) {
+    if (base == 10) {
+      const size_t len = (size_t)(pend - p);
+      if (len == 0) {
+        if (has_leading_zeros) {
+          value = 0;
+          answer.ec = std::errc();
+          answer.ptr = p;
+        } else {
+          answer.ec = std::errc::invalid_argument;
+          answer.ptr = first;
+        }
+        return answer;
+      }
+
+      uint32_t digits;
+
+#if FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && FASTFLOAT_HAS_BIT_CAST
+      if (std::is_constant_evaluated()) {
+        uint8_t str[4]{};
+        for (size_t j = 0; j < 4 && j < len; ++j) {
+          str[j] = static_cast<uint8_t>(p[j]);
+        }
+        digits = std::bit_cast<uint32_t>(str);
+#if FASTFLOAT_IS_BIG_ENDIAN
+        digits = byteswap_32(digits);
+#endif
+      }
+#else
+      if (false) {
+      }
+#endif
+      else if (len >= 4) {
+        ::memcpy(&digits, p, 4);
+#if FASTFLOAT_IS_BIG_ENDIAN
+        digits = byteswap_32(digits);
+#endif
+      } else {
+        uint32_t b0 = static_cast<uint8_t>(p[0]);
+        uint32_t b1 = (len > 1) ? static_cast<uint8_t>(p[1]) : 0xFFu;
+        uint32_t b2 = (len > 2) ? static_cast<uint8_t>(p[2]) : 0xFFu;
+        uint32_t b3 = 0xFFu;
+        digits = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
+      }
+
+      uint32_t magic =
+          ((digits + 0x46464646u) | (digits - 0x30303030u)) & 0x80808080u;
+      uint32_t tz = (uint32_t)countr_zero_32(magic); // 7, 15, 23, 31, or 32
+      uint32_t nd = (tz == 32) ? 4 : (tz >> 3);
+      nd = (uint32_t)(nd < len ? nd : len);
+      if (nd == 0) {
+        if (has_leading_zeros) {
+          value = 0;
+          answer.ec = std::errc();
+          answer.ptr = p;
+          return answer;
+        }
+        answer.ec = std::errc::invalid_argument;
+        answer.ptr = first;
+        return answer;
+      }
+      if (nd > 3) {
+        const UC *q = p + nd;
+        size_t rem = len - nd;
+        while (rem) {
+          if (*q < UC('0') || *q > UC('9'))
+            break;
+          ++q;
+          --rem;
+        }
+        answer.ec = std::errc::result_out_of_range;
+        answer.ptr = q;
+        return answer;
+      }
+
+      digits ^= 0x30303030u;
+      digits <<= ((4 - nd) * 8);
+
+      uint32_t check = ((digits >> 24) & 0xff) | ((digits >> 8) & 0xff00) |
+                       ((digits << 8) & 0xff0000);
+      if (check > 0x00020505) {
+        answer.ec = std::errc::result_out_of_range;
+        answer.ptr = p + nd;
+        return answer;
+      }
+      value = (uint8_t)((0x640a01 * digits) >> 24);
+      answer.ec = std::errc();
+      answer.ptr = p + nd;
+      return answer;
+    }
+  }
+
+  FASTFLOAT_IF_CONSTEXPR17((std::is_same<T, std::uint16_t>::value)) {
+    if (base == 10) {
+      const size_t len = size_t(pend - p);
+      if (len == 0) {
+        if (has_leading_zeros) {
+          value = 0;
+          answer.ec = std::errc();
+          answer.ptr = p;
+        } else {
+          answer.ec = std::errc::invalid_argument;
+          answer.ptr = first;
+        }
+        return answer;
+      }
+
+      if (len >= 4) {
+        uint32_t digits = read4_to_u32(p);
+        if (is_made_of_four_digits_fast(digits)) {
+          uint32_t v = parse_four_digits_unrolled(digits);
+          if (len >= 5 && is_integer(p[4])) {
+            v = v * 10 + uint32_t(p[4] - '0');
+            if (len >= 6 && is_integer(p[5])) {
+              answer.ec = std::errc::result_out_of_range;
+              const UC *q = p + 5;
+              while (q != pend && is_integer(*q)) {
+                q++;
+              }
+              answer.ptr = q;
+              return answer;
+            }
+            if (v > 65535) {
+              answer.ec = std::errc::result_out_of_range;
+              answer.ptr = p + 5;
+              return answer;
+            }
+            value = uint16_t(v);
+            answer.ec = std::errc();
+            answer.ptr = p + 5;
+            return answer;
+          }
+          // 4 digits
+          value = uint16_t(v);
+          answer.ec = std::errc();
+          answer.ptr = p + 4;
+          return answer;
+        }
+      }
+    }
+  }
+
  uint64_t i = 0;
  if (base == 10) {
    loop_parse_if_eight_digits(p, pend, i); // use SIMD if possible
@@ -1,7 +1,6 @@
 #ifndef FASTFLOAT_DIGIT_COMPARISON_H
 #define FASTFLOAT_DIGIT_COMPARISON_H

-#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <iterator>
@@ -38,11 +37,8 @@ constexpr static uint64_t powers_of_ten_uint64[] = {1UL,
 // this algorithm is not even close to optimized, but it has no practical
 // effect on performance: in order to have a faster algorithm, we'd need
 // to slow down performance for faster algorithms, and this is still fast.
-template <typename UC>
 fastfloat_really_inline FASTFLOAT_CONSTEXPR14 int32_t
-scientific_exponent(parsed_number_string_t<UC> &num) noexcept {
-  uint64_t mantissa = num.mantissa;
-  int32_t exponent = int32_t(num.exponent);
+scientific_exponent(uint64_t mantissa, int32_t exponent) noexcept {
  while (mantissa >= 10000) {
    mantissa /= 10000;
    exponent += 4;
@@ -112,7 +108,7 @@ fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am,
  if (-am.power2 >= mantissa_shift) {
    // have a denormal float
    int32_t shift = -am.power2 + 1;
-    cb(am, std::min<int32_t>(shift, 64));
+    cb(am, (shift < 64 ? shift : 64));
    // check for round-up: if rounding-nearest carried us to the hidden bit.
    am.power2 = (am.mantissa <
                 (uint64_t(1) << binary_format<T>::mantissa_explicit_bits()))
@@ -398,7 +394,7 @@ inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp(
    FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp)));
  }

-  // compare digits, and use it to director rounding
+  // compare digits, and use it to direct rounding
  int ord = real_digits.compare(theor_digits);
  adjusted_mantissa answer = am;
  round<T>(answer, [ord](adjusted_mantissa &a, int32_t shift) {
@@ -419,7 +415,7 @@ inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp(
  return answer;
 }

-// parse the significant digits as a big integer to unambiguously round the
+// parse the significant digits as a big integer to unambiguously round
 // the significant digits. here, we are trying to determine how to round
 // an extended float representation close to `b+h`, halfway between `b`
 // (the float rounded-down) and `b+u`, the next positive float. this
@@ -438,7 +434,8 @@ digit_comp(parsed_number_string_t<UC> &num, adjusted_mantissa am) noexcept {
  // remove the invalid exponent bias
  am.power2 -= invalid_am_bias;

-  int32_t sci_exp = scientific_exponent(num);
+  int32_t sci_exp =
+      scientific_exponent(num.mantissa, static_cast<int32_t>(num.exponent));
  size_t max_digits = binary_format<T>::max_digits();
  size_t digits = 0;
  bigint bigmant;
@@ -63,6 +63,20 @@ integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept;
 FASTFLOAT_CONSTEXPR20 inline double
 integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept;

+/**
+ * This function is a template overload of `integer_times_pow10()`
+ * that returns a floating-point value of type `T` that is one of
+ * supported floating-point types (e.g. `double`, `float`).
+ */
+template <typename T>
+FASTFLOAT_CONSTEXPR20
+    typename std::enable_if<is_supported_float_type<T>::value, T>::type
+    integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept;
+template <typename T>
+FASTFLOAT_CONSTEXPR20
+    typename std::enable_if<is_supported_float_type<T>::value, T>::type
+    integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept;
+
 /**
 * from_chars for integer types.
 */
@@ -2,6 +2,7 @@
 #define FASTFLOAT_FLOAT_COMMON_H

 #include <cfloat>
+#include <cstddef>
 #include <cstdint>
 #include <cassert>
 #include <cstring>
@@ -16,8 +17,8 @@
 #include "constexpr_feature_detect.h"

 #define FASTFLOAT_VERSION_MAJOR 8
-#define FASTFLOAT_VERSION_MINOR 1
-#define FASTFLOAT_VERSION_PATCH 0
+#define FASTFLOAT_VERSION_MINOR 2
+#define FASTFLOAT_VERSION_PATCH 5

 #define FASTFLOAT_STRINGIZE_IMPL(x) #x
 #define FASTFLOAT_STRINGIZE(x) FASTFLOAT_STRINGIZE_IMPL(x)
@@ -267,18 +268,147 @@ struct is_supported_char_type
                             > {
 };

+template <typename UC>
+inline FASTFLOAT_CONSTEXPR14 bool
+fastfloat_strncasecmp3(UC const *actual_mixedcase,
+                       UC const *expected_lowercase) {
+  uint64_t mask{0};
+  FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; }
+  else FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) {
+    mask = 0x0020002000200020;
+  }
+  else FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) {
+    mask = 0x0000002000000020;
+  }
+  else {
+    return false;
+  }
+
+  uint64_t val1{0}, val2{0};
+  if (cpp20_and_in_constexpr()) {
+    for (size_t i = 0; i < 3; i++) {
+      if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) {
+        return false;
+      }
+    }
+    return true;
+  } else {
+    FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1 || sizeof(UC) == 2) {
+      ::memcpy(&val1, actual_mixedcase, 3 * sizeof(UC));
+      ::memcpy(&val2, expected_lowercase, 3 * sizeof(UC));
+      val1 |= mask;
+      val2 |= mask;
+      return val1 == val2;
+    }
+    else FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) {
+      ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC));
+      ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC));
+      val1 |= mask;
+      if (val1 != val2) {
+        return false;
+      }
+      return (actual_mixedcase[2] | 32) == (expected_lowercase[2]);
+    }
+    else {
+      return false;
+    }
+  }
+}
+
+template <typename UC>
+inline FASTFLOAT_CONSTEXPR14 bool
+fastfloat_strncasecmp5(UC const *actual_mixedcase,
+                       UC const *expected_lowercase) {
+  uint64_t mask{0};
+  uint64_t val1{0}, val2{0};
+  if (cpp20_and_in_constexpr()) {
+    for (size_t i = 0; i < 5; i++) {
+      if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) {
+        return false;
+      }
+    }
+    return true;
+  } else {
+    FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) {
+      mask = 0x2020202020202020;
+      ::memcpy(&val1, actual_mixedcase, 5 * sizeof(UC));
+      ::memcpy(&val2, expected_lowercase, 5 * sizeof(UC));
+      val1 |= mask;
+      val2 |= mask;
+      return val1 == val2;
+    }
+    else FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) {
+      mask = 0x0020002000200020;
+      ::memcpy(&val1, actual_mixedcase, 4 * sizeof(UC));
+      ::memcpy(&val2, expected_lowercase, 4 * sizeof(UC));
+      val1 |= mask;
+      if (val1 != val2) {
+        return false;
+      }
+      return (actual_mixedcase[4] | 32) == (expected_lowercase[4]);
+    }
+    else FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) {
+      mask = 0x0000002000000020;
+      ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC));
+      ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC));
+      val1 |= mask;
+      if (val1 != val2) {
+        return false;
+      }
+      ::memcpy(&val1, actual_mixedcase + 2, 2 * sizeof(UC));
+      ::memcpy(&val2, expected_lowercase + 2, 2 * sizeof(UC));
+      val1 |= mask;
+      if (val1 != val2) {
+        return false;
+      }
+      return (actual_mixedcase[4] | 32) == (expected_lowercase[4]);
+    }
+    else {
+      return false;
+    }
+  }
+}
+
 // Compares two ASCII strings in a case insensitive manner.
 template <typename UC>
 inline FASTFLOAT_CONSTEXPR14 bool
 fastfloat_strncasecmp(UC const *actual_mixedcase, UC const *expected_lowercase,
                      size_t length) {
-  for (size_t i = 0; i < length; ++i) {
-    UC const actual = actual_mixedcase[i];
-    if ((actual < 256 ? actual | 32 : actual) != expected_lowercase[i]) {
-      return false;
-    }
+  uint64_t mask{0};
+  FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; }
+  else FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) {
+    mask = 0x0020002000200020;
+  }
+  else FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) {
+    mask = 0x0000002000000020;
+  }
+  else {
+    return false;
+  }
+
+  if (cpp20_and_in_constexpr()) {
+    for (size_t i = 0; i < length; i++) {
+      if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) {
+        return false;
+      }
+    }
+    return true;
+  } else {
+    uint64_t val1{0}, val2{0};
+    size_t sz{8 / (sizeof(UC))};
+    for (size_t i = 0; i < length; i += sz) {
+      val1 = val2 = 0;
+      sz = sz < (length - i) ? sz : length - i;
+      ::memcpy(&val1, actual_mixedcase + i, sz * sizeof(UC));
+      ::memcpy(&val2, expected_lowercase + i, sz * sizeof(UC));
+      val1 |= mask;
+      val2 |= mask;
+      if (val1 != val2) {
+        return false;
+      }
+    }
+    return true;
  }
-  return true;
 }

 #ifndef FLT_EVAL_METHOD
@@ -362,6 +492,52 @@ leading_zeroes(uint64_t input_num) {
 #endif
 }

+/* Helper C++14 constexpr generic implementation of countr_zero for 32-bit */
+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 int
+countr_zero_generic_32(uint32_t input_num) {
+  if (input_num == 0) {
+    return 32;
+  }
+  int last_bit = 0;
+  if (!(input_num & 0x0000FFFF)) {
+    input_num >>= 16;
+    last_bit |= 16;
+  }
+  if (!(input_num & 0x00FF)) {
+    input_num >>= 8;
+    last_bit |= 8;
+  }
+  if (!(input_num & 0x0F)) {
+    input_num >>= 4;
+    last_bit |= 4;
+  }
+  if (!(input_num & 0x3)) {
+    input_num >>= 2;
+    last_bit |= 2;
+  }
+  if (!(input_num & 0x1)) {
+    last_bit |= 1;
+  }
+  return last_bit;
+}
+
+/* count trailing zeroes for 32-bit integers */
+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 int
+countr_zero_32(uint32_t input_num) {
+  if (cpp20_and_in_constexpr()) {
+    return countr_zero_generic_32(input_num);
+  }
+#ifdef FASTFLOAT_VISUAL_STUDIO
+  unsigned long trailing_zero = 0;
+  if (_BitScanForward(&trailing_zero, input_num)) {
+    return (int)trailing_zero;
+  }
+  return 32;
+#else
+  return input_num == 0 ? 32 : __builtin_ctz(input_num);
+#endif
+}
+
 // slow emulation routine for 32-bit
 fastfloat_really_inline constexpr uint64_t emulu(uint32_t x, uint32_t y) {
  return x * (uint64_t)y;
@@ -406,8 +582,8 @@ full_multiplication(uint64_t a, uint64_t b) {
  // But MinGW on ARM64 doesn't have native support for 64-bit multiplications
  answer.high = __umulh(a, b);
  answer.low = a * b;
-#elif defined(FASTFLOAT_32BIT) ||                                              \
-    (defined(_WIN64) && !defined(__clang__) && !defined(_M_ARM64))
+#elif defined(FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__) &&   \
+                                   !defined(_M_ARM64) && !defined(__GNUC__))
  answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64
 #elif defined(FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__)
  __uint128_t r = ((__uint128_t)a) * b;
@@ -1166,6 +1342,9 @@ static_assert(std::is_same<equiv_uint_t<std::float64_t>, uint64_t>::value,
 static_assert(
    std::numeric_limits<std::float64_t>::is_iec559,
    "std::float64_t must fulfill the requirements of IEC 559 (IEEE 754)");
+
+template <>
+struct binary_format<std::float64_t> : public binary_format<double> {};
 #endif // __STDCPP_FLOAT64_T__

 #ifdef __STDCPP_FLOAT32_T__
@@ -1174,6 +1353,9 @@ static_assert(std::is_same<equiv_uint_t<std::float32_t>, uint32_t>::value,
 static_assert(
    std::numeric_limits<std::float32_t>::is_iec559,
    "std::float32_t must fulfill the requirements of IEC 559 (IEEE 754)");
+
+template <>
+struct binary_format<std::float32_t> : public binary_format<float> {};
 #endif // __STDCPP_FLOAT32_T__

 #ifdef __STDCPP_FLOAT16_T__
@@ -1245,7 +1427,6 @@ constexpr chars_format adjust_for_feature_macros(chars_format fmt) {
      ;
 }
 } // namespace detail
-
 } // namespace fast_float

 #endif
@@ -35,7 +35,7 @@ from_chars_result_t<UC>
    ++first;
  }
  if (last - first >= 3) {
-    if (fastfloat_strncasecmp(first, str_const_nan<UC>(), 3)) {
+    if (fastfloat_strncasecmp3(first, str_const_nan<UC>())) {
      answer.ptr = (first += 3);
      value = minusSign ? -std::numeric_limits<T>::quiet_NaN()
                        : std::numeric_limits<T>::quiet_NaN();
@@ -54,9 +54,9 @@ from_chars_result_t<UC>
      }
      return answer;
    }
-    if (fastfloat_strncasecmp(first, str_const_inf<UC>(), 3)) {
+    if (fastfloat_strncasecmp3(first, str_const_inf<UC>())) {
      if ((last - first >= 8) &&
-          fastfloat_strncasecmp(first + 3, str_const_inf<UC>() + 3, 5)) {
+          fastfloat_strncasecmp5(first + 3, str_const_inf<UC>() + 3)) {
        answer.ptr = first + 8;
      } else {
        answer.ptr = first + 3;
@@ -155,7 +155,7 @@ template <> struct from_chars_caller<std::float32_t> {
    // if std::float32_t is defined, and we are in C++23 mode; macro set for
    // float32; set value to float due to equivalence between float and
    // float32_t
-    float val;
+    float val = 0.0f;
    auto ret = from_chars_advanced(first, last, val, options);
    value = val;
    return ret;
@@ -172,7 +172,7 @@ template <> struct from_chars_caller<std::float64_t> {
    // if std::float64_t is defined, and we are in C++23 mode; macro set for
    // float64; set value as double due to equivalence between double and
    // float64_t
-    double val;
+    double val = 0.0;
    auto ret = from_chars_advanced(first, last, val, options);
    value = val;
    return ret;
@@ -251,7 +251,7 @@ clinger_fast_path_impl(uint64_t mantissa, int64_t exponent, bool is_negative,
 * parsing options or other parsing custom function implemented by user.
 */
 template <typename T, typename UC>
-FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
 from_chars_advanced(parsed_number_string_t<UC> &pns, T &value) noexcept {
  static_assert(is_supported_float_type<T>::value,
                "only some floating-point types are supported");
@@ -290,7 +290,7 @@ from_chars_advanced(parsed_number_string_t<UC> &pns, T &value) noexcept {
 }

 template <typename T, typename UC>
-FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
 from_chars_float_advanced(UC const *first, UC const *last, T &value,
                          parse_options_t<UC> options) noexcept {

@@ -344,44 +344,79 @@ from_chars(UC const *first, UC const *last, T &value, int base) noexcept {
  return from_chars_advanced(first, last, value, options);
 }

-FASTFLOAT_CONSTEXPR20 inline double
-integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept {
-  double value;
+template <typename T>
+FASTFLOAT_CONSTEXPR20
+    typename std::enable_if<is_supported_float_type<T>::value, T>::type
+    integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept {
+  T value;
  if (clinger_fast_path_impl(mantissa, decimal_exponent, false, value))
    return value;

  adjusted_mantissa am =
-      compute_float<binary_format<double>>(decimal_exponent, mantissa);
+      compute_float<binary_format<T>>(decimal_exponent, mantissa);
  to_float(false, am, value);
  return value;
 }

-FASTFLOAT_CONSTEXPR20 inline double
-integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept {
+template <typename T>
+FASTFLOAT_CONSTEXPR20
+    typename std::enable_if<is_supported_float_type<T>::value, T>::type
+    integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept {
  const bool is_negative = mantissa < 0;
  const uint64_t m = static_cast<uint64_t>(is_negative ? -mantissa : mantissa);

-  double value;
+  T value;
  if (clinger_fast_path_impl(m, decimal_exponent, is_negative, value))
    return value;

-  adjusted_mantissa am =
-      compute_float<binary_format<double>>(decimal_exponent, m);
+  adjusted_mantissa am = compute_float<binary_format<T>>(decimal_exponent, m);
  to_float(is_negative, am, value);
  return value;
 }

+FASTFLOAT_CONSTEXPR20 inline double
+integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept {
+  return integer_times_pow10<double>(mantissa, decimal_exponent);
+}
+
+FASTFLOAT_CONSTEXPR20 inline double
+integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept {
+  return integer_times_pow10<double>(mantissa, decimal_exponent);
+}
+
 // the following overloads are here to avoid surprising ambiguity for int,
 // unsigned, etc.
+template <typename T, typename Int>
+FASTFLOAT_CONSTEXPR20
+    typename std::enable_if<is_supported_float_type<T>::value &&
+                                std::is_integral<Int>::value &&
+                                !std::is_signed<Int>::value,
+                            T>::type
+    integer_times_pow10(Int mantissa, int decimal_exponent) noexcept {
+  return integer_times_pow10<T>(static_cast<uint64_t>(mantissa),
+                                decimal_exponent);
+}
+
+template <typename T, typename Int>
+FASTFLOAT_CONSTEXPR20
+    typename std::enable_if<is_supported_float_type<T>::value &&
+                                std::is_integral<Int>::value &&
+                                std::is_signed<Int>::value,
+                            T>::type
+    integer_times_pow10(Int mantissa, int decimal_exponent) noexcept {
+  return integer_times_pow10<T>(static_cast<int64_t>(mantissa),
+                                decimal_exponent);
+}
+
 template <typename Int>
-FASTFLOAT_CONSTEXPR20 inline typename std::enable_if<
+FASTFLOAT_CONSTEXPR20 typename std::enable_if<
    std::is_integral<Int>::value && !std::is_signed<Int>::value, double>::type
 integer_times_pow10(Int mantissa, int decimal_exponent) noexcept {
  return integer_times_pow10(static_cast<uint64_t>(mantissa), decimal_exponent);
 }

 template <typename Int>
-FASTFLOAT_CONSTEXPR20 inline typename std::enable_if<
+FASTFLOAT_CONSTEXPR20 typename std::enable_if<
    std::is_integral<Int>::value && std::is_signed<Int>::value, double>::type
 integer_times_pow10(Int mantissa, int decimal_exponent) noexcept {
  return integer_times_pow10(static_cast<int64_t>(mantissa), decimal_exponent);
@@ -421,7 +456,7 @@ template <size_t TypeIx> struct from_chars_advanced_caller {

 template <> struct from_chars_advanced_caller<1> {
  template <typename T, typename UC>
-  FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
+  fastfloat_really_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
  call(UC const *first, UC const *last, T &value,
       parse_options_t<UC> options) noexcept {
    return from_chars_float_advanced(first, last, value, options);
@@ -430,7 +465,7 @@ template <> struct from_chars_advanced_caller<1> {

 template <> struct from_chars_advanced_caller<2> {
  template <typename T, typename UC>
-  FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
+  fastfloat_really_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
  call(UC const *first, UC const *last, T &value,
       parse_options_t<UC> options) noexcept {
    return from_chars_int_advanced(first, last, value, options);
@@ -438,7 +473,7 @@ template <> struct from_chars_advanced_caller<2> {
 };

 template <typename T, typename UC>
-FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
 from_chars_advanced(UC const *first, UC const *last, T &value,
                    parse_options_t<UC> options) noexcept {
  return from_chars_advanced_caller<
@@ -1,12 +1,11 @@
 /* 7zFile.h -- File IO
-2023-03-05 : Igor Pavlov : Public domain */
+: Igor Pavlov : Public domain */

 #ifndef ZIP7_INC_FILE_H
 #define ZIP7_INC_FILE_H

 #ifdef _WIN32
 #define USE_WINDOWS_FILE
-// #include <windows.h>
 #endif

 #ifdef USE_WINDOWS_FILE
@@ -1,5 +1,5 @@
 /* 7zTypes.h -- Basic types
-2024-01-24 : Igor Pavlov : Public domain */
+: Igor Pavlov : Public domain */

 #ifndef ZIP7_7Z_TYPES_H
 #define ZIP7_7Z_TYPES_H
@@ -46,8 +46,9 @@ typedef int SRes;


 #ifdef _MSC_VER
+  #define MY_ALIGN_IN_STRUCT(n) __declspec(align(n))
  #if _MSC_VER > 1200
-    #define MY_ALIGN(n) __declspec(align(n))
+    #define MY_ALIGN(n) MY_ALIGN_IN_STRUCT(n)
  #else
    #define MY_ALIGN(n)
  #endif
@@ -58,6 +59,7 @@ typedef int SRes;
  #define MY_ALIGN(n) alignas(n)
  */
  #define MY_ALIGN(n) __attribute__ ((aligned(n)))
+  #define MY_ALIGN_IN_STRUCT(n) MY_ALIGN(n)
 #endif


@@ -1,7 +1,7 @@
-#define MY_VER_MAJOR 25
-#define MY_VER_MINOR 0
+#define MY_VER_MAJOR 26
+#define MY_VER_MINOR 1
 #define MY_VER_BUILD 0
-#define MY_VERSION_NUMBERS "25.00"
+#define MY_VERSION_NUMBERS "26.01"
 #define MY_VERSION MY_VERSION_NUMBERS

 #ifdef MY_CPU_NAME
@@ -10,12 +10,12 @@
  #define MY_VERSION_CPU MY_VERSION
 #endif

-#define MY_DATE "2025-07-05"
+#define MY_DATE "2026-04-27"
 #undef MY_COPYRIGHT
 #undef MY_VERSION_COPYRIGHT_DATE
 #define MY_AUTHOR_NAME "Igor Pavlov"
 #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain"
-#define MY_COPYRIGHT_CR "Copyright (c) 1999-2025 Igor Pavlov"
+#define MY_COPYRIGHT_CR "Copyright (c) 1999-2026 Igor Pavlov"

 #ifdef USE_COPYRIGHT_CR
  #define MY_COPYRIGHT MY_COPYRIGHT_CR
@@ -1,11 +1,17 @@
-/* 7zWindows.h -- StdAfx
-2023-04-02 : Igor Pavlov : Public domain */
+/* 7zWindows.h -- Windows.h and related code
+Igor Pavlov : Public domain */

 #ifndef ZIP7_INC_7Z_WINDOWS_H
 #define ZIP7_INC_7Z_WINDOWS_H

 #ifdef _WIN32

+#if defined(_MSC_VER) && _MSC_VER >= 1950 && !defined(__clang__) // VS2026
+// <Windows.h> and some another windows files need that option
+// VS2026: wtypesbase.h: warning C4865: 'tagCLSCTX': the underlying type will change from 'int' to 'unsigned int' when '/Zc:enumTypes' is specified on the command line
+#pragma warning(disable : 4865)
+#endif
+
 #if defined(__clang__)
 # pragma clang diagnostic push
 #endif
@@ -1,5 +1,5 @@
 /* Alloc.h -- Memory allocation functions
-2024-01-22 : Igor Pavlov : Public domain */
+: Igor Pavlov : Public domain */

 #ifndef ZIP7_INC_ALLOC_H
 #define ZIP7_INC_ALLOC_H
@@ -25,40 +25,40 @@ void *MyRealloc(void *address, size_t size);
 void *z7_AlignedAlloc(size_t size);
 void  z7_AlignedFree(void *p);

+extern const ISzAlloc g_Alloc;
+extern const ISzAlloc g_AlignedAlloc;
+
 #ifdef _WIN32
+  void *MidAlloc(size_t size);
+  void MidFree(void *address);
+  extern const ISzAlloc g_MidAlloc;
+#else
+  #define MidAlloc(size)    z7_AlignedAlloc(size)
+  #define MidFree(address)  z7_AlignedFree(address)
+  #define g_MidAlloc g_AlignedAlloc
+#endif

 #ifdef Z7_LARGE_PAGES
-void SetLargePageSize(void);
-#endif

-void *MidAlloc(size_t size);
-void MidFree(void *address);
-void *BigAlloc(size_t size);
-void BigFree(void *address);
-
-/* #define Z7_BIG_ALLOC_IS_ZERO_FILLED */
+#define Z7_LARGE_PAGES_FLAG_USE_HUGEPAGE  (1 << 0)  //    PAGE_ALIGNED / MADV_HUGEPAGE
+#define Z7_LARGE_PAGES_FLAG_NO_PAGECODE   (1 << 1)  // no PAGE_ALIGNED / no madvise
+#define Z7_LARGE_PAGES_FLAG_NO_MADVISE    (1 << 2)  //    PAGE_ALIGNED / no madvise : for THP=always
+#define Z7_LARGE_PAGES_FLAG_NO_HUGEPAGE   (1 << 3)  //    PAGE_ALIGNED / MADV_NOHUGEPAGE
+#define Z7_LARGE_PAGES_FLAG_FAIL_STOP     (1 << 15) // for benchmarks
+#define Z7_LARGE_PAGES_FLAG_DIRECT_PAGE_SIZE  (1 << 16)
+#define Z7_LARGE_PAGES_FLAG_DIRECT_THRESHOLD  (1 << 17)

+void z7_LargePage_Set(UInt32 flags, size_t pageSize, size_t threshold);
+  
+  void *BigAlloc(size_t size);
+  void BigFree(void *address);
+  extern const ISzAlloc g_BigAlloc;
 #else
-
-#define MidAlloc(size)    z7_AlignedAlloc(size)
-#define MidFree(address)  z7_AlignedFree(address)
-#define BigAlloc(size)    z7_AlignedAlloc(size)
-#define BigFree(address)  z7_AlignedFree(address)
-
+  #define BigAlloc(size)    MidAlloc(size)
+  #define BigFree(address)  MidFree(address)
+  #define g_BigAlloc g_MidAlloc
 #endif

-extern const ISzAlloc g_Alloc;
-
-#ifdef _WIN32
-extern const ISzAlloc g_BigAlloc;
-extern const ISzAlloc g_MidAlloc;
-#else
-#define g_BigAlloc g_AlignedAlloc
-#define g_MidAlloc g_AlignedAlloc
-#endif
-
-extern const ISzAlloc g_AlignedAlloc;
-

 typedef struct
 {
@@ -54,6 +54,12 @@
 #pragma GCC diagnostic ignored "-Wexcess-padding"
 #endif

+#if defined(Z7_APPLE_CLANG_VERSION) && __clang_major__ >= 21
+// warning: function MyAlloc might be an allocator wrapper
+// clang in xcode: clang 21.0.0
+#pragma GCC diagnostic ignored "-Wallocator-wrappers"
+#endif
+
 #if __clang_major__ >= 16
 #pragma GCC diagnostic ignored "-Wunsafe-buffer-usage"
 #endif
@@ -72,7 +78,7 @@

 #endif // __clang__

-#if defined(_WIN32) && defined(__clang__) && __clang_major__ >= 16
+#if defined(__clang__) && __clang_major__ >= 16
 // #pragma GCC diagnostic ignored "-Wcast-function-type-strict"
 #define Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION \
  _Pragma("GCC diagnostic ignored \"-Wcast-function-type-strict\"")
@@ -31,7 +31,12 @@ MY_CPU_64BIT means that processor can work with 64-bit registers.
    #define MY_CPU_NAME "x32"
    #define MY_CPU_SIZEOF_POINTER 4
  #else
-    #define MY_CPU_NAME "x64"
+    #if defined(__APX_EGPR__) || defined(__EGPR__)
+      #define MY_CPU_NAME "x64-apx"
+      #define MY_CPU_AMD64_APX
+    #else
+      #define MY_CPU_NAME "x64"
+    #endif
    #define MY_CPU_SIZEOF_POINTER 8
  #endif
  #define MY_CPU_64BIT
@@ -249,11 +254,12 @@ MY_CPU_64BIT means that processor can work with 64-bit registers.
 #endif


+// _LITTLE_ENDIAN macro can be defined for big-endian platform with some compilers
+ 
 #if defined(MY_CPU_X86_OR_AMD64) \
    || defined(MY_CPU_ARM_LE) \
    || defined(MY_CPU_ARM64_LE) \
    || defined(MY_CPU_IA64_LE) \
-    || defined(_LITTLE_ENDIAN) \
    || defined(__LITTLE_ENDIAN__) \
    || defined(__ARMEL__) \
    || defined(__THUMBEL__) \
@@ -596,8 +602,20 @@ problem-4 : performace:
 #define SetBe32a(p, v)   { *(UInt32 *)(void *)(p) = (v); }
 #define SetBe16a(p, v)   { *(UInt16 *)(void *)(p) = (v); }

+// gcc and clang for powerpc can transform load byte access to load reverse word access.
+// sp we can use byte access instead of word access. Z7_BSWAP64 cab be slow
+#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_64BIT)
+#define GetUi64a(p)   Z7_BSWAP64 (*(const UInt64 *)(const void *)(p))
+#else
 #define GetUi64a(p)      GetUi64(p)
+#endif
+
+#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED)
+#define GetUi32a(p)   Z7_BSWAP32 (*(const UInt32 *)(const void *)(p))
+#else
 #define GetUi32a(p)      GetUi32(p)
+#endif
+
 #define GetUi16a(p)      GetUi16(p)
 #define SetUi32a(p, v)   SetUi32(p, v)
 #define SetUi16a(p, v)   SetUi16(p, v)
@@ -1,5 +1,5 @@
 /* Precomp.h -- precompilation file
-2024-01-25 : Igor Pavlov : Public domain */
+: Igor Pavlov : Public domain */

 #ifndef ZIP7_INC_PRECOMP_H
 #define ZIP7_INC_PRECOMP_H
@@ -40,18 +40,18 @@
 #endif
 */

+#ifndef Z7_LARGE_PAGES
+#if !defined(Z7_NO_LARGE_PAGES) && !defined(UNDER_CE)
+#define Z7_LARGE_PAGES 1
+#endif
+#endif
+
 #ifdef _WIN32
 /*
  this "Precomp.h" file must be included before <windows.h>,
  if we want to define _WIN32_WINNT before <windows.h>.
 */

-#ifndef Z7_LARGE_PAGES
-#ifndef Z7_NO_LARGE_PAGES
-#define Z7_LARGE_PAGES 1
-#endif
-#endif
-
 #ifndef Z7_LONG_PATH
 #ifndef Z7_NO_LONG_PATH
 #define Z7_LONG_PATH 1
@@ -1,5 +1,5 @@
 /* 7zArcIn.c -- 7z Input functions
-2023-09-07 : Igor Pavlov : Public domain */
+: Igor Pavlov : Public domain */

 #include "Precomp.h"

@@ -289,9 +289,9 @@ static SRes WaitId(CSzData *sd, UInt32 id)
  }
 }

-static SRes RememberBitVector(CSzData *sd, UInt32 numItems, const Byte **v)
+static SRes RememberBitVector(CSzData *sd, size_t numItems, const Byte **v)
 {
-  const UInt32 numBytes = (numItems + 7) >> 3;
+  const size_t numBytes = (numItems + 7) >> 3;
  if (numBytes > sd->Size)
    return SZ_ERROR_ARCHIVE;
  *v = sd->Data;
@@ -317,11 +317,11 @@ static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems)
  return sum;
 }

-static Z7_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc)
+static Z7_NO_INLINE SRes ReadBitVector(CSzData *sd, size_t numItems, Byte **v, ISzAllocPtr alloc)
 {
  Byte allAreDefined;
  Byte *v2;
-  const UInt32 numBytes = (numItems + 7) >> 3;
+  const size_t numBytes = (numItems + 7) >> 3;
  *v = NULL;
  SZ_READ_BYTE(allAreDefined)
  if (numBytes == 0)
@@ -345,9 +345,9 @@ static Z7_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, I
  return SZ_OK;
 }

-static Z7_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc)
+static Z7_NO_INLINE SRes ReadUi32s(CSzData *sd2, size_t numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc)
 {
-  UInt32 i;
+  size_t i;
  CSzData sd;
  UInt32 *vals;
  const Byte *defs;
@@ -366,7 +366,7 @@ static Z7_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *c
  return SZ_OK;
 }

-static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc)
+static SRes ReadBitUi32s(CSzData *sd, size_t numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc)
 {
  SzBitUi32s_Free(crcs, alloc);
  RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc))
@@ -1027,42 +1027,39 @@ static SRes SzReadAndDecodePackedStreams(
  return SZ_OK;
 }

+// (size & 1) == 0
+// (data) is aligned for 2-bytes
 static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size_t *offsets)
 {
-  size_t pos = 0;
+  const Byte *p, *lim;
  *offsets++ = 0;
  if (numFiles == 0)
    return (size == 0) ? SZ_OK : SZ_ERROR_ARCHIVE;
  if (size < 2)
    return SZ_ERROR_ARCHIVE;
-  if (data[size - 2] != 0 || data[size - 1] != 0)
+  lim = data + size;
+  if (*(const UInt16 *)(const void *)(lim - 2))
    return SZ_ERROR_ARCHIVE;
+  p = data;
  do
  {
-    const Byte *p;
-    if (pos == size)
+    if (p >= lim)
      return SZ_ERROR_ARCHIVE;
-    for (p = data + pos;
-      #ifdef _WIN32
-      *(const UInt16 *)(const void *)p != 0
-      #else
-      p[0] != 0 || p[1] != 0
-      #endif
-      ; p += 2);
-    pos = (size_t)(p - data) + 2;
-    *offsets++ = (pos >> 1);
+    for (; *(const UInt16 *)(const void *)p; p += 2);
+    p += 2;
+    *offsets++ = (size_t)(p - data) >> 1;
  }
  while (--numFiles);
-  return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE;
+  return (p == lim) ? SZ_OK : SZ_ERROR_ARCHIVE;
 }

-static Z7_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num,
+static Z7_NO_INLINE SRes ReadTime(CSzBitUi64s *p, size_t num,
    CSzData *sd2,
    const CBuf *tempBufs, UInt32 numTempBufs,
    ISzAllocPtr alloc)
 {
  CSzData sd;
-  UInt32 i;
+  size_t i;
  CNtfsFileTime *vals;
  Byte *defs;
  Byte external;
@@ -1215,6 +1212,7 @@ static SRes SzReadHeader2(
        {
          namesSize = (size_t)size - 1;
          namesData = sd->Data;
+          SKIP_DATA(sd, namesSize)
        }
        else
        {
@@ -1226,15 +1224,11 @@ static SRes SzReadHeader2(
          namesSize = (tempBufs)[index].size;
        }

-        if ((namesSize & 1) != 0)
+        if (namesSize & 1)
          return SZ_ERROR_ARCHIVE;
        MY_ALLOC(size_t, p->FileNameOffsets, numFiles + 1, allocMain)
        MY_ALLOC_ZE_AND_CPY(p->FileNames, namesSize, namesData, allocMain)
        RINOK(SzReadFileNames(p->FileNames, namesSize, numFiles, p->FileNameOffsets))
-        if (external == 0)
-        {
-          SKIP_DATA(sd, namesSize)
-        }
        break;
      }
      case k7zIdEmptyStream:
@@ -1,5 +1,5 @@
 /* Alloc.c -- Memory allocation functions
-2024-02-18 : Igor Pavlov : Public domain */
+: Igor Pavlov : Public domain */

 #include "Precomp.h"

@@ -24,8 +24,6 @@
 #endif

 // #define SZ_ALLOC_DEBUG
-/* #define SZ_ALLOC_DEBUG */
-
 /* use SZ_ALLOC_DEBUG to debug alloc/free operations */
 #ifdef SZ_ALLOC_DEBUG

@@ -34,9 +32,10 @@
 static int g_allocCount = 0;
 #ifdef _WIN32
 static int g_allocCountMid = 0;
+#ifdef Z7_LARGE_PAGES
 static int g_allocCountBig = 0;
 #endif
-
+#endif

 #define CONVERT_INT_TO_STR(charType, tempSize) \
  char temp[tempSize]; unsigned i = 0; \
@@ -140,8 +139,10 @@ static void PrintAddr(void *p)
 #else

 #ifdef _WIN32
+#ifdef Z7_LARGE_PAGES
 #define PRINT_ALLOC(name, cnt, size, ptr)
 #endif
+#endif
 #define PRINT_FREE(name, cnt, ptr)
 #define Print(s)
 #define PrintLn()
@@ -245,6 +246,7 @@ void MidFree(void *address)
 }

 #ifdef Z7_LARGE_PAGES
+// #pragma message("Z7_LARGE_PAGES")

 #ifdef MEM_LARGE_PAGES
  #define MY_MEM_LARGE_PAGES  MEM_LARGE_PAGES
@@ -253,32 +255,14 @@ void MidFree(void *address)
 #endif

 extern
-SIZE_T g_LargePageSize;
-SIZE_T g_LargePageSize = 0;
-typedef SIZE_T (WINAPI *Func_GetLargePageMinimum)(VOID);
-
-void SetLargePageSize(void)
-{
-  SIZE_T size;
-#ifdef Z7_USE_DYN_GetLargePageMinimum
-Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION
-
-  const
-   Func_GetLargePageMinimum fn =
-  (Func_GetLargePageMinimum) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")),
-       "GetLargePageMinimum");
-  if (!fn)
-    return;
-  size = fn();
-#else
-  size = GetLargePageMinimum();
-#endif
-  if (size == 0 || (size & (size - 1)) != 0)
-    return;
-  g_LargePageSize = size;
-}
-
-#endif // Z7_LARGE_PAGES
+size_t g_LargePageSize;
+size_t g_LargePageSize = 0;
+extern
+size_t g_LargePageThresholdMin;
+size_t g_LargePageThresholdMin = 0;
+extern
+UInt32 g_LargePageFlags;
+UInt32 g_LargePageFlags = 0;

 void *BigAlloc(size_t size)
 {
@@ -289,12 +273,10 @@ void *BigAlloc(size_t size)

  #ifdef Z7_LARGE_PAGES
  {
-    SIZE_T ps = g_LargePageSize;
-    if (ps != 0 && ps <= (1 << 30) && size > (ps / 2))
+    const size_t ps = g_LargePageSize - 1;
+    if (ps < (1u << 30) && size > g_LargePageThresholdMin)
    {
-      size_t size2;
-      ps--;
-      size2 = (size + ps) & ~ps;
+      const size_t size2 = (size + ps) & ~ps;
      if (size2 >= size)
      {
        void *p = VirtualAlloc(NULL, size2, MEM_COMMIT | MY_MEM_LARGE_PAGES, PAGE_READWRITE);
@@ -303,6 +285,8 @@ void *BigAlloc(size_t size)
          PRINT_ALLOC("Alloc-BM ", g_allocCountMid, size2, p)
          return p;
        }
+        if (g_LargePageFlags & Z7_LARGE_PAGES_FLAG_FAIL_STOP)
+          return p;
      }
    }
  }
@@ -317,6 +301,7 @@ void BigFree(void *address)
  MidFree(address);
 }

+#endif // Z7_LARGE_PAGES
 #endif // _WIN32


@@ -327,9 +312,12 @@ const ISzAlloc g_Alloc = { SzAlloc, SzFree };
 #ifdef _WIN32
 static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p)  return MidAlloc(size); }
 static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p)  MidFree(address); }
+const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree };
+#endif
+
+#if defined(Z7_LARGE_PAGES)
 static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p)  return BigAlloc(size); }
 static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p)  BigFree(address); }
-const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree };
 const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
 #endif

@@ -371,10 +359,16 @@ typedef

 #endif

-#if !defined(_WIN32) \
-    && (defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) \
-        || defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L))
+#ifndef _WIN32
+#include <unistd.h> // for _POSIX_ADVISORY_INFO : for some linux
+#if (defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) \
+        || defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) \
+        || defined(_POSIX_ADVISORY_INFO) && (_POSIX_ADVISORY_INFO >= 200112L) \
+        || defined(__APPLE__) \
+        /* || defined(__linux__) */)
  #define USE_posix_memalign
+  // #pragma message("USE_posix_memalign")
+#endif
 #endif

 #ifndef USE_posix_memalign
@@ -488,6 +482,181 @@ static void SzAlignedFree(ISzAllocPtr pp, void *address)
 #endif
 }

+#ifndef _WIN32
+
+#ifdef Z7_LARGE_PAGES
+
+#if 0 // 1 for debug
+  #include <stdio.h>
+  #include <string.h>  // for strerror()
+  #define PRF(x) x
+#else
+  #define PRF(x)
+#endif
+
+#ifdef USE_posix_memalign
+  /* madvise():
+     glibc <= 2.19 : _BSD_SOURCE
+     glibc  > 2.19 : _DEFAULT_SOURCE
+  */
+  /* && (defined(_DEFAULT_SOURCE) || defined(_BSD_SOURCE)) */
+#if 1 && !defined(Z7_NO_MADVISE) && \
+  (defined(__linux__) || defined(__unix__) || defined(__APPLE__))
+#include <sys/mman.h> // for madvise
+// #pragma message("sys/mman.h")
+#if (defined(MADV_HUGEPAGE) && defined(MADV_NOHUGEPAGE))
+  #define Z7_USE_BIG_ALLOC_MADVISE
+  // #pragma message("Z7_USE_BIG_ALLOC_MADVISE")
+#endif
+#endif
+#endif // USE_posix_memalign
+
+#ifdef Z7_USE_BIG_ALLOC_MADVISE
+#define LARGE_PAGE_SIZE_DEFAULT (1 << 21)
+#else
+#define LARGE_PAGE_SIZE_DEFAULT 0
+#endif
+
+extern
+size_t g_LargePageSize;
+size_t g_LargePageSize = LARGE_PAGE_SIZE_DEFAULT;
+extern
+size_t g_LargePageThresholdMin;
+size_t g_LargePageThresholdMin = LARGE_PAGE_SIZE_DEFAULT / 2;
+extern
+UInt32 g_LargePageFlags;
+UInt32 g_LargePageFlags = 0;
+
+void *BigAlloc(size_t size)
+{
+  if (size == 0)
+    return NULL;
+#ifdef USE_posix_memalign
+  {
+    const size_t pageSize = g_LargePageSize;
+    void *buf = NULL; // on Linux (and other systems), posix_memalign() does not modify memptr on failure (POSIX.1-2008 TC2).
+    PRF(printf("\nBigAlloc 0x%08x=%5uMB", (unsigned)(size), (unsigned)(size >> 20));)
+    if (pageSize && size > g_LargePageThresholdMin)
+    {
+      int res;
+      const size_t mask = pageSize - 1;
+      /* we can allocate aligned size, so data at the end of buffer also will use huge page
+         if (size2 for madvise() is not aligned for huge page size)
+           { Last data block will use small pages. It reduces memory allocation,
+             but last data block with small pages can work slower.
+             It's useful, if we have very large HUGE_PAGE: 32MB or 512MB. }
+      */
+      size_t size2 = (size + mask) & ~mask;
+      if (size2 < size || (size & mask) <= g_LargePageThresholdMin)
+        size2 = size;
+      res = posix_memalign(&buf, pageSize, size2);
+      PRF(printf(" posix_memalign size=0x%08x=%5uMB align=%u",
+          (unsigned)(size2), (unsigned)(size2 >> 20), (unsigned)pageSize);)
+      PRF(printf(" buf=%p", (void *)buf);)
+      if (res == 0)
+      {
+#ifdef Z7_USE_BIG_ALLOC_MADVISE
+        if ((g_LargePageFlags & Z7_LARGE_PAGES_FLAG_NO_MADVISE) == 0)
+        {
+          // Advise the kernel to use huge pages for this memory range
+          // MADV_HUGEPAGE / MADV_NOHUGEPAGE : since Linux 2.6.38
+          // madvise() only operates on whole pages, therefore addr must be page-aligned (4KB/8KB/16KB/64KB).
+          // The value of size is rounded up to a multiple of page size.
+          PRF(printf(" madvise g_LargePageFlags=%x", (unsigned)g_LargePageFlags);)
+          res = madvise(buf, size2, (g_LargePageFlags & Z7_LARGE_PAGES_FLAG_NO_HUGEPAGE) ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
+          if (res)
+          {
+            PRF(printf("\nERROR res=%d, errno=%d=%s\n", res, (int)errno, strerror(errno));)
+            if (g_LargePageFlags & Z7_LARGE_PAGES_FLAG_FAIL_STOP)
+            {
+              free(buf);
+              return NULL;
+            }
+          }
+        }
+#endif // Z7_USE_BIG_ALLOC_MADVISE
+        PRF(printf("\n");)
+        return buf;
+      }
+      PRF(printf("\nERROR res=%d=%s\n", res, strerror(res));)
+      if (g_LargePageFlags & Z7_LARGE_PAGES_FLAG_FAIL_STOP)
+        return NULL;
+      // (res == ENOMEM) "Out of memory" is possible, if pageSize is too big.
+      // so we do second attempt with smaller alignment
+    }
+  }
+#endif // !USE_posix_memalign
+  PRF(printf(" z7_AlignedAlloc size=0x%08x=%5uMB\n", (unsigned)(size), (unsigned)(size >> 20));)
+  return z7_AlignedAlloc(size);
+}
+
+
+void BigFree(void *address)
+{
+  z7_AlignedFree(address);
+}
+#endif // Z7_LARGE_PAGES
+#endif // !_WIN32
+
+
+#ifdef Z7_LARGE_PAGES
+void z7_LargePage_Set(UInt32 flags, size_t pageSize, size_t threshold)
+{
+  g_LargePageFlags = flags;
+
+#ifdef _WIN32
+  if ((flags & Z7_LARGE_PAGES_FLAG_USE_HUGEPAGE) == 0)
+  {
+    g_LargePageSize = 0;
+    g_LargePageThresholdMin = 0;
+  }
+  else
+  {
+    if ((flags & Z7_LARGE_PAGES_FLAG_DIRECT_PAGE_SIZE) == 0)
+    {
+#ifdef Z7_USE_DYN_GetLargePageMinimum
+      Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION
+typedef SIZE_T (WINAPI *Func_GetLargePageMinimum)(VOID);
+      const
+        Func_GetLargePageMinimum fn =
+       (Func_GetLargePageMinimum) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")),
+            "GetLargePageMinimum");
+      if (fn)
+        pageSize = fn();
+      else
+        pageSize = 0;
+#else
+      pageSize = GetLargePageMinimum();
+#endif
+      if (pageSize & (pageSize - 1))
+        pageSize = 0;
+    }
+    g_LargePageSize = pageSize;
+    if ((flags & Z7_LARGE_PAGES_FLAG_DIRECT_THRESHOLD) == 0)
+      threshold = pageSize / 2;
+    g_LargePageThresholdMin = threshold;
+  }
+
+#else // !_WIN32
+
+  if (flags & Z7_LARGE_PAGES_FLAG_NO_PAGECODE)
+  {
+    g_LargePageSize = 0;
+    g_LargePageThresholdMin = 0;
+  }
+  else
+  {
+    if ((flags & Z7_LARGE_PAGES_FLAG_DIRECT_PAGE_SIZE) == 0)
+      pageSize = LARGE_PAGE_SIZE_DEFAULT;
+    g_LargePageSize = pageSize;
+    if ((flags & Z7_LARGE_PAGES_FLAG_DIRECT_THRESHOLD) == 0)
+      threshold = pageSize / 2;
+    g_LargePageThresholdMin = threshold;
+  }
+  // PRF(printf("\ng_LargePageSize=%x g_LargePageThresholdMin = %x g_LargePageFlags = %x", (unsigned)g_LargePageSize, (unsigned)g_LargePageThresholdMin, (unsigned)g_LargePageFlags);)
+#endif // !_WIN32
+}
+#endif // Z7_LARGE_PAGES

 const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree };

@@ -859,7 +859,7 @@ BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; }

 #if defined(__GLIBC__) && (__GLIBC__ * 100 + __GLIBC_MINOR__ >= 216)
  #define Z7_GETAUXV_AVAILABLE
-#else
+#elif !defined(__QNXNTO__)
 // #pragma message("=== is not NEW GLIBC === ")
  #if defined __has_include
  #if __has_include (<sys/auxv.h>)
@@ -877,7 +877,7 @@ BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; }

 #ifdef USE_HWCAP

-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined(__OpenBSD__)
 static unsigned long MY_getauxval(int aux)
 {
  unsigned long val;
@@ -598,7 +598,7 @@ void MatchFinder_Init(void *_p)

 #ifdef MY_CPU_X86_OR_AMD64
  #if defined(__clang__) && (__clang_major__ >= 4) \
-    || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40701)
+    || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900)
    // || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900)

      #define USE_LZFIND_SATUR_SUB_128
@@ -2351,10 +2351,9 @@ static void LzmaEnc_Construct(CLzmaEnc *p)

 CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc)
 {
-  void *p;
-  p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc));
+  CLzmaEncHandle p = (CLzmaEncHandle)ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc));
  if (p)
-    LzmaEnc_Construct((CLzmaEnc *)p);
+    LzmaEnc_Construct(p);
  return p;
 }

@@ -153,6 +153,17 @@ static void PrintProcess_Info()
 #endif
 #endif

+/* if we send (stackSize=0) to CreateThread(), it will
+   use default value PE::SizeOfStackReserve from exe file.
+   PE::SizeOfStackReserve == 1 MiB in exe file with default linker options.
+   Windows aligns specified value to the next 64 KB range. */
+static const unsigned k_StackSize_ReserveSize =
+  #ifdef UNDER_CE
+    1 << 17;
+  #else
+    1 << 20;
+  #endif
+
 WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
 {
  /* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */
@@ -160,12 +171,15 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
  #ifdef USE_THREADS_CreateThread

  DWORD threadId;
-  *p = CreateThread(NULL, 0, func, param, 0, &threadId);
+  *p = CreateThread(NULL, k_StackSize_ReserveSize, func, param, STACK_SIZE_PARAM_IS_A_RESERVATION, &threadId);
  
  #else
+
+#define CALL_beginthreadex(func2, param2, flags, threadIdPtr) \
+    ((HANDLE)(_beginthreadex(NULL, k_StackSize_ReserveSize, func2, param2, (flags) | STACK_SIZE_PARAM_IS_A_RESERVATION, threadIdPtr)))
  
  unsigned threadId;
-  *p = (HANDLE)(_beginthreadex(NULL, 0, func, param, 0, &threadId));
+  *p = CALL_beginthreadex(func, param, 0, &threadId);

 #if 0 // 1 : for debug
  {
@@ -223,7 +237,7 @@ WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param
  HANDLE h;
  WRes wres;
  unsigned threadId;
-  h = (HANDLE)(_beginthreadex(NULL, 0, func, param, CREATE_SUSPENDED, &threadId));
+  h = CALL_beginthreadex(func, param, CREATE_SUSPENDED, &threadId);
  *p = h;
  wres = HandleToWRes(h);
  if (h)
@@ -272,7 +286,7 @@ WRes Thread_Create_With_Group(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, u
  HANDLE h;
  WRes wres;
  unsigned threadId;
-  h = (HANDLE)(_beginthreadex(NULL, 0, func, param, CREATE_SUSPENDED, &threadId));
+  h = CALL_beginthreadex(func, param, CREATE_SUSPENDED, &threadId);
  *p = h;
  wres = HandleToWRes(h);
  if (h)
@@ -279,7 +279,7 @@ SRes Xz_StateCoder_Bc_SetFromMethod_Func(IStateCoder *p, UInt64 id,
    decoder = (CXzBcFilterState *)ISzAlloc_Alloc(alloc, sizeof(CXzBcFilterState));
    if (!decoder)
      return SZ_ERROR_MEM;
-    decoder->buf = ISzAlloc_Alloc(alloc, BRA_BUF_SIZE);
+    decoder->buf = (Byte *)ISzAlloc_Alloc(alloc, BRA_BUF_SIZE);
    if (!decoder->buf)
    {
      ISzAlloc_Free(alloc, decoder);
@@ -1243,7 +1243,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
            UInt32 digest32[XZ_CHECK_SIZE_MAX / 4];
            p->state = XZ_STATE_BLOCK_HEADER;
            p->pos = 0;
-            if (XzCheck_Final(&p->check, (void *)digest32) && memcmp(digest32, p->buf, checkSize) != 0)
+            if (XzCheck_Final(&p->check, (Byte *)(void *)digest32) && memcmp(digest32, p->buf, checkSize) != 0)
              return SZ_ERROR_CRC;
            if (p->decodeOnlyOneBlock)
            {
@@ -1292,7 +1292,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
            p->state = XZ_STATE_STREAM_INDEX_CRC;
            p->indexSize += 4;
            p->pos = 0;
-            Sha256_Final(&p->sha, (void *)digest32);
+            Sha256_Final(&p->sha, (Byte *)(void *)digest32);
            if (memcmp(digest32, p->shaDigest32, SHA256_DIGEST_SIZE) != 0)
              return SZ_ERROR_CRC;
          }
@@ -7,6 +7,7 @@ add_library(vixl
 	include/vixl/aarch64/decoder-aarch64.h
 	include/vixl/aarch64/decoder-constants-aarch64.h
 	include/vixl/aarch64/decoder-visitor-map-aarch64.h
+	include/vixl/aarch64/debugger-aarch64.h
 	include/vixl/aarch64/disasm-aarch64.h
 	include/vixl/aarch64/instructions-aarch64.h
 	include/vixl/aarch64/macro-assembler-aarch64.h
@@ -31,6 +32,7 @@ add_library(vixl
 	src/aarch64/cpu-aarch64.cc
 	src/aarch64/cpu-features-auditor-aarch64.cc
 	src/aarch64/decoder-aarch64.cc
+	src/aarch64/debugger-aarch64.cc
 	src/aarch64/disasm-aarch64.cc
 	src/aarch64/instructions-aarch64.cc
 	src/aarch64/logic-aarch64.cc
@@ -39,6 +41,7 @@ add_library(vixl
 	src/aarch64/operands-aarch64.cc
 	src/aarch64/pointer-auth-aarch64.cc
 	src/aarch64/registers-aarch64.cc
+	src/aarch64/simulator-aarch64.cc
 	src/code-buffer-vixl.cc
 	src/compiler-intrinsics-vixl.cc
 	src/cpu-features.cc
@@ -1,14 +1,16 @@
-VIXL: Armv8 Runtime Code Generation Library, 3.0.0
-==================================================
+VIXL: Arm Runtime Code Generation Library
+=========================================

 Contents:

- * Overview
- * Licence
- * Requirements
- * Known limitations
- * Usage
-
+ * [Overview](#overview)
+ * [Licence](#licence)
+ * [Requirements](#requirements)
+ * [Versioning](#versioning)
+ * [Supported Arm Architecture Features](#supported-arm-architecture-features)
+ * [Known limitations](#known-limitations)
+ * [Bug reports](#bug-reports)
+ * [Usage](#usage)

 Overview
 ========
@@ -16,17 +18,16 @@ Overview
 VIXL contains three components.

 1. Programmatic **assemblers** to generate A64, A32 or T32 code at runtime. The
-    assemblers abstract some of the constraints of each ISA; for example, most
+    assemblers abstract some of the constraints of each ISA; for example, some
    instructions support any immediate.
 2. **Disassemblers** that can print any instruction emitted by the assemblers.
- 3. A **simulator** that can simulate any instruction emitted by the A64
+ 3. An **A64 simulator** that can simulate any instruction emitted by the A64
    assembler. The simulator allows generated code to be run on another
    architecture without the need for a full ISA model.

-The VIXL git repository can be found [on 'https://git.linaro.org'][vixl].
+The VIXL git repository can be found [on GitLab][vixl].

-Changes from previous versions of VIXL can be found in the
-[Changelog](doc/changelog.md).
+ Build status: [![Build Status](https://gitlab.arm.com/runtimes/vixl/badges/main/pipeline.svg)](https://gitlab.arm.com/runtimes/vixl/-/pipelines)


 Licence
@@ -35,32 +36,78 @@ Licence
 This software is covered by the licence described in the [LICENCE](LICENCE)
 file.

+Contributions, as pull requests or via other means, are accepted under the terms
+of the same [LICENCE](LICENCE).

 Requirements
 ============

 To build VIXL the following software is required:

- 1. Python 2.7
+ 1. Python 3.5+
 2. SCons 2.0
- 3. GCC 4.8+ or Clang 3.4+
+ 3. GCC 4.8+ or Clang 4.0+

 A 64-bit host machine is required, implementing an LP64 data model. VIXL has
 been tested using GCC on AArch64 Debian, GCC and Clang on amd64 Ubuntu
 systems.

-To run the linter and code formatting stages of the tests, the following
-software is also required:
+To run the code formatting stages of the tests, the following software is also required:

- 1. Git
- 2. [Google's `cpplint.py`][cpplint]
- 3. clang-format-3.8
+ 1. clang-format 11+
+ 2. clang-tidy 11+

 Refer to the 'Usage' section for details.

+Note that in Ubuntu 18.04, clang-tidy-4.0 will only work if the clang-4.0
+package is also installed.

-Known Limitations for AArch64 code generation
-=============================================
+Versioning
+==========
+
+VIXL uses [Semantic Versioning 2.0.0][semver] - see [VERSIONS](VERSIONS.md) for details.
+
+Supported Arm Architecture Features
+===================================
+
+| Feature    | VIXL CPUFeatures Flag         | Notes                           |
+|------------|-------------------------------|---------------------------------|
+| BTI        | kBTI                          | Per-page enabling not supported |
+| CSSC       | kCSSC                         |                                 |
+| DotProd    | kDotProduct                   |                                 |
+| FCMA       | kFcma                         |                                 |
+| FHM        | kFHM                          |                                 |
+| FP16       | kFPHalf, kNEONHalf            |                                 |
+| FRINTTS    | kFrintToFixedSizedInt         |                                 |
+| FlagM      | kFlagM                        |                                 |
+| FlagM2     | kAXFlag                       |                                 |
+| I8MM       | kI8MM                         |                                 |
+| JSCVT      | kJSCVT                        |                                 |
+| LOR        | kLORegions                    |                                 |
+| LRCPC      | kRCpc                         |                                 |
+| LRCPC2     | kRCpcImm                      |                                 |
+| LSE        | kAtomics                      |                                 |
+| MOPS       | kMOPS                         |                                 |
+| MTE        | kMTEInstructions, kMTE, kMTE3 |                                 |
+| PAuth      | kPAuth, kPAuthGeneric         | Not ERETAA, ERETAB              |
+| RAS        | kRAS                          |                                 |
+| RDM        | kRDM                          |                                 |
+| SVE        | kSVE                          |                                 |
+| SVE2       | kSVE2                         |                                 |
+| SVEBitPerm | kSVEBitPerm                   |                                 |
+| SVEF32MM   | kSVEF32MM                     |                                 |
+| SVEF64MM   | kSVEF64MM                     |                                 |
+| SVEI8MM    | kSVEI8MM                      |                                 |
+
+Enable generating code for an architecture feature by combining a flag with
+the MacroAssembler's defaults. For example, to generate code for SVE, use
+`masm.GetCPUFeatures()->Combine(CPUFeatures::kSVE);`.
+
+See [the cpu features header file](src/cpu-features.h) for more information.
+
+
+Known Limitations
+=================

 VIXL was developed for JavaScript engines so a number of features from A64 were
 deemed unnecessary:
@@ -79,11 +126,6 @@ builds and mostly works for 32-bit x86 platforms, there are a number of
 floating-point operations which do not work correctly, and a number of tests
 fail as a result.

-VIXL may not build using Clang 3.7, due to a compiler warning. A workaround is
-to disable conversion of warnings to errors, or to delete the offending
-`return` statement reported and rebuild. This problem will be fixed in the next
-release.
-
 Debug Builds
 ------------

@@ -125,6 +167,45 @@ Instructions affected by these limitations:
  `stlxrh`, `stlxr`, `ldaxrb`, `ldaxrh`, `ldaxr`, `stlxp`, `ldaxp`, `stlrb`,
  `stlrh`, `stlr`, `ldarb`, `ldarh`, `ldar`, `clrex`.

+Security Considerations
+-----------------------
+
+VIXL allows callers to generate any code they want. The generated code is
+arbitrary, and can therefore call back into any other component in the process.
+As with any self-modifying code, vulnerabilities in the client or in VIXL itself
+could lead to arbitrary code generation.
+
+For performance reasons, VIXL's Assembler only performs debug-mode checking of
+instruction operands (such as immediate field encodability). This can minimise
+code-generation overheads for advanced compilers that already model instructions
+accurately, and might consider the Assembler's checks to be redundant. The
+Assembler should only be used directly where encodability is independently
+checked, and where fine control over all generated code is required.
+
+The MacroAssembler synthesises multiple-instruction sequences to support _some_
+unencodable operand combinations. The MacroAssembler can provide a useful safety
+check in cases where the Assembler's precision is not required; an unexpected
+unencodable operand should result in a macro with the correct behaviour, rather
+than an invalid instruction.
+
+In general, the MacroAssembler handles operands which are likely to vary with
+user-supplied data, but does not usually handle inputs which are likely to be
+easily covered by tests. For example, move-immediate arguments are likely to be
+data-dependent, but register types (e.g. `x` vs `w`) are not.
+
+We recommend that _all_ users use the MacroAssembler, using `ExactAssemblyScope`
+to invoke the Assembler when specific instruction sequences are required. This
+approach is recommended even in cases where a compiler can model the
+instructions precisely, because, subject to the limitations described above, it
+offers an additional layer of protection against logic bugs in instruction
+selection.
+
+Bug reports
+===========
+
+Bug reports may be made in the Issues section of GitLab, or sent to
+vixl@arm.com. Please provide any steps required to recreate a bug, along with
+build environment and host system information.

 Usage
 =====
@@ -137,23 +218,10 @@ with VIXL, in both release and debug mode. It is a useful script for verifying
 that all of VIXL's dependencies are in place and that VIXL is working as it
 should.

-By default, the `tools/test.py` script runs a linter to check that the source
-code conforms with the code style guide, and to detect several common errors
-that the compiler may not warn about. This is most useful for VIXL developers.
-The linter has the following dependencies:
-
- 1. Git must be installed, and the VIXL project must be in a valid Git
-    repository, such as one produced using `git clone`.
- 2. `cpplint.py`, [as provided by Google][cpplint], must be available (and
-    executable) on the `PATH`.
-
-It is possible to tell `tools/test.py` to skip the linter stage by passing
-`--nolint`. This removes the dependency on `cpplint.py` and Git. The `--nolint`
-option is implied if the VIXL project is a snapshot (with no `.git` directory).
-
-Additionally, `tools/test.py` tests code formatting using `clang-format-3.8`.
-If you don't have `clang-format-3.8`, disable the test using the
-`--noclang-format` option.
+By default, `tools/test.py` tests code formatting using `clang-format-4.0`,
+and performs static analysis using `clang-tidy-4.0`. If you don't have these
+tools, disable the test using `--noclang-format` or `--noclang-tidy`,
+respectively.

 Also note that the tests for the tracing features depend upon external `diff`
 and `sed` tools. If these tools are not available in `PATH`, these tests will
@@ -173,11 +241,11 @@ aarch32_examples` or `scons aarch64_examples` from the root directory, or use



-[cpplint]: http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
-           "Google's cpplint.py script."
+[vixl]: https://gitlab.arm.com/runtimes/vixl
+        "The VIXL repository on GitLab."

-[vixl]: https://git.linaro.org/arm/vixl.git
-        "The VIXL repository at 'https://git.linaro.org'."
+[semver]: https://semver.org/spec/v2.0.0.html
+          "Semantic Versioning 2.0.0 Specification"

 [getting-started-aarch32]: doc/aarch32/getting-started-aarch32.md
                           "Introduction to VIXL for AArch32."
@@ -538,12 +538,18 @@ class Assembler : public vixl::internal::AssemblerBase {
  // Conditional branch to label.
  void b(Label* label, Condition cond);

+  // Conditional branch consistent to label.
+  void bc(Label* label, Condition cond);
+
  // Unconditional branch to PC offset.
  void b(int64_t imm26);

  // Conditional branch to PC offset.
  void b(int64_t imm19, Condition cond);

+  // Conditional branch consistent to PC offset.
+  void bc(int64_t imm19, Condition cond);
+
  // Branch with link to label.
  void bl(Label* label);

@@ -2081,6 +2087,9 @@ class Assembler : public vixl::internal::AssemblerBase {
  // Prefetch from pc + imm19 << 2 (allowing unallocated hints).
  void prfm(int op, int64_t imm19);

+  // Yield.
+  void yield();
+
  // Move instructions. The default shift of -1 indicates that the move
  // instruction will calculate an appropriate 16-bit immediate and left shift
  // that is equal to the 64-bit immediate argument. If an explicit left shift
@@ -2450,6 +2459,16 @@ class Assembler : public vixl::internal::AssemblerBase {
  // FP convert to unsigned integer, round towards +infinity.
  void fcvtpu(const VRegister& vd, const VRegister& vn);

+  // Floating-point convert from single-precision to BFloat16 format (scalar).
+  void bfcvt(const VRegister& vd, const VRegister& vn);
+
+  // Floating-point convert from single-precision to BFloat16 format (vector).
+  void bfcvtn(const VRegister& vd, const VRegister& vn);
+
+  // Floating-point convert from single-precision to BFloat16 format (second
+  // part).
+  void bfcvtn2(const VRegister& vd, const VRegister& vn);
+
  // Convert signed integer or fixed point to FP.
  void scvtf(const VRegister& fd, const Register& rn, int fbits = 0);

@@ -3732,6 +3751,12 @@ class Assembler : public vixl::internal::AssemblerBase {
               const VRegister& vm,
               int index);

+  // SM4 Encode.
+  void sm4e(const VRegister& vd, const VRegister& vn);
+
+  // SM4 Key.
+  void sm4ekey(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
  // Scalable Vector Extensions.

  // Absolute value (predicated).
@@ -3839,6 +3864,12 @@ class Assembler : public vixl::internal::AssemblerBase {
            const PRegisterWithLaneSize& pn,
            const PRegisterWithLaneSize& pm);

+  // Floating-point down convert to BFloat16 format (predicated).
+  void bfcvt(const ZRegister& zd, const PRegisterM& pg, const ZRegister& zn);
+
+  // Floating-point down convert and narrow to BFloat16 (top, predicated).
+  void bfcvtnt(const ZRegister& zd, const PRegisterM& pg, const ZRegister& zn);
+
  // Break after first true condition.
  void brka(const PRegisterWithLaneSize& pd,
            const PRegister& pg,
@@ -173,6 +173,7 @@ class AA64ISAR2 : public IDRegister {
  static const Field kRPRES;
  static const Field kMOPS;
  static const Field kCSSC;
+  static const Field kHBC;
 };

 class AA64MMFR0 : public IDRegister {
@@ -111,9 +111,10 @@ class CPUFeaturesAuditor : public DecoderVisitor {
  class RecordInstructionFeaturesScope;

 #define DECLARE(A) virtual void Visit##A(const Instruction* instr);
-  VISITOR_LIST(DECLARE)
+  SIM_AUD_VISITOR_LIST(DECLARE)
 #undef DECLARE
  void VisitCryptoSM3(const Instruction* instr);
+  void VisitCryptoSM4(const Instruction* instr);

  void LoadStoreHelper(const Instruction* instr);
  void LoadStorePairHelper(const Instruction* instr);
@@ -30,240 +30,244 @@
 #include <list>
 #include <map>
 #include <string>
+#include <unordered_map>
+#include <unordered_set>

 #include "../globals-vixl.h"

 #include "instructions-aarch64.h"

 // List macro containing all visitors needed by the decoder class.
-#define VISITOR_LIST_THAT_RETURN(V)                              \
-  V(AddSubExtended)                                              \
-  V(AddSubImmediate)                                             \
-  V(AddSubShifted)                                               \
-  V(AddSubWithCarry)                                             \
-  V(AtomicMemory)                                                \
-  V(Bitfield)                                                    \
-  V(CompareBranch)                                               \
-  V(ConditionalBranch)                                           \
-  V(ConditionalCompareImmediate)                                 \
-  V(ConditionalCompareRegister)                                  \
-  V(ConditionalSelect)                                           \
-  V(Crypto2RegSHA)                                               \
-  V(Crypto3RegSHA)                                               \
-  V(CryptoAES)                                                   \
-  V(DataProcessing1Source)                                       \
-  V(DataProcessing2Source)                                       \
-  V(DataProcessing3Source)                                       \
-  V(EvaluateIntoFlags)                                           \
-  V(Exception)                                                   \
-  V(Extract)                                                     \
-  V(FPCompare)                                                   \
-  V(FPConditionalCompare)                                        \
-  V(FPConditionalSelect)                                         \
-  V(FPDataProcessing1Source)                                     \
-  V(FPDataProcessing2Source)                                     \
-  V(FPDataProcessing3Source)                                     \
-  V(FPFixedPointConvert)                                         \
-  V(FPImmediate)                                                 \
-  V(FPIntegerConvert)                                            \
-  V(LoadLiteral)                                                 \
-  V(LoadStoreExclusive)                                          \
-  V(LoadStorePAC)                                                \
-  V(LoadStorePairNonTemporal)                                    \
-  V(LoadStorePairOffset)                                         \
-  V(LoadStorePairPostIndex)                                      \
-  V(LoadStorePairPreIndex)                                       \
-  V(LoadStorePostIndex)                                          \
-  V(LoadStorePreIndex)                                           \
-  V(LoadStoreRCpcUnscaledOffset)                                 \
-  V(LoadStoreRegisterOffset)                                     \
-  V(LoadStoreUnscaledOffset)                                     \
-  V(LoadStoreUnsignedOffset)                                     \
-  V(LogicalImmediate)                                            \
-  V(LogicalShifted)                                              \
-  V(MoveWideImmediate)                                           \
-  V(NEON2RegMisc)                                                \
-  V(NEON2RegMiscFP16)                                            \
-  V(NEON3Different)                                              \
-  V(NEON3Same)                                                   \
-  V(NEON3SameExtra)                                              \
-  V(NEON3SameFP16)                                               \
-  V(NEONAcrossLanes)                                             \
-  V(NEONByIndexedElement)                                        \
-  V(NEONCopy)                                                    \
-  V(NEONExtract)                                                 \
-  V(NEONLoadStoreMultiStruct)                                    \
-  V(NEONLoadStoreMultiStructPostIndex)                           \
-  V(NEONLoadStoreSingleStruct)                                   \
-  V(NEONLoadStoreSingleStructPostIndex)                          \
-  V(NEONModifiedImmediate)                                       \
-  V(NEONPerm)                                                    \
-  V(NEONScalar2RegMisc)                                          \
+#define VISITOR_LIST_THAT_RETURN(V) \
+  V(SVEBroadcastBitmaskImm)         \
+  V(Unallocated)                    \
+  V(Unimplemented)
+
+#define SIM_AUD_VISITOR_LIST_THAT_RETURN(V)                      \
  V(NEONScalar2RegMiscFP16)                                      \
-  V(NEONScalar3Diff)                                             \
-  V(NEONScalar3Same)                                             \
-  V(NEONScalar3SameExtra)                                        \
  V(NEONScalar3SameFP16)                                         \
-  V(NEONScalarByIndexedElement)                                  \
-  V(NEONScalarCopy)                                              \
-  V(NEONScalarPairwise)                                          \
-  V(NEONScalarShiftImmediate)                                    \
-  V(NEONShiftImmediate)                                          \
-  V(NEONTable)                                                   \
-  V(PCRelAddressing)                                             \
-  V(RotateRightIntoFlags)                                        \
-  V(SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsets)           \
-  V(SVE32BitGatherLoad_VectorPlusImm)                            \
+  V(NEONScalar3SameExtra)                                        \
  V(SVE32BitGatherLoadHalfwords_ScalarPlus32BitScaledOffsets)    \
  V(SVE32BitGatherLoadWords_ScalarPlus32BitScaledOffsets)        \
+  V(SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsets)           \
  V(SVE32BitGatherPrefetch_ScalarPlus32BitScaledOffsets)         \
-  V(SVE32BitGatherPrefetch_VectorPlusImm)                        \
  V(SVE32BitScatterStore_ScalarPlus32BitScaledOffsets)           \
  V(SVE32BitScatterStore_ScalarPlus32BitUnscaledOffsets)         \
-  V(SVE32BitScatterStore_VectorPlusImm)                          \
  V(SVE64BitGatherLoad_ScalarPlus32BitUnpackedScaledOffsets)     \
  V(SVE64BitGatherLoad_ScalarPlus64BitScaledOffsets)             \
  V(SVE64BitGatherLoad_ScalarPlus64BitUnscaledOffsets)           \
  V(SVE64BitGatherLoad_ScalarPlusUnpacked32BitUnscaledOffsets)   \
-  V(SVE64BitGatherLoad_VectorPlusImm)                            \
-  V(SVE64BitGatherPrefetch_ScalarPlus64BitScaledOffsets)         \
-  V(SVE64BitGatherPrefetch_ScalarPlusUnpacked32BitScaledOffsets) \
-  V(SVE64BitGatherPrefetch_VectorPlusImm)                        \
  V(SVE64BitScatterStore_ScalarPlus64BitScaledOffsets)           \
  V(SVE64BitScatterStore_ScalarPlus64BitUnscaledOffsets)         \
  V(SVE64BitScatterStore_ScalarPlusUnpacked32BitScaledOffsets)   \
  V(SVE64BitScatterStore_ScalarPlusUnpacked32BitUnscaledOffsets) \
-  V(SVE64BitScatterStore_VectorPlusImm)                          \
-  V(SVEAddressGeneration)                                        \
-  V(SVEBitwiseLogicalUnpredicated)                               \
-  V(SVEBitwiseShiftUnpredicated)                                 \
+  V(SVEBitwiseLogical_Predicated)                                \
+  V(SVEBitwiseShiftByVector_Predicated)                          \
+  V(SVEConditionallyBroadcastElementToVector)                    \
+  V(SVEConditionallyExtractElementToSIMDFPScalar)                \
+  V(SVEConstructivePrefix_Unpredicated)                          \
+  V(SVEContiguousNonTemporalLoad_ScalarPlusScalar)               \
+  V(SVEContiguousNonTemporalStore_ScalarPlusScalar)              \
+  V(SVEContiguousStore_ScalarPlusScalar)                         \
+  V(SVEExtractElementToSIMDFPScalarRegister)                     \
  V(SVEFFRInitialise)                                            \
  V(SVEFFRWriteFromPredicate)                                    \
-  V(SVEFPAccumulatingReduction)                                  \
-  V(SVEFPArithmeticUnpredicated)                                 \
-  V(SVEFPCompareVectors)                                         \
-  V(SVEFPCompareWithZero)                                        \
-  V(SVEFPComplexAddition)                                        \
-  V(SVEFPComplexMulAdd)                                          \
+  V(SVEFPConvertToInt)                                           \
+  V(SVEInsertSIMDFPScalarRegister)                               \
+  V(SVEIntAddSubtractVectors_Predicated)                         \
+  V(SVEIntMinMaxDifference_Predicated)                           \
+  V(SVEIntMinMaxImm_Unpredicated)                                \
+  V(SVEIntMulImm_Unpredicated)                                   \
+  V(SVEIntMulVectors_Predicated)                                 \
+  V(SVELoadAndBroadcastQOWord_ScalarPlusScalar)                  \
+  V(SVELoadMultipleStructures_ScalarPlusImm)                     \
+  V(SVELoadMultipleStructures_ScalarPlusScalar)                  \
+  V(SVEPartitionBreakCondition)                                  \
+  V(SVEPermutePredicateElements)                                 \
+  V(SVEPredicateFirstActive)                                     \
+  V(SVEPredicateReadFromFFR_Unpredicated)                        \
+  V(SVEPredicateTest)                                            \
+  V(SVEPredicateZero)                                            \
+  V(SVEPropagateBreakToNextPartition)                            \
+  V(SVEStoreMultipleStructures_ScalarPlusImm)                    \
+  V(SVEStoreMultipleStructures_ScalarPlusScalar)                 \
+  V(SVETableLookup)                                              \
+  V(SVEUnpackPredicateElements)                                  \
+  V(SVEVectorSplice)                                             \
  V(SVEFPComplexMulAddIndex)                                     \
-  V(SVEFPFastReduction)                                          \
  V(SVEFPMulIndex)                                               \
-  V(SVEFPMulAdd)                                                 \
  V(SVEFPMulAddIndex)                                            \
-  V(SVEFPUnaryOpUnpredicated)                                    \
  V(SVEIncDecByPredicateCount)                                   \
-  V(SVEIndexGeneration)                                          \
  V(SVEIntArithmeticUnpredicated)                                \
  V(SVEIntCompareSignedImm)                                      \
  V(SVEIntCompareUnsignedImm)                                    \
  V(SVEIntCompareVectors)                                        \
  V(SVEIntMulAddPredicated)                                      \
-  V(SVEIntMulAddUnpredicated)                                    \
-  V(SVEIntReduction)                                             \
-  V(SVEIntUnaryArithmeticPredicated)                             \
  V(SVEMovprfx)                                                  \
-  V(SVEMulIndex)                                                 \
  V(SVEPermuteVectorExtract)                                     \
  V(SVEPermuteVectorInterleaving)                                \
  V(SVEPredicateCount)                                           \
-  V(SVEPredicateLogical)                                         \
+  V(SVEPredicateNextActive)                                      \
+  V(SVEPredicateReadFromFFR_Predicated)                          \
  V(SVEPropagateBreak)                                           \
  V(SVEStackFrameAdjustment)                                     \
  V(SVEStackFrameSize)                                           \
-  V(SVEVectorSelect)                                             \
-  V(SVEBitwiseLogical_Predicated)                                \
-  V(SVEBitwiseLogicalWithImm_Unpredicated)                       \
-  V(SVEBitwiseShiftByImm_Predicated)                             \
-  V(SVEBitwiseShiftByVector_Predicated)                          \
-  V(SVEBitwiseShiftByWideElements_Predicated)                    \
-  V(SVEBroadcastBitmaskImm)                                      \
-  V(SVEBroadcastFPImm_Unpredicated)                              \
-  V(SVEBroadcastGeneralRegister)                                 \
-  V(SVEBroadcastIndexElement)                                    \
-  V(SVEBroadcastIntImm_Unpredicated)                             \
-  V(SVECompressActiveElements)                                   \
-  V(SVEConditionallyBroadcastElementToVector)                    \
-  V(SVEConditionallyExtractElementToSIMDFPScalar)                \
-  V(SVEConditionallyExtractElementToGeneralRegister)             \
-  V(SVEConditionallyTerminateScalars)                            \
-  V(SVEConstructivePrefix_Unpredicated)                          \
-  V(SVEContiguousFirstFaultLoad_ScalarPlusScalar)                \
-  V(SVEContiguousLoad_ScalarPlusImm)                             \
  V(SVEContiguousLoad_ScalarPlusScalar)                          \
-  V(SVEContiguousNonFaultLoad_ScalarPlusImm)                     \
-  V(SVEContiguousNonTemporalLoad_ScalarPlusImm)                  \
-  V(SVEContiguousNonTemporalLoad_ScalarPlusScalar)               \
-  V(SVEContiguousNonTemporalStore_ScalarPlusImm)                 \
-  V(SVEContiguousNonTemporalStore_ScalarPlusScalar)              \
-  V(SVEContiguousPrefetch_ScalarPlusImm)                         \
-  V(SVEContiguousPrefetch_ScalarPlusScalar)                      \
-  V(SVEContiguousStore_ScalarPlusImm)                            \
-  V(SVEContiguousStore_ScalarPlusScalar)                         \
-  V(SVECopySIMDFPScalarRegisterToVector_Predicated)              \
-  V(SVECopyFPImm_Predicated)                                     \
-  V(SVECopyGeneralRegisterToVector_Predicated)                   \
-  V(SVECopyIntImm_Predicated)                                    \
-  V(SVEElementCount)                                             \
-  V(SVEExtractElementToSIMDFPScalarRegister)                     \
-  V(SVEExtractElementToGeneralRegister)                          \
-  V(SVEFPArithmetic_Predicated)                                  \
-  V(SVEFPArithmeticWithImm_Predicated)                           \
-  V(SVEFPConvertPrecision)                                       \
-  V(SVEFPConvertToInt)                                           \
-  V(SVEFPExponentialAccelerator)                                 \
-  V(SVEFPRoundToIntegralValue)                                   \
-  V(SVEFPTrigMulAddCoefficient)                                  \
-  V(SVEFPTrigSelectCoefficient)                                  \
-  V(SVEFPUnaryOp)                                                \
-  V(SVEIncDecRegisterByElementCount)                             \
-  V(SVEIncDecVectorByElementCount)                               \
-  V(SVEInsertSIMDFPScalarRegister)                               \
-  V(SVEInsertGeneralRegister)                                    \
-  V(SVEIntAddSubtractImm_Unpredicated)                           \
-  V(SVEIntAddSubtractVectors_Predicated)                         \
-  V(SVEIntCompareScalarCountAndLimit)                            \
-  V(SVEIntConvertToFP)                                           \
-  V(SVEIntDivideVectors_Predicated)                              \
-  V(SVEIntMinMaxImm_Unpredicated)                                \
-  V(SVEIntMinMaxDifference_Predicated)                           \
-  V(SVEIntMulImm_Unpredicated)                                   \
-  V(SVEIntMulVectors_Predicated)                                 \
-  V(SVELoadAndBroadcastElement)                                  \
-  V(SVELoadAndBroadcastQOWord_ScalarPlusImm)                     \
-  V(SVELoadAndBroadcastQOWord_ScalarPlusScalar)                  \
-  V(SVELoadMultipleStructures_ScalarPlusImm)                     \
-  V(SVELoadMultipleStructures_ScalarPlusScalar)                  \
-  V(SVELoadPredicateRegister)                                    \
-  V(SVELoadVectorRegister)                                       \
-  V(SVEPartitionBreakCondition)                                  \
-  V(SVEPermutePredicateElements)                                 \
-  V(SVEPredicateFirstActive)                                     \
+  V(RotateRightIntoFlags)                                        \
+  V(EvaluateIntoFlags)                                           \
+  V(ConditionalCompareRegister)                                  \
+  V(ConditionalCompareImmediate)                                 \
+  V(PCRelAddressing)                                             \
+  V(UnconditionalBranch)                                         \
+  V(DataProcessing1Source)                                       \
+  V(CompareBranch)                                               \
+  V(TestBranch)                                                  \
+  V(LoadStoreRCpcUnscaledOffset)                                 \
+  V(LoadStoreUnscaledOffset)                                     \
+  V(LoadLiteral)                                                 \
+  V(LoadStorePairNonTemporal)                                    \
+  V(LoadStorePAC)                                                \
+  V(FPCompare)                                                   \
+  V(FPConditionalCompare)                                        \
+  V(FPConditionalSelect)                                         \
+  V(FPDataProcessing2Source)                                     \
+  V(FPDataProcessing3Source)                                     \
+  V(FPIntegerConvert)                                            \
+  V(FPFixedPointConvert)                                         \
+  V(Exception)                                                   \
+  V(Crypto2RegSHA)                                               \
+  V(Crypto3RegSHA)                                               \
+  V(CryptoAES)                                                   \
+  V(NEON2RegMiscFP16)                                            \
+  V(SVEContiguousLoad_ScalarPlusImm)                             \
  V(SVEPredicateInitialize)                                      \
-  V(SVEPredicateNextActive)                                      \
-  V(SVEPredicateReadFromFFR_Predicated)                          \
-  V(SVEPredicateReadFromFFR_Unpredicated)                        \
-  V(SVEPredicateTest)                                            \
-  V(SVEPredicateZero)                                            \
-  V(SVEPropagateBreakToNextPartition)                            \
+  V(SVEIndexGeneration)                                          \
+  V(SVEAddressGeneration)                                        \
+  V(SVELoadAndBroadcastQOWord_ScalarPlusImm)                     \
+  V(SVEIntAddSubtractImm_Unpredicated)                           \
+  V(SVEIntCompareScalarCountAndLimit)                            \
+  V(FPImmediate)                                                 \
+  V(FPDataProcessing1Source)                                     \
+  V(NEONModifiedImmediate)                                       \
+  V(NEONTable)                                                   \
+  V(SVEIntReduction)                                             \
+  V(SVEMulIndex)                                                 \
+  V(NEON3SameFP16)                                               \
+  V(NEONLoadStoreSingleStruct)                                   \
+  V(NEONLoadStoreSingleStructPostIndex)                          \
+  V(NEONLoadStoreMultiStructPostIndex)                           \
+  V(LoadStorePreIndex)                                           \
+  V(LoadStorePostIndex)                                          \
+  V(LoadStoreUnsignedOffset)                                     \
+  V(LoadStoreRegisterOffset)                                     \
+  V(LoadStorePairPostIndex)                                      \
+  V(LoadStorePairOffset)                                         \
+  V(LoadStorePairPreIndex)                                       \
+  V(SVEBitwiseShiftByWideElements_Predicated)                    \
+  V(NEONScalarPairwise)                                          \
+  V(SVEFPUnaryOp)                                                \
+  V(SVEFPRoundToIntegralValue)                                   \
+  V(SVEFPUnaryOpUnpredicated)                                    \
+  V(SVEFPMulAdd)                                                 \
+  V(SVEFPFastReduction)                                          \
+  V(SVEFPComplexMulAdd)                                          \
+  V(SVEFPComplexAddition)                                        \
+  V(SVEFPCompareWithZero)                                        \
+  V(SVEFPCompareVectors)                                         \
+  V(SVEFPArithmeticUnpredicated)                                 \
+  V(SVEFPAccumulatingReduction)                                  \
+  V(SVEIntUnaryArithmeticPredicated)                             \
+  V(SVEBitwiseShiftUnpredicated)                                 \
+  V(SVEUnpackVectorElements)                                     \
+  V(SVEIntConvertToFP)                                           \
+  V(SVEReverseWithinElements)                                    \
  V(SVEReversePredicateElements)                                 \
  V(SVEReverseVectorElements)                                    \
-  V(SVEReverseWithinElements)                                    \
-  V(SVESaturatingIncDecRegisterByElementCount)                   \
-  V(SVESaturatingIncDecVectorByElementCount)                     \
-  V(SVEStoreMultipleStructures_ScalarPlusImm)                    \
-  V(SVEStoreMultipleStructures_ScalarPlusScalar)                 \
-  V(SVEStorePredicateRegister)                                   \
-  V(SVEStoreVectorRegister)                                      \
-  V(SVETableLookup)                                              \
-  V(SVEUnpackPredicateElements)                                  \
-  V(SVEUnpackVectorElements)                                     \
-  V(SVEVectorSplice)                                             \
-  V(System)                                                      \
-  V(TestBranch)                                                  \
-  V(Unallocated)                                                 \
-  V(UnconditionalBranch)                                         \
+  V(SVEInsertGeneralRegister)                                    \
+  V(SVEFPTrigSelectCoefficient)                                  \
+  V(SVEFPTrigMulAddCoefficient)                                  \
+  V(SVEFPExponentialAccelerator)                                 \
+  V(SVEFPArithmetic_Predicated)                                  \
+  V(SVEFPArithmeticWithImm_Predicated)                           \
+  V(SVEBitwiseLogicalWithImm_Unpredicated)                       \
  V(UnconditionalBranchToRegister)                               \
-  V(Unimplemented)
+  V(NEONExtract)                                                 \
+  V(ConditionalSelect)                                           \
+  V(LogicalImmediate)                                            \
+  V(AddSubImmediate)                                             \
+  V(AddSubShifted)                                               \
+  V(AddSubExtended)                                              \
+  V(LogicalShifted)                                              \
+  V(Extract)                                                     \
+  V(DataProcessing2Source)                                       \
+  V(DataProcessing3Source)                                       \
+  V(ConditionalBranch)                                           \
+  V(Bitfield)                                                    \
+  V(AtomicMemory)                                                \
+  V(SVEVectorSelect)                                             \
+  V(SVEPredicateLogical)                                         \
+  V(SVEIntDivideVectors_Predicated)                              \
+  V(SVEBroadcastIntImm_Unpredicated)                             \
+  V(SVEBroadcastFPImm_Unpredicated)                              \
+  V(SVEBroadcastGeneralRegister)                                 \
+  V(SVECompressActiveElements)                                   \
+  V(SVEConditionallyTerminateScalars)                            \
+  V(SVEConditionallyExtractElementToGeneralRegister)             \
+  V(SVEBitwiseShiftByImm_Predicated)                             \
+  V(SVECopyGeneralRegisterToVector_Predicated)                   \
+  V(SVECopyIntImm_Predicated)                                    \
+  V(SVECopySIMDFPScalarRegisterToVector_Predicated)              \
+  V(SVECopyFPImm_Predicated)                                     \
+  V(SVEExtractElementToGeneralRegister)                          \
+  V(SVEIntMulAddUnpredicated)                                    \
+  V(SVEContiguousStore_ScalarPlusImm)                            \
+  V(SVEContiguousPrefetch_ScalarPlusScalar)                      \
+  V(SVEContiguousPrefetch_ScalarPlusImm)                         \
+  V(SVEContiguousNonFaultLoad_ScalarPlusImm)                     \
+  V(SVEBitwiseLogicalUnpredicated)                               \
+  V(SVE64BitGatherPrefetch_ScalarPlus64BitScaledOffsets)         \
+  V(SVE64BitGatherPrefetch_ScalarPlusUnpacked32BitScaledOffsets) \
+  V(SVE64BitGatherPrefetch_VectorPlusImm)                        \
+  V(SVE32BitGatherPrefetch_VectorPlusImm)                        \
+  V(SVELoadVectorRegister)                                       \
+  V(SVEStoreVectorRegister)                                      \
+  V(SVELoadPredicateRegister)                                    \
+  V(SVEStorePredicateRegister)                                   \
+  V(SVEFPConvertPrecision)                                       \
+  V(SVE32BitGatherLoad_VectorPlusImm)                            \
+  V(SVE32BitScatterStore_VectorPlusImm)                          \
+  V(SVE64BitScatterStore_VectorPlusImm)                          \
+  V(SVEContiguousFirstFaultLoad_ScalarPlusScalar)                \
+  V(SVEContiguousNonTemporalLoad_ScalarPlusImm)                  \
+  V(SVEContiguousNonTemporalStore_ScalarPlusImm)                 \
+  V(SVELoadAndBroadcastElement)                                  \
+  V(NEONPerm)                                                    \
+  V(NEONLoadStoreMultiStruct)                                    \
+  V(NEON3Same)                                                   \
+  V(NEON3SameExtra)                                              \
+  V(NEON2RegMisc)                                                \
+  V(NEONByIndexedElement)                                        \
+  V(SVE64BitGatherLoad_VectorPlusImm)                            \
+  V(NEONShiftImmediate)                                          \
+  V(NEONCopy)                                                    \
+  V(NEONScalar2RegMisc)                                          \
+  V(NEONScalar3Diff)                                             \
+  V(NEONScalar3Same)                                             \
+  V(NEONScalarCopy)                                              \
+  V(NEONScalarByIndexedElement)                                  \
+  V(NEONAcrossLanes)                                             \
+  V(NEONScalarShiftImmediate)                                    \
+  V(NEON3Different)                                              \
+  V(MoveWideImmediate)                                           \
+  V(SVEElementCount)                                             \
+  V(SVEIncDecRegisterByElementCount)                             \
+  V(SVEIncDecVectorByElementCount)                               \
+  V(SVESaturatingIncDecVectorByElementCount)                     \
+  V(SVESaturatingIncDecRegisterByElementCount)                   \
+  V(LoadStoreExclusive)                                          \
+  V(SVEBroadcastIndexElement)                                    \
+  V(System)                                                      \
+  V(AddSubWithCarry)

 #define VISITOR_LIST_THAT_DONT_RETURN(V) V(Reserved)

@@ -271,6 +275,10 @@
  VISITOR_LIST_THAT_RETURN(V) \
  VISITOR_LIST_THAT_DONT_RETURN(V)

+#define SIM_AUD_VISITOR_LIST(V) \
+  VISITOR_LIST(V)               \
+  SIM_AUD_VISITOR_LIST_THAT_RETURN(V)
+
 namespace vixl {
 namespace aarch64 {

@@ -311,7 +319,10 @@ class CompiledDecodeNode;
 // handles the instruction.
 class Decoder {
 public:
-  Decoder() { ConstructDecodeGraph(); }
+  Decoder() {
+    ConstructDecodeGraph();
+    PopulatePerInstructionUnallocatedMap(&form_to_unalloc_);
+  }

  // Top-level wrappers around the actual decoding function.
  void Decode(const Instruction* instr);
@@ -392,6 +403,13 @@ class Decoder {

  // Map of node names to DecodeNodes.
  std::map<std::string, DecodeNode> decode_nodes_;
+
+  // Map from instruction form strings to a mask/value of encodings for that
+  // form.
+  using FormToUnallocMap = std::unordered_multimap<uint32_t, uint64_t>;
+  FormToUnallocMap form_to_unalloc_;
+
+  static void PopulatePerInstructionUnallocatedMap(FormToUnallocMap* ftm);
 };

 typedef void (Decoder::*DecodeFnPtr)(const Instruction*);
@@ -471,7 +489,7 @@ class CompiledDecodeNode {
  bool IsLeafNode() const {
    VIXL_ASSERT(((instruction_name_ == "node") && (bit_extract_fn_ != NULL)) ||
                ((instruction_name_ != "node") && (bit_extract_fn_ == NULL)));
-    return instruction_name_ != "node";
+    return bit_extract_fn_ == NULL;
  }

  // Get a pointer to the next node required in the decode process, based on the
@@ -1923,8 +1923,24 @@ static const DecodeMapping kDecodeMapping[] = {
  },

  { "_kjsrkm",
-    {18, 17, 16, 13, 12, 11, 10, 9, 8, 7, 4, 3, 2, 1, 0},
-    { {"000000000011111"_b, "_zztypv"},
+    {11, 10, 9, 8},
+    { {"0000"_b, "_flags"},
+      {"xxx1"_b, "_msrimm"},
+      {"xx1x"_b, "_msrimm"},
+      {"x1xx"_b, "_msrimm"},
+      {"1xxx"_b, "_msrimm"},
+    },
+  },
+
+  { "_msrimm",
+    {13, 12, 4, 3, 2, 1, 0},
+    { {"0011111"_b, "msr_si_pstate"},
+    },
+  },
+
+  { "_flags",
+    {18, 17, 16, 13, 12, 7, 4, 3, 2, 1, 0},
+    { {"00000011111"_b, "_zztypv"},
    },
  },

@@ -117,11 +117,17 @@ class Disassembler : public DecoderVisitor {
  VISITOR_LIST(DECLARE)
 #undef DECLARE

+  std::string GetMnemonicAlias(const Instruction* instr);
+
  using FormToVisitorFnMap = std::unordered_map<
      uint32_t,
      std::function<void(Disassembler*, const Instruction*)>>;
  static const FormToVisitorFnMap* GetFormToVisitorFnMap();

+  using FormToStringMap = std::unordered_map<uint32_t, const char*>;
+  static void PopulateFormToStringMap(FormToStringMap* fts);
+
+  FormToStringMap form_to_string_;
  std::string mnemonic_;
  uint32_t form_hash_;

@@ -132,119 +138,6 @@ class Disassembler : public DecoderVisitor {
    }
  }

-  void Disassemble_PdT_PgZ_ZnT_ZmT(const Instruction* instr);
-  void Disassemble_ZdB_Zn1B_Zn2B_imm(const Instruction* instr);
-  void Disassemble_ZdB_ZnB_ZmB(const Instruction* instr);
-  void Disassemble_ZdD_PgM_ZnS(const Instruction* instr);
-  void Disassemble_ZdD_ZnD_ZmD(const Instruction* instr);
-  void Disassemble_ZdD_ZnD_ZmD_imm(const Instruction* instr);
-  void Disassemble_ZdD_ZnS_ZmS_imm(const Instruction* instr);
-  void Disassemble_ZdH_PgM_ZnS(const Instruction* instr);
-  void Disassemble_ZdH_ZnH_ZmH_imm(const Instruction* instr);
-  void Disassemble_ZdS_PgM_ZnD(const Instruction* instr);
-  void Disassemble_ZdS_PgM_ZnH(const Instruction* instr);
-  void Disassemble_ZdS_PgM_ZnS(const Instruction* instr);
-  void Disassemble_ZdS_ZnH_ZmH_imm(const Instruction* instr);
-  void Disassemble_ZdS_ZnS_ZmS(const Instruction* instr);
-  void Disassemble_ZdS_ZnS_ZmS_imm(const Instruction* instr);
-  void Disassemble_ZdT_PgM_ZnT(const Instruction* instr);
-  void Disassemble_ZdT_PgZ_ZnT_ZmT(const Instruction* instr);
-  void Disassemble_ZdT_Pg_Zn1T_Zn2T(const Instruction* instr);
-  void Disassemble_ZdT_Zn1T_Zn2T_ZmT(const Instruction* instr);
-  void Disassemble_ZdT_ZnT_ZmT(const Instruction* instr);
-  void Disassemble_ZdT_ZnT_ZmTb(const Instruction* instr);
-  void Disassemble_ZdT_ZnTb(const Instruction* instr);
-  void Disassemble_ZdT_ZnTb_ZmTb(const Instruction* instr);
-  void Disassemble_ZdaD_ZnD_ZmD_imm(const Instruction* instr);
-  void Disassemble_ZdaD_ZnH_ZmH_imm_const(const Instruction* instr);
-  void Disassemble_ZdaD_ZnS_ZmS_imm(const Instruction* instr);
-  void Disassemble_ZdaH_ZnH_ZmH_imm(const Instruction* instr);
-  void Disassemble_ZdaH_ZnH_ZmH_imm_const(const Instruction* instr);
-  void Disassemble_ZdaS_ZnB_ZmB_imm_const(const Instruction* instr);
-  void Disassemble_ZdaS_ZnH_ZmH(const Instruction* instr);
-  void Disassemble_ZdaS_ZnH_ZmH_imm(const Instruction* instr);
-  void Disassemble_ZdaS_ZnS_ZmS_imm(const Instruction* instr);
-  void Disassemble_ZdaS_ZnS_ZmS_imm_const(const Instruction* instr);
-  void Disassemble_ZdaT_PgM_ZnTb(const Instruction* instr);
-  void Disassemble_ZdaT_ZnT_ZmT(const Instruction* instr);
-  void Disassemble_ZdaT_ZnT_ZmT_const(const Instruction* instr);
-  void Disassemble_ZdaT_ZnT_const(const Instruction* instr);
-  void Disassemble_ZdaT_ZnTb_ZmTb(const Instruction* instr);
-  void Disassemble_ZdaT_ZnTb_ZmTb_const(const Instruction* instr);
-  void Disassemble_ZdnB_ZdnB(const Instruction* instr);
-  void Disassemble_ZdnB_ZdnB_ZmB(const Instruction* instr);
-  void Disassemble_ZdnS_ZdnS_ZmS(const Instruction* instr);
-  void Disassemble_ZdnT_PgM_ZdnT_ZmT(const Instruction* instr);
-  void Disassemble_ZdnT_PgM_ZdnT_const(const Instruction* instr);
-  void Disassemble_ZdnT_ZdnT_ZmT_const(const Instruction* instr);
-  void Disassemble_ZtD_PgZ_ZnD_Xm(const Instruction* instr);
-  void Disassemble_ZtD_Pg_ZnD_Xm(const Instruction* instr);
-  void Disassemble_ZtS_PgZ_ZnS_Xm(const Instruction* instr);
-  void Disassemble_ZtS_Pg_ZnS_Xm(const Instruction* instr);
-  void Disassemble_ZdaS_ZnB_ZmB(const Instruction* instr);
-  void Disassemble_Vd4S_Vn16B_Vm16B(const Instruction* instr);
-
-  void DisassembleCpy(const Instruction* instr);
-  void DisassembleSet(const Instruction* instr);
-  void DisassembleMinMaxImm(const Instruction* instr);
-
-  void DisassembleSVEShiftLeftImm(const Instruction* instr);
-  void DisassembleSVEShiftRightImm(const Instruction* instr);
-  void DisassembleSVEAddSubCarry(const Instruction* instr);
-  void DisassembleSVEAddSubHigh(const Instruction* instr);
-  void DisassembleSVEComplexIntAddition(const Instruction* instr);
-  void DisassembleSVEBitwiseTernary(const Instruction* instr);
-  void DisassembleSVEFlogb(const Instruction* instr);
-  void DisassembleSVEFPPair(const Instruction* instr);
-
-  void DisassembleNoArgs(const Instruction* instr);
-
-  void DisassembleNEONMulByElementLong(const Instruction* instr);
-  void DisassembleNEONDotProdByElement(const Instruction* instr);
-  void DisassembleNEONFPMulByElement(const Instruction* instr);
-  void DisassembleNEONHalfFPMulByElement(const Instruction* instr);
-  void DisassembleNEONFPMulByElementLong(const Instruction* instr);
-  void DisassembleNEONComplexMulByElement(const Instruction* instr);
-  void DisassembleNEON2RegLogical(const Instruction* instr);
-  void DisassembleNEON2RegExtract(const Instruction* instr);
-  void DisassembleNEON2RegAddlp(const Instruction* instr);
-  void DisassembleNEON2RegCompare(const Instruction* instr);
-  void DisassembleNEON2RegFPCompare(const Instruction* instr);
-  void DisassembleNEON2RegFPConvert(const Instruction* instr);
-  void DisassembleNEON2RegFP(const Instruction* instr);
-  void DisassembleNEON3SameLogical(const Instruction* instr);
-  void DisassembleNEON3SameFHM(const Instruction* instr);
-  void DisassembleNEON3SameNoD(const Instruction* instr);
-  void DisassembleNEONShiftLeftLongImm(const Instruction* instr);
-  void DisassembleNEONShiftRightImm(const Instruction* instr);
-  void DisassembleNEONShiftRightNarrowImm(const Instruction* instr);
-  void DisassembleNEONScalarSatMulLongIndex(const Instruction* instr);
-  void DisassembleNEONFPScalarMulIndex(const Instruction* instr);
-  void DisassembleNEONFPScalar3Same(const Instruction* instr);
-  void DisassembleNEONScalar3SameOnlyD(const Instruction* instr);
-  void DisassembleNEONFPAcrossLanes(const Instruction* instr);
-  void DisassembleNEONFP16AcrossLanes(const Instruction* instr);
-  void DisassembleNEONScalarShiftImmOnlyD(const Instruction* instr);
-  void DisassembleNEONScalarShiftRightNarrowImm(const Instruction* instr);
-  void DisassembleNEONScalar2RegMiscOnlyD(const Instruction* instr);
-  void DisassembleNEONFPScalar2RegMisc(const Instruction* instr);
-  void DisassembleNEONPolynomialMul(const Instruction* instr);
-  void DisassembleNEON4Same(const Instruction* instr);
-  void DisassembleNEONXar(const Instruction* instr);
-  void DisassembleNEONRax1(const Instruction* instr);
-  void DisassembleSHA512(const Instruction* instr);
-
-  void DisassembleMTELoadTag(const Instruction* instr);
-  void DisassembleMTEStoreTag(const Instruction* instr);
-  void DisassembleMTEStoreTagPair(const Instruction* instr);
-
-  void Disassemble_XdSP_XnSP_Xm(const Instruction* instr);
-  void Disassemble_XdSP_XnSP_uimm6_uimm4(const Instruction* instr);
-  void Disassemble_Xd_XnSP_Xm(const Instruction* instr);
-  void Disassemble_Xd_XnSP_XmSP(const Instruction* instr);
-
-  void VisitCryptoSM3(const Instruction* instr);
-
  void Format(const Instruction* instr,
              const char* mnemonic,
              const char* format0,
@@ -253,42 +146,39 @@ class Disassembler : public DecoderVisitor {
                                 const char* format0,
                                 const char* format1 = NULL);

-  void Substitute(const Instruction* instr, const char* string);
+  int Substitute(const Instruction* instr, const char* string);
  int SubstituteField(const Instruction* instr, const char* format);
  int SubstituteRegisterField(const Instruction* instr, const char* format);
  int SubstitutePredicateRegisterField(const Instruction* instr,
                                       const char* format);
  int SubstituteImmediateField(const Instruction* instr, const char* format);
  int SubstituteLiteralField(const Instruction* instr, const char* format);
-  int SubstituteBitfieldImmediateField(const Instruction* instr,
-                                       const char* format);
-  int SubstituteShiftField(const Instruction* instr, const char* format);
-  int SubstituteExtendField(const Instruction* instr, const char* format);
-  int SubstituteConditionField(const Instruction* instr, const char* format);
  int SubstitutePCRelAddressField(const Instruction* instr, const char* format);
  int SubstituteBranchTargetField(const Instruction* instr, const char* format);
  int SubstituteLSRegOffsetField(const Instruction* instr, const char* format);
-  int SubstitutePrefetchField(const Instruction* instr, const char* format);
-  int SubstituteBarrierField(const Instruction* instr, const char* format);
-  int SubstituteSysOpField(const Instruction* instr, const char* format);
-  int SubstituteCrField(const Instruction* instr, const char* format);
  int SubstituteIntField(const Instruction* instr, const char* format);
-  int SubstituteSVESize(const Instruction* instr, const char* format);
+  int SubstituteFPField(const Instruction* instr, const char* format);
  int SubstituteTernary(const Instruction* instr, const char* format);
+  int SubstituteConditionalBlock(const Instruction* instr, const char* format);
+  int SubstituteGenericArray(const Instruction* instr, const char* format);
+  int SubstituteGenericHash(const Instruction* instr, const char* format);
+  int SubstituteExpression(const Instruction* instr, const char* format);
+  int SubstituteEnd(const Instruction* instr, const char* format);

  std::pair<unsigned, unsigned> GetRegNumForField(const Instruction* instr,
                                                  char reg_prefix,
                                                  const char* field);

-  bool RdIsZROrSP(const Instruction* instr) const {
+ public:
+  static bool RdIsZROrSP(const Instruction* instr) {
    return (instr->GetRd() == kZeroRegCode);
  }

-  bool RnIsZROrSP(const Instruction* instr) const {
+  static bool RnIsZROrSP(const Instruction* instr) {
    return (instr->GetRn() == kZeroRegCode);
  }

-  bool RmIsZROrSP(const Instruction* instr) const {
+  static bool RmIsZROrSP(const Instruction* instr) {
    return (instr->GetRm() == kZeroRegCode);
  }

@@ -298,6 +188,7 @@ class Disassembler : public DecoderVisitor {

  bool IsMovzMovnImm(unsigned reg_size, uint64_t value);

+ private:
  int64_t code_address_offset() const { return code_address_offset_; }

 protected:
@@ -373,6 +373,7 @@ class Instruction {

  std::pair<int, int> GetSVEPermuteIndexAndLaneSizeLog2() const;

+  std::pair<int, int> GetNEONMulRmAndIndex() const;
  std::pair<int, int> GetSVEMulZmAndIndex() const;
  std::pair<int, int> GetSVEMulLongZmAndIndex() const;

@@ -855,11 +856,13 @@ class NEONFormatDecoder {
  // Set the format mapping for all or individual substitutions.
  void SetFormatMaps(const NEONFormatMap* format0,
                     const NEONFormatMap* format1 = NULL,
-                     const NEONFormatMap* format2 = NULL) {
+                     const NEONFormatMap* format2 = NULL,
+                     const NEONFormatMap* format3 = NULL) {
    VIXL_ASSERT(format0 != NULL);
    formats_[0] = format0;
    formats_[1] = (format1 == NULL) ? formats_[0] : format1;
    formats_[2] = (format2 == NULL) ? formats_[1] : format2;
+    formats_[3] = (format3 == NULL) ? formats_[2] : format3;
  }
  void SetFormatMap(unsigned index, const NEONFormatMap* format) {
    VIXL_ASSERT(index <= ArrayLength(formats_));
@@ -878,12 +881,15 @@ class NEONFormatDecoder {
  const char* Substitute(const char* string,
                         SubstitutionMode mode0 = kFormat,
                         SubstitutionMode mode1 = kFormat,
-                         SubstitutionMode mode2 = kFormat) {
+                         SubstitutionMode mode2 = kFormat,
+                         SubstitutionMode mode3 = kFormat) {
    const char* subst0 = GetSubstitute(0, mode0);
    const char* subst1 = GetSubstitute(1, mode1);
    const char* subst2 = GetSubstitute(2, mode2);
+    const char* subst3 = GetSubstitute(3, mode3);

-    if ((subst0 == NULL) || (subst1 == NULL) || (subst2 == NULL)) {
+    if ((subst0 == NULL) || (subst1 == NULL) || (subst2 == NULL) ||
+        (subst3 == NULL)) {
      return NULL;
    }

@@ -892,7 +898,8 @@ class NEONFormatDecoder {
             string,
             subst0,
             subst1,
-             subst2);
+             subst2,
+             subst3);
    return form_buffer_;
  }

@@ -1130,7 +1137,7 @@ class NEONFormatDecoder {
  }

  Instr instrbits_;
-  const NEONFormatMap* formats_[3];
+  const NEONFormatMap* formats_[4];
  char form_buffer_[64];
  char mne_buffer_[16];
 };
@@ -1099,11 +1099,24 @@ class MacroAssembler : public Assembler, public MacroAssemblerInterface {
    }
  }

+  void B(Label* label);
  void B(Label* label, BranchType type, Register reg = NoReg, int bit = -1);

-  void B(Label* label);
-  void B(Label* label, Condition cond);
+  void B(Label* label, Condition cond) {
+    Bcommon(label, cond, /* use_bc = */ false);
+  }
+  void Bc(Label* label, Condition cond) {
+    Bcommon(label, cond, /* use_bc = */ true);
+  }
+  // Aliases that match the instruction set ordering.
  void B(Condition cond, Label* label) { B(label, cond); }
+  void Bc(Condition cond, Label* label) { Bc(label, cond); }
+
+ private:
+  // Common method for B and Bc.
+  void Bcommon(Label* label, Condition cond, bool use_bc);
+
+ public:
  void Bfm(const Register& rd,
           const Register& rn,
           unsigned immr,
@@ -1190,6 +1203,31 @@ class MacroAssembler : public Assembler, public MacroAssemblerInterface {
    SingleEmissionCheckScope guard(this);
    retab();
  }
+  void Bfcvt(const VRegister& vd, const VRegister& vn) {
+    VIXL_ASSERT(allow_macro_instructions_);
+    SingleEmissionCheckScope guard(this);
+    bfcvt(vd, vn);
+  }
+  void Bfcvtn(const VRegister& vd, const VRegister& vn) {
+    VIXL_ASSERT(allow_macro_instructions_);
+    SingleEmissionCheckScope guard(this);
+    bfcvtn(vd, vn);
+  }
+  void Bfcvtn2(const VRegister& vd, const VRegister& vn) {
+    VIXL_ASSERT(allow_macro_instructions_);
+    SingleEmissionCheckScope guard(this);
+    bfcvtn2(vd, vn);
+  }
+  void Bfcvt(const ZRegister& zd, const PRegisterM& pg, const ZRegister& zn) {
+    VIXL_ASSERT(allow_macro_instructions_);
+    SingleEmissionCheckScope guard(this);
+    bfcvt(zd, pg, zn);
+  }
+  void Bfcvtnt(const ZRegister& zd, const PRegisterM& pg, const ZRegister& zn) {
+    VIXL_ASSERT(allow_macro_instructions_);
+    SingleEmissionCheckScope guard(this);
+    bfcvtnt(zd, pg, zn);
+  }
  void Braa(const Register& xn, const Register& xm) {
    VIXL_ASSERT(allow_macro_instructions_);
    SingleEmissionCheckScope guard(this);
@@ -2814,6 +2852,7 @@ class MacroAssembler : public Assembler, public MacroAssemblerInterface {
  V(shsub, Shsub)                \
  V(sm3partw1, Sm3partw1)        \
  V(sm3partw2, Sm3partw2)        \
+  V(sm4ekey, Sm4ekey)            \
  V(smax, Smax)                  \
  V(smaxp, Smaxp)                \
  V(smin, Smin)                  \
@@ -2964,6 +3003,7 @@ class MacroAssembler : public Assembler, public MacroAssemblerInterface {
  V(sha1su1, Sha1su1)            \
  V(sha256su0, Sha256su0)        \
  V(sha512su0, Sha512su0)        \
+  V(sm4e, Sm4e)                  \
  V(smaxv, Smaxv)                \
  V(sminv, Sminv)                \
  V(sqabs, Sqabs)                \
@@ -7878,6 +7918,12 @@ class MacroAssembler : public Assembler, public MacroAssemblerInterface {
  void Umax(const Register& rd, const Register& rn, const Operand& op);
  void Umin(const Register& rd, const Register& rn, const Operand& op);

+  void Yield() {
+    VIXL_ASSERT(allow_macro_instructions_);
+    SingleEmissionCheckScope guard(this);
+    yield();
+  }
+
  template <typename T>
  Literal<T>* CreateLiteralDestroyedWithPool(T value) {
    return new Literal<T>(value,
@@ -8249,9 +8295,10 @@ class MacroAssembler : public Assembler, public MacroAssemblerInterface {
      UseScratchRegisterScope* scratch_scope);

  bool LabelIsOutOfRange(Label* label, ImmBranchType branch_type) {
+    int64_t offset = label->GetLocation() - GetCursorOffset();
+    VIXL_ASSERT(IsMultiple(offset, kInstructionSize));
    return !Instruction::IsValidImmPCOffset(branch_type,
-                                            label->GetLocation() -
-                                                GetCursorOffset());
+                                            offset / kInstructionSize);
  }

  void ConfigureSimulatorCPUFeaturesHelper(const CPUFeatures& features,
@@ -643,7 +643,7 @@ class SimVRegister : public SimRegisterBase<kZRegMaxSize> {
 class LogicPRegister {
 public:
  inline LogicPRegister(
-      SimPRegister& other)  // NOLINT(runtime/references)(runtime/explicit)
+      SimPRegister& other)  // NOLINT(google-runtime-references)
      : register_(other) {}

  // Set a conveniently-sized block to 16 bits as the minimum predicate length
@@ -744,7 +744,7 @@ using vixl_uint128_t = std::pair<uint64_t, uint64_t>;
 class LogicVRegister {
 public:
  inline LogicVRegister(
-      SimVRegister& other)  // NOLINT(runtime/references)(runtime/explicit)
+      SimVRegister& other)  // NOLINT(google-runtime-references)
      : register_(other) {
    for (size_t i = 0; i < ArrayLength(saturated_); i++) {
      saturated_[i] = kNotSaturated;
@@ -872,10 +872,9 @@ class LogicVRegister {
      SetUint(vform, index, value.second);
      return;
    }
-    // TODO: Extend this to SVE.
-    VIXL_ASSERT((vform == kFormat1Q) && (index == 0));
-    SetUint(kFormat2D, 0, value.second);
-    SetUint(kFormat2D, 1, value.first);
+    VIXL_ASSERT((vform == kFormat1Q) || (vform == kFormatVnQ));
+    SetUint(kFormatVnD, 2 * index, value.second);
+    SetUint(kFormatVnD, 2 * index + 1, value.first);
  }

  void SetUintArray(VectorFormat vform, const uint64_t* src) const {
@@ -1447,6 +1446,7 @@ class Simulator : public DecoderVisitor {

 #define DECLARE(A) virtual void Visit##A(const Instruction* instr);
  VISITOR_LIST_THAT_RETURN(DECLARE)
+  SIM_AUD_VISITOR_LIST_THAT_RETURN(DECLARE)
 #undef DECLARE
 #define DECLARE(A) \
  VIXL_NO_RETURN virtual void Visit##A(const Instruction* instr);
@@ -1504,6 +1504,7 @@ class Simulator : public DecoderVisitor {
  void SimulateSVESaturatingMulAddHigh(const Instruction* instr);
  void SimulateSVESaturatingMulHighIndex(const Instruction* instr);
  void SimulateSVEFPConvertLong(const Instruction* instr);
+  void SimulateSVEPmull128(const Instruction* instr);
  void SimulateMatrixMul(const Instruction* instr);
  void SimulateSVEFPMatrixMul(const Instruction* instr);
  void SimulateNEONMulByElementLong(const Instruction* instr);
@@ -1532,8 +1533,16 @@ class Simulator : public DecoderVisitor {
  void SimulateSignedMinMax(const Instruction* instr);
  void SimulateUnsignedMinMax(const Instruction* instr);
  void SimulateSHA512(const Instruction* instr);
+  void SimulateFPConvert(const Instruction* instr);
+  void SimulateFPRoundInt(const Instruction* instr);
+  void SimulateFPRoundIntToSize(const Instruction* instr);
+  void SimulateNEONRoundInt(const Instruction* instr);
+  void SimulateNEONRoundIntToSize(const Instruction* instr);
+  void SimulateNEONFPConvert(const Instruction* instr);
+  void SimulateNEONFP2RegMisc(const Instruction* instr);

  void VisitCryptoSM3(const Instruction* instr);
+  void VisitCryptoSM4(const Instruction* instr);

  // Integer register accessors.

@@ -2574,6 +2583,14 @@ class Simulator : public DecoderVisitor {
  void PrintPWrite(int rt_code, uintptr_t address) {
    PrintPAccess(rt_code, "->", address);
  }
+  void PrintWriteU64(uint64_t x, uintptr_t address) {
+    fprintf(stream_,
+            "#      0x%016" PRIx64 " -> %s0x%016" PRIxPTR "%s\n",
+            x,
+            clr_memory_address,
+            address,
+            clr_normal);
+  }

  // Like Print* (above), but respect GetTraceParameters().
  void LogRead(int rt_code, PrintRegisterFormat format, uintptr_t address) {
@@ -2608,6 +2625,9 @@ class Simulator : public DecoderVisitor {
  void LogPWrite(int rt_code, uintptr_t address) {
    if (ShouldTraceWrites()) PrintPWrite(rt_code, address);
  }
+  void LogWriteU64(uint64_t x, uintptr_t address) {
+    if (ShouldTraceWrites()) PrintWriteU64(x, address);
+  }
  void LogMemTransfer(uintptr_t dst, uintptr_t src, uint8_t value) {
    if (ShouldTraceWrites()) PrintMemTransfer(dst, src, value);
  }
@@ -4587,6 +4607,11 @@ class Simulator : public DecoderVisitor {
                        int index,
                        bool is_a);

+  LogicVRegister sm4(LogicVRegister dst,
+                     const LogicVRegister& src1,
+                     const LogicVRegister& src2,
+                     bool is_key);
+
 #define NEON_3VREG_LOGIC_LIST(V) \
  V(addhn)                       \
  V(addhn2)                      \
@@ -4883,6 +4908,12 @@ class Simulator : public DecoderVisitor {
  LogicVRegister fcvtxn2(VectorFormat vform,
                         LogicVRegister dst,
                         const LogicVRegister& src);
+  LogicVRegister bfcvtn(VectorFormat vform,
+                        LogicVRegister dst,
+                        const LogicVRegister& src);
+  LogicVRegister bfcvtn2(VectorFormat vform,
+                         LogicVRegister dst,
+                         const LogicVRegister& src);
  LogicVRegister fsqrt(VectorFormat vform,
                       LogicVRegister dst,
                       const LogicVRegister& src);
@@ -5000,7 +5031,7 @@ class Simulator : public DecoderVisitor {
  uint32_t Crc32Checksum(uint32_t acc, T val, uint32_t poly);
  uint32_t Crc32Checksum(uint32_t acc, uint64_t val, uint32_t poly);

-  void SysOp_W(int op, int64_t val);
+  bool SysOp_W(int op, int64_t val);

  template <typename T>
  T FPRecipSqrtEstimate(T op);
@@ -5343,7 +5374,7 @@ class Simulator : public DecoderVisitor {
                         std::function<void(Simulator*, const Instruction*)>>;
  static const FormToVisitorFnMap* GetFormToVisitorFnMap();

-  uint32_t form_hash_;
+  uint32_t form_hash_{};

  static const PACKey kPACKeyIA;
  static const PACKey kPACKeyIB;
@@ -5421,9 +5452,10 @@ class Simulator : public DecoderVisitor {
  // in vreg is non-zero. Clear the flag, otherwise. This is almost the opposite
  // operation to ExpandToSimVRegister(), except that any non-zero lane is
  // interpreted as true.
-  void ExtractFromSimVRegister(VectorFormat vform,
-                               SimPRegister& pd,  // NOLINT(runtime/references)
-                               SimVRegister vreg);
+  void ExtractFromSimVRegister(
+      VectorFormat vform,
+      SimPRegister& pd,  // NOLINT(google-runtime-references)
+      SimVRegister vreg);

  bool coloured_trace_;

@@ -5450,6 +5482,9 @@ class Simulator : public DecoderVisitor {
  // A configurable size of SVE vector registers.
  unsigned vector_length_;

+  // DC ZVA enable (= 0) status and block size.
+  unsigned dczid_ = (0 << 4) | 4;  // 2^4 words => 64-byte block size.
+
  // Representation of memory attributes such as MTE tagging and BTI page
  // protection in addition to branch interceptions.
  MetaDataDepot meta_data_;
@@ -5462,19 +5497,54 @@ class Simulator : public DecoderVisitor {

  // The Guarded Control Stack is represented using a vector, where the more
  // recently stored addresses are at higher-numbered indices.
-  using GuardedControlStack = std::vector<uint64_t>;
+  using GuardedControlStackStorage = std::vector<uint64_t>;

+ public:
+  struct GuardedControlStack {
+    GuardedControlStackStorage* ptr;
+    uint64_t token;
+  };
+
+ private:
  // The GCSManager handles the synchronisation of GCS across multiple
  // Simulator instances. Each Simulator has its own stack, but all share
  // a GCSManager instance. This allows exchanging stacks between Simulators
  // in a threaded application.
  class GCSManager {
   public:
-    // Allocate a new Guarded Control Stack and add it to the vector of stacks.
+    // Interface for users outside the Simulator.
+
+    // Allocate a new Guarded Control Stack. This method returns a token, which
+    // uniquely identifies the GCS, and can be passed *once* to the stack
+    // switching instructions (GSSS*), when the stack has not been used yet.
+    // Later arguments to the stack switching instructions must either be
+    // fresh tokens, or return values from the stack switching instructions.
    uint64_t AllocateStack() {
+      GuardedControlStack gcs = AllocateStackInternal();
+      return gcs.token;
+    }
+
+    // Free a Guarded Control Stack based on its token.
+    void FreeStack(uint64_t token) {
+      const std::lock_guard<std::mutex> lock(stacks_mtx_);
+      uint64_t gcs_index = GetGCSIndexFromToken(token);
+      GuardedControlStackStorage* gcsptr = stacks_[gcs_index];
+      if (gcsptr == nullptr) {
+        VIXL_ABORT_WITH_MSG("Tried to double free GCS ");
+      } else {
+        delete gcsptr;
+        // To ensure other tokens remain valid, we do not remove this element
+        // but set it to nullptr instead.
+        stacks_[gcs_index] = nullptr;
+      }
+    }
+
+   private:
+    // Allocate a new Guarded Control Stack and add it to the vector of stacks.
+    GuardedControlStack AllocateStackInternal() {
      const std::lock_guard<std::mutex> lock(stacks_mtx_);

-      GuardedControlStack* new_stack = new GuardedControlStack;
+      GuardedControlStackStorage* new_stack = new GuardedControlStackStorage;
      uint64_t result;

      // Put the new stack into the first available slot.
@@ -5490,42 +5560,46 @@ class Simulator : public DecoderVisitor {
        stacks_.push_back(new_stack);
      }

-      // Shift the index to look like a stack pointer aligned to a page.
-      result <<= kPageSizeLog2;
+      result = GetGCSTokenFromIndex(result);

      // Push the tagged index onto the new stack as a seal.
      new_stack->push_back(result + 1);
-      return result;
+
+      return {new_stack, result};
    }

-    // Free a Guarded Control Stack and set the stacks_ slot to null.
-    void FreeStack(uint64_t gcs) {
+    // Get a pointer to the GCS storage using a GCS index.
+    GuardedControlStackStorage* GetGCSPtr(uint64_t gcs_index) {
      const std::lock_guard<std::mutex> lock(stacks_mtx_);
-      uint64_t gcs_index = GetGCSIndex(gcs);
-      GuardedControlStack* gcsptr = stacks_[gcs_index];
-      if (gcsptr == nullptr) {
-        VIXL_ABORT_WITH_MSG("Tried to free unallocated GCS ");
-      } else {
-        delete gcsptr;
-        stacks_[gcs_index] = nullptr;
-      }
+      return stacks_.at(GetGCSIndexFromToken(gcs_index));
    }

-    // Get a pointer to the GCS vector using a GCS id.
-    GuardedControlStack* GetGCSPtr(uint64_t gcs) const {
-      return stacks_[GetGCSIndex(gcs)];
+    // Get an index into stacks_ given a GCS token.
+    static uint64_t GetGCSIndexFromToken(uint64_t token) {
+      return token >> kPageSizeLog2;
    }

-   private:
-    uint64_t GetGCSIndex(uint64_t gcs) const { return gcs >> 12; }
+    // Get a GCS token from an index into stacks_.
+    static uint64_t GetGCSTokenFromIndex(uint64_t index) {
+      // Shift the index to look like a stack pointer aligned to a page.
+      return index << kPageSizeLog2;
+    }

-    std::vector<GuardedControlStack*> stacks_;
+    std::vector<GuardedControlStackStorage*> stacks_;
    std::mutex stacks_mtx_;
+
+    friend class Simulator;
  };

+  GuardedControlStackStorage* GetGCSStorage() { return gcs_.ptr; }
+  uint64_t GetGCSToken() { return gcs_.token; }
+
  // A GCS id indicating no GCS has been allocated.
  static const uint64_t kGCSNoStack = kPageSize - 1;
-  uint64_t gcs_;
+  // We cache both the GCS token, and the pointer to the GCS underlying
+  // storage, which allows us to avoid calls into GCSManager that
+  // would require synchronisation.
+  GuardedControlStack gcs_;
  bool gcs_enabled_;

 public:
@@ -5539,37 +5613,45 @@ class Simulator : public DecoderVisitor {
  bool IsGCSCheckEnabled() const { return gcs_enabled_; }

 private:
-  bool IsAllocatedGCS(uint64_t gcs) const { return gcs != kGCSNoStack; }
  void ResetGCSState() {
    GCSManager& m = GetGCSManager();
-    if (IsAllocatedGCS(gcs_)) {
-      m.FreeStack(gcs_);
+    // This method is also called in the constructor, before we have set up the
+    // GCS, so the call to FreeStack must be conditional.
+    if (GetGCSStorage() != nullptr) {
+      m.FreeStack(GetGCSToken());
    }
-    ActivateGCS(m.AllocateStack());
+    ActivateGCS(m.AllocateStackInternal());
    GCSPop();  // Remove seal.
  }

-  GuardedControlStack* GetGCSPtr(uint64_t gcs) {
+  GuardedControlStackStorage* GetGCSPtr(uint64_t gcs) {
    GCSManager& m = GetGCSManager();
-    GuardedControlStack* result = m.GetGCSPtr(gcs);
+    GuardedControlStackStorage* result = m.GetGCSPtr(gcs);
    return result;
  }
-  GuardedControlStack* GetActiveGCSPtr() { return GetGCSPtr(gcs_); }

-  uint64_t ActivateGCS(uint64_t gcs) {
-    uint64_t outgoing_gcs = gcs_;
-    gcs_ = gcs;
-    return outgoing_gcs;
+  GuardedControlStack ActivateGCS(GuardedControlStack incoming) {
+    GuardedControlStack outgoing = gcs_;
+    gcs_ = incoming;
+    return outgoing;
+  }
+
+  GuardedControlStack ActivateGCS(uint64_t token) {
+    GuardedControlStack incoming = {GetGCSPtr(token), token};
+    GuardedControlStack outgoing = gcs_;
+    gcs_ = incoming;
+    return outgoing;
  }

  void GCSPush(uint64_t addr) {
-    GetActiveGCSPtr()->push_back(addr);
-    size_t entry = GetActiveGCSPtr()->size() - 1;
+    GuardedControlStackStorage* gcs = GetGCSStorage();
+    gcs->push_back(addr);
+    size_t entry = gcs->size() - 1;
    LogGCS(/* is_push = */ true, addr, entry);
  }

  uint64_t GCSPop() {
-    GuardedControlStack* gcs = GetActiveGCSPtr();
+    GuardedControlStackStorage* gcs = GetGCSStorage();
    if (gcs->empty()) {
      return 0;
    }
@@ -5581,7 +5663,7 @@ class Simulator : public DecoderVisitor {
  }

  uint64_t GCSPeek() {
-    GuardedControlStack* gcs = GetActiveGCSPtr();
+    GuardedControlStackStorage* gcs = GetGCSStorage();
    if (gcs->empty()) {
      return 0;
    }
@@ -5590,8 +5672,8 @@ class Simulator : public DecoderVisitor {
  }

  void ReportGCSFailure(const char* msg) {
+    GuardedControlStackStorage* gcs = GetGCSStorage();
    if (IsGCSCheckEnabled()) {
-      GuardedControlStack* gcs = GetActiveGCSPtr();
      printf("%s", msg);
      if (gcs == nullptr) {
        printf("GCS pointer is null\n");
@@ -5604,7 +5686,7 @@ class Simulator : public DecoderVisitor {
            gcs->pop_back();
            int index = most_recent_index - i;
            printf(" gcs%" PRIu64 "[%d]: 0x%016" PRIx64 "\n",
-                   gcs_,
+                   GCSManager::GetGCSIndexFromToken(GetGCSToken()),
                   index,
                   entry);
          }
@@ -202,7 +202,8 @@ namespace vixl {
  V(kEBF16,               "EBF16",                  "ebf16")                   \
  V(kSVE_EBF16,           "EBF16 (SVE)",            "sveebf16")                \
  V(kCSSC,                "CSSC",                   "cssc")                    \
-  V(kGCS,                 "GCS",                    "gcs")
+  V(kGCS,                 "GCS",                    "gcs")                     \
+  V(kHBC,                 "HBC",                    "hbc")
 // clang-format on


@@ -49,7 +49,7 @@ namespace vixl {
 #ifdef __GNUC__
 #define VIXL_HAS_DEPRECATED_WITH_MSG
 #elif defined(__clang__)
-#if __has_extension(attribute_deprecated_with_message)
+#ifdef __has_extension(attribute_deprecated_with_message)
 #define VIXL_HAS_DEPRECATED_WITH_MSG
 #endif
 #endif
@@ -258,9 +258,22 @@ class Float16 {
  uint16_t rawbits_;
 };

-// Floating point representation.
 uint16_t Float16ToRawbits(Float16 value);
+Float16 RawbitsToFloat16(uint16_t bits);

+class BFloat16 {
+ public:
+  explicit BFloat16(float value);
+  BFloat16() : rawbits_(0) {}
+  friend uint16_t BFloat16ToRawbits(BFloat16 value);
+  friend BFloat16 RawbitsToBFloat16(uint16_t bits);
+
+ protected:
+  uint16_t rawbits_;
+};
+
+uint16_t BFloat16ToRawbits(BFloat16 value);
+BFloat16 RawbitsToBFloat16(uint16_t bits);

 uint32_t FloatToRawbits(float value);
 VIXL_DEPRECATED("FloatToRawbits",
@@ -274,8 +287,6 @@ VIXL_DEPRECATED("DoubleToRawbits",
  return DoubleToRawbits(value);
 }

-Float16 RawbitsToFloat16(uint16_t bits);
-
 float RawbitsToFloat(uint32_t bits);
 VIXL_DEPRECATED("RawbitsToFloat",
                inline float rawbits_to_float(uint32_t bits)) {
@@ -629,8 +640,8 @@ bool IsRepeatingPattern(T value) {
  VIXL_ASSERT(IsMultiple(sizeof(value) * kBitsPerByte, BITS));
  VIXL_ASSERT(IsMultiple(BITS, 2));
  VIXL_STATIC_ASSERT(BITS >= 2);
-#if (defined(__x86_64__) || defined(__i386)) && \
-    __clang_major__ >= 17 && __clang_major__ <= 19
+#if (defined(__x86_64__) || defined(__i386)) && __clang_major__ >= 17 && \
+    __clang_major__ <= 19
  // Workaround for https://github.com/llvm/llvm-project/issues/108722
  unsigned hbits = BITS / 2;
  T midmask = (~static_cast<T>(0) >> BITS) << hbits;
@@ -1132,6 +1143,8 @@ const unsigned kFloatMantissaBits = 23;
 const unsigned kFloatExponentBits = 8;
 const unsigned kFloat16MantissaBits = 10;
 const unsigned kFloat16ExponentBits = 5;
+const unsigned kBFloat16MantissaBits = 7;
+const unsigned kBFloat16ExponentBits = kFloatExponentBits;

 enum FPRounding {
  // The first four values are encodable directly by FPCR<RMode>.
@@ -1388,6 +1401,16 @@ static inline float FPRoundToFloat(int64_t sign,
  return RawbitsToFloat(bits);
 }

+// See FPRound for a description of this function.
+static inline BFloat16 FPRoundToBFloat16(int64_t sign,
+                                         int64_t exponent,
+                                         uint64_t mantissa,
+                                         FPRounding round_mode) {
+  return RawbitsToBFloat16(
+      FPRound<uint16_t, kBFloat16ExponentBits, kBFloat16MantissaBits>(
+          sign, exponent, mantissa, round_mode));
+}
+

 float FPToFloat(Float16 value, UseDefaultNaN DN, bool* exception = NULL);
 float FPToFloat(double value,
@@ -1408,6 +1431,16 @@ Float16 FPToFloat16(double value,
                    UseDefaultNaN DN,
                    bool* exception = NULL);

+BFloat16 FPToBFloat16(float value,
+                      FPRounding round_mode,
+                      UseDefaultNaN DN,
+                      bool* exception = NULL);
+
+BFloat16 FPToBFloat16(double value,
+                      FPRounding round_mode,
+                      UseDefaultNaN DN,
+                      bool* exception = NULL);
+
 // Like static_cast<T>(value), but with specialisations for the Float16 type.
 template <typename T, typename F>
 T StaticCastFPTo(F value) {
@@ -1480,7 +1513,7 @@ constexpr uint32_t Hash(const char* str, uint32_t hash = 0) {
  }
 }

-constexpr uint32_t operator"" _h(const char* x, size_t) { return Hash(x); }
+constexpr uint32_t operator""_h(const char* x, size_t) { return Hash(x); }

 }  // namespace vixl

@@ -263,6 +263,10 @@ void Assembler::b(int64_t imm19, Condition cond) {
  Emit(B_cond | ImmCondBranch(imm19) | cond);
 }

+void Assembler::bc(int64_t imm19, Condition cond) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kHBC));
+  Emit(B_cond | ImmCondBranch(imm19) | (1 << 4) | cond);
+}

 void Assembler::b(Label* label) {
  int64_t offset = LinkAndGetInstructionOffsetTo(label);
@@ -277,6 +281,11 @@ void Assembler::b(Label* label, Condition cond) {
  b(static_cast<int>(offset), cond);
 }

+void Assembler::bc(Label* label, Condition cond) {
+  int64_t offset = LinkAndGetInstructionOffsetTo(label);
+  VIXL_ASSERT(Instruction::IsValidImmPCOffset(CondBranchType, offset));
+  bc(static_cast<int>(offset), cond);
+}

 void Assembler::bl(int64_t imm26) { Emit(BL | ImmUncondBranch(imm26)); }

@@ -3689,6 +3698,23 @@ void Assembler::fjcvtzs(const Register& rd, const VRegister& vn) {
  Emit(FJCVTZS | Rn(vn) | Rd(rd));
 }

+void Assembler::bfcvt(const VRegister& vd, const VRegister& vn) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kBF16));
+  VIXL_ASSERT(vd.Is1H() && vn.Is1S());
+  Emit(0x1e634000 | Rn(vn) | Rd(vd));
+}
+
+void Assembler::bfcvtn(const VRegister& vd, const VRegister& vn) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kBF16));
+  VIXL_ASSERT(vn.Is4S() && vd.Is4H());
+  Emit(0x0ea16800 | Rn(vn) | Rd(vd));
+}
+
+void Assembler::bfcvtn2(const VRegister& vd, const VRegister& vn) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kNEON, CPUFeatures::kBF16));
+  VIXL_ASSERT(vn.Is4S() && vd.Is8H());
+  Emit(0x4ea16800 | Rn(vn) | Rd(vd));
+}

 void Assembler::NEONFPConvertToInt(const Register& rd,
                                   const VRegister& vn,
@@ -4527,7 +4553,10 @@ void Assembler::fcmla(const VRegister& vd,
  VIXL_ASSERT(vd.IsVector() && AreSameFormat(vd, vn));
  VIXL_ASSERT((vm.IsH() && (vd.Is8H() || vd.Is4H())) ||
              (vm.IsS() && vd.Is4S()));
-  if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
+  if (vd.IsLaneSizeH()) {
+    VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
+    VIXL_ASSERT(vd.Is8H() || (vm_index <= 1));
+  }
  int index_num_bits = vd.Is4S() ? 1 : 2;
  Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA_byelement |
       ImmNEONHLM(vm_index, index_num_bits) | ImmRotFcmlaSca(rot) | Rn(vn) |
@@ -6117,6 +6146,24 @@ void Assembler::sm3tt2b(const VRegister& vd, const VRegister& vn, const VRegiste
  Emit(0xce408c00 | Rd(vd) | Rn(vn) | Rm(vm) | i);
 }

+void Assembler::sm4e(const VRegister& vd, const VRegister& vn) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
+  VIXL_ASSERT(CPUHas(CPUFeatures::kSM4));
+  VIXL_ASSERT(vd.Is4S() && vn.Is4S());
+
+  Emit(0xcec08400 | Rd(vd) | Rn(vn));
+}
+
+void Assembler::sm4ekey(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kNEON));
+  VIXL_ASSERT(CPUHas(CPUFeatures::kSM4));
+  VIXL_ASSERT(vd.Is4S() && vn.Is4S() && vm.Is4S());
+
+  Emit(0xce60c800 | Rd(vd) | Rn(vn) | Rm(vm));
+}
+
+void Assembler::yield() { hint(YIELD); }
+
 // Note:
 // For all ToImm instructions below, a difference in case
 // for the same letter indicates a negated bit.
@@ -7161,6 +7208,7 @@ bool Assembler::CPUHas(SystemRegister sysreg) const {
      return CPUHas(CPUFeatures::kRNG);
    case FPCR:
    case NZCV:
+    case DCZID_EL0:
      break;
  }
  return true;
@@ -7410,13 +7410,13 @@ void Assembler::pmullb(const ZRegister& zd,
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>

  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
+  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEPmull128) || !zd.IsLaneSizeQ());
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS());
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
-  // SVEPmull128 is not supported
-  VIXL_ASSERT(!zd.IsLaneSizeQ());
+  Instr size = zd.IsLaneSizeQ() ? 0 : SVESize(zd);

-  Emit(0x45006800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
+  Emit(0x45006800 | size | Rd(zd) | Rn(zn) | Rm(zm));
 }

 void Assembler::pmullt(const ZRegister& zd,
@@ -7427,13 +7427,13 @@ void Assembler::pmullt(const ZRegister& zd,
  //  size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0>

  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2));
+  VIXL_ASSERT(CPUHas(CPUFeatures::kSVEPmull128) || !zd.IsLaneSizeQ());
  VIXL_ASSERT(AreSameLaneSize(zn, zm));
  VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS());
  VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2);
-  // SVEPmull128 is not supported
-  VIXL_ASSERT(!zd.IsLaneSizeQ());
+  Instr size = zd.IsLaneSizeQ() ? 0 : SVESize(zd);

-  Emit(0x45006c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm));
+  Emit(0x45006c00 | size | Rd(zd) | Rn(zn) | Rm(zm));
 }

 void Assembler::raddhnb(const ZRegister& zd,
@@ -9895,5 +9895,25 @@ void Assembler::sudot(const ZRegister& zda,
  Emit(0x44a01c00 | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn));
 }

+void Assembler::bfcvt(const ZRegister& zd,
+                      const PRegisterM& pg,
+                      const ZRegister& zn) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
+  VIXL_ASSERT(CPUHas(CPUFeatures::kBF16));
+  VIXL_ASSERT(zd.IsLaneSizeH() && zn.IsLaneSizeS());
+
+  Emit(0x658aa000 | Rd(zd) | PgLow8(pg) | Rn(zn));
+}
+
+void Assembler::bfcvtnt(const ZRegister& zd,
+                        const PRegisterM& pg,
+                        const ZRegister& zn) {
+  VIXL_ASSERT(CPUHas(CPUFeatures::kSVE));
+  VIXL_ASSERT(CPUHas(CPUFeatures::kBF16));
+  VIXL_ASSERT(zd.IsLaneSizeH() && zn.IsLaneSizeS());
+
+  Emit(0x648aa000 | Rd(zd) | PgLow8(pg) | Rn(zn));
+}
+
 }  // namespace aarch64
 }  // namespace vixl
@@ -82,6 +82,7 @@ const IDRegister::Field AA64ISAR1::kI8MM(52);
 const IDRegister::Field AA64ISAR2::kWFXT(0);
 const IDRegister::Field AA64ISAR2::kRPRES(4);
 const IDRegister::Field AA64ISAR2::kMOPS(16);
+const IDRegister::Field AA64ISAR2::kHBC(20);
 const IDRegister::Field AA64ISAR2::kCSSC(52);

 const IDRegister::Field AA64MMFR0::kECV(60);
@@ -198,6 +199,7 @@ CPUFeatures AA64ISAR2::GetCPUFeatures() const {
  if (Get(kWFXT) >= 2) f.Combine(CPUFeatures::kWFXT);
  if (Get(kRPRES) >= 1) f.Combine(CPUFeatures::kRPRES);
  if (Get(kMOPS) >= 1) f.Combine(CPUFeatures::kMOPS);
+  if (Get(kHBC) >= 1) f.Combine(CPUFeatures::kHBC);
  if (Get(kCSSC) >= 1) f.Combine(CPUFeatures::kCSSC);
  return f;
 }
@@ -41,6 +41,111 @@ CPUFeaturesAuditor::GetFormToVisitorFnMap() {
  static const FormToVisitorFnMap form_to_visitor = {
      DEFAULT_FORM_TO_VISITOR_MAP(CPUFeaturesAuditor),
      SIM_AUD_VISITOR_MAP(CPUFeaturesAuditor),
+      {"fcvt_dh_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fcvt_ds_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fcvt_hd_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fcvt_hs_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fcvt_sd_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fcvt_sh_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fmov_d_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fmov_h_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"fmov_s_floatdp1"_h, &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint32x_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint32x_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint32z_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint32z_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint64x_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint64x_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint64z_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint64z_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frinta_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frinta_h_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frinta_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frinti_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frinti_h_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frinti_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintm_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintm_h_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintm_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintn_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintn_h_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintn_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintp_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintp_h_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintp_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintx_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintx_h_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintx_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintz_d_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintz_h_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frintz_s_floatdp1"_h,
+       &CPUFeaturesAuditor::VisitFPDataProcessing1Source},
+      {"frint32x_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frint32z_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frint64x_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frint64z_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frinta_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frinti_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frintm_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frintn_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frintp_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frintx_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frintz_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtas_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtau_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtl_asimdmisc_l"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtms_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtmu_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtns_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtnu_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtn_asimdmisc_n"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtps_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtpu_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtxn_asimdmisc_n"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtzs_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcvtzu_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fabs_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcmeq_asimdmisc_fz"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcmge_asimdmisc_fz"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcmgt_asimdmisc_fz"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcmle_asimdmisc_fz"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fcmlt_asimdmisc_fz"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fneg_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frecpe_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"frsqrte_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"fsqrt_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"scvtf_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"ucvtf_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"urecpe_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
+      {"ursqrte_asimdmisc_r"_h, &CPUFeaturesAuditor::VisitNEON2RegMisc},
      {"fcmla_asimdelem_c_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
      {"fcmla_asimdelem_c_s"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
      {"fmlal2_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
@@ -226,6 +331,9 @@ void CPUFeaturesAuditor::VisitCompareBranch(const Instruction* instr) {
 void CPUFeaturesAuditor::VisitConditionalBranch(const Instruction* instr) {
  RecordInstructionFeaturesScope scope(this);
  USE(instr);
+  if (form_hash_ == "bc_only_condbranch"_h) {
+    scope.Record(CPUFeatures::kHBC);
+  }
 }

 void CPUFeaturesAuditor::VisitConditionalCompareImmediate(
@@ -285,6 +393,12 @@ void CPUFeaturesAuditor::VisitCryptoSM3(const Instruction* instr) {
  USE(instr);
 }

+void CPUFeaturesAuditor::VisitCryptoSM4(const Instruction* instr) {
+  RecordInstructionFeaturesScope scope(this);
+  scope.Record(CPUFeatures::kNEON, CPUFeatures::kSM4);
+  USE(instr);
+}
+
 void CPUFeaturesAuditor::VisitDataProcessing1Source(const Instruction* instr) {
  RecordInstructionFeaturesScope scope(this);
  switch (instr->Mask(DataProcessing1SourceMask)) {
@@ -1876,6 +1990,18 @@ void CPUFeaturesAuditor::Visit(Metadata* metadata, const Instruction* instr) {
         CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kSHA512)},
        {"sha512su1_vvv2_cryptosha512_3"_h,
         CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kSHA512)},
+        {"pmullb_z_zz_q"_h,
+         CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEPmull128)},
+        {"pmullt_z_zz_q"_h,
+         CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEPmull128)},
+        {"bfcvt_bs_floatdp1"_h,
+         CPUFeatures(CPUFeatures::kFP, CPUFeatures::kBF16)},
+        {"bfcvtn_asimdmisc_4s"_h,
+         CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kBF16)},
+        {"bfcvt_z_p_z_s2bf"_h,
+         CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kBF16)},
+        {"bfcvtnt_z_p_z_s2bf"_h,
+         CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kBF16)},
    };

    if (features.count(form_hash_) > 0) {
@@ -0,0 +1,498 @@
+// Copyright 2023, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifdef VIXL_INCLUDE_SIMULATOR_AARCH64
+
+#include "debugger-aarch64.h"
+
+#include <cerrno>
+#include <cmath>
+#include <cstring>
+#include <errno.h>
+#include <limits>
+
+namespace vixl {
+namespace aarch64 {
+
+
+Debugger::Debugger(Simulator* sim)
+    : sim_(sim), input_stream_(&std::cin), ostream_(sim->GetOutputStream()) {
+  // Register all basic debugger commands.
+  RegisterCmd<HelpCmd>();
+  RegisterCmd<BreakCmd>();
+  RegisterCmd<StepCmd>();
+  RegisterCmd<ContinueCmd>();
+  RegisterCmd<PrintCmd>();
+  RegisterCmd<TraceCmd>();
+  RegisterCmd<GdbCmd>();
+}
+
+
+template <class T>
+void Debugger::RegisterCmd() {
+  auto new_command = std::make_unique<T>(sim_);
+
+  // Check that the new command word and alias, don't already exist.
+  std::string_view new_cmd_word = new_command->GetCommandWord();
+  std::string_view new_cmd_alias = new_command->GetCommandAlias();
+  for (const auto& cmd : debugger_cmds_) {
+    std::string_view cmd_word = cmd->GetCommandWord();
+    std::string_view cmd_alias = cmd->GetCommandAlias();
+
+    if (new_cmd_word == cmd_word) {
+      VIXL_ABORT_WITH_MSG("Command word matches an existing command word.");
+    } else if (new_cmd_word == cmd_alias) {
+      VIXL_ABORT_WITH_MSG("Command word matches an existing command alias.");
+    }
+
+    if (new_cmd_alias != "") {
+      if (new_cmd_alias == cmd_word) {
+        VIXL_ABORT_WITH_MSG("Command alias matches an existing command word.");
+      } else if (new_cmd_alias == cmd_alias) {
+        VIXL_ABORT_WITH_MSG("Command alias matches an existing command alias.");
+      }
+    }
+  }
+
+  debugger_cmds_.push_back(std::move(new_command));
+}
+
+
+bool Debugger::IsAtBreakpoint() const {
+  return IsBreakpoint(reinterpret_cast<uint64_t>(sim_->ReadPc()));
+}
+
+
+void Debugger::Debug() {
+  DebugReturn done = DebugContinue;
+  while (done == DebugContinue) {
+    // Disassemble the next instruction to execute.
+    PrintDisassembler print_disasm = PrintDisassembler(ostream_);
+    print_disasm.Disassemble(sim_->ReadPc());
+
+    // Read the command line.
+    fprintf(ostream_, "sim> ");
+    std::string line;
+    std::getline(*input_stream_, line);
+
+    // Remove all control characters from the command string.
+    line.erase(std::remove_if(line.begin(),
+                              line.end(),
+                              [](char c) { return std::iscntrl(c); }),
+               line.end());
+
+    // Assume input from std::cin has already been output (e.g: by a terminal)
+    // but input from elsewhere (e.g: from a testing input stream) has not.
+    if (input_stream_ != &std::cin) {
+      fprintf(ostream_, "%s\n", line.c_str());
+    }
+
+    // Parse the command into tokens.
+    std::vector<std::string> tokenized_cmd = Tokenize(line);
+    if (!tokenized_cmd.empty()) {
+      done = ExecDebugCommand(tokenized_cmd);
+    }
+  }
+}
+
+
+std::optional<uint64_t> Debugger::ParseUint64String(std::string_view uint64_str,
+                                                    int base) {
+  // Clear any previous errors.
+  errno = 0;
+
+  // strtoull uses 0 to indicate that no conversion was possible so first
+  // check that the string isn't zero.
+  if (IsZeroUint64String(uint64_str, base)) {
+    return 0;
+  }
+
+  // Cannot use stoi as it might not be possible to use exceptions.
+  char* end;
+  uint64_t value = std::strtoull(uint64_str.data(), &end, base);
+  if (value == 0 || *end != '\0' || errno == ERANGE) {
+    return std::nullopt;
+  }
+
+  return value;
+}
+
+
+std::optional<Debugger::RegisterParsedFormat> Debugger::ParseRegString(
+    std::string_view reg_str) {
+  // A register should only have 2 (e.g: X0) or 3 (e.g: X31) characters.
+  if (reg_str.size() < 2 || reg_str.size() > 3) {
+    return std::nullopt;
+  }
+
+  // Check for aliases of registers.
+  if (reg_str == "lr") {
+    return {{'X', kLinkRegCode}};
+  } else if (reg_str == "sp") {
+    return {{'X', kSpRegCode}};
+  }
+
+  unsigned max_reg_num;
+  char reg_prefix = std::toupper(reg_str.front());
+  switch (reg_prefix) {
+    case 'W':
+      VIXL_FALLTHROUGH();
+    case 'X':
+      max_reg_num = kNumberOfRegisters - 1;
+      break;
+    case 'V':
+      max_reg_num = kNumberOfVRegisters - 1;
+      break;
+    case 'Z':
+      max_reg_num = kNumberOfZRegisters - 1;
+      break;
+    case 'P':
+      max_reg_num = kNumberOfPRegisters - 1;
+      break;
+    default:
+      return std::nullopt;
+  }
+
+  std::string_view str_code = reg_str.substr(1, reg_str.size());
+  auto reg_code = ParseUint64String(str_code, 10);
+  if (!reg_code) {
+    return std::nullopt;
+  }
+
+  if (*reg_code > max_reg_num) {
+    return std::nullopt;
+  }
+
+  return {{reg_prefix, static_cast<unsigned int>(*reg_code)}};
+}
+
+
+void Debugger::PrintUsage() {
+  for (const auto& cmd : debugger_cmds_) {
+    // Print commands in the following format:
+    //  foo / f
+    //      foo <arg>
+    //      A description of the foo command.
+    //
+
+    std::string_view cmd_word = cmd->GetCommandWord();
+    std::string_view cmd_alias = cmd->GetCommandAlias();
+    if (cmd_alias != "") {
+      fprintf(ostream_, "%s / %s\n", cmd_word.data(), cmd_alias.data());
+    } else {
+      fprintf(ostream_, "%s\n", cmd_word.data());
+    }
+
+    std::string_view args_str = cmd->GetArgsString();
+    if (args_str != "") {
+      fprintf(ostream_, "\t%s %s\n", cmd_word.data(), args_str.data());
+    }
+
+    std::string_view description = cmd->GetDescription();
+    if (description != "") {
+      fprintf(ostream_, "\t%s\n", description.data());
+    }
+  }
+}
+
+
+std::vector<std::string> Debugger::Tokenize(std::string_view input_line,
+                                            char separator) {
+  std::vector<std::string> words;
+
+  if (input_line.empty()) {
+    return words;
+  }
+
+  for (auto separator_pos = input_line.find(separator);
+       separator_pos != input_line.npos;
+       separator_pos = input_line.find(separator)) {
+    // Skip consecutive, repeated separators.
+    if (separator_pos != 0) {
+      words.push_back(std::string{input_line.substr(0, separator_pos)});
+    }
+
+    // Remove characters up to and including the separator.
+    input_line.remove_prefix(separator_pos + 1);
+  }
+
+  // Add the rest of the string to the vector.
+  words.push_back(std::string{input_line});
+
+  return words;
+}
+
+
+DebugReturn Debugger::ExecDebugCommand(
+    const std::vector<std::string>& tokenized_cmd) {
+  std::string cmd_word = tokenized_cmd.front();
+  for (const auto& cmd : debugger_cmds_) {
+    if (cmd_word == cmd->GetCommandWord() ||
+        cmd_word == cmd->GetCommandAlias()) {
+      const std::vector<std::string> args(tokenized_cmd.begin() + 1,
+                                          tokenized_cmd.end());
+
+      // Call the handler for the command and pass the arguments.
+      return cmd->Action(args);
+    }
+  }
+
+  fprintf(ostream_, "Error: command '%s' not found\n", cmd_word.c_str());
+  return DebugContinue;
+}
+
+
+bool Debugger::IsZeroUint64String(std::string_view uint64_str, int base) {
+  // Remove any hex prefixes.
+  if (base == 0 || base == 16) {
+    std::string_view prefix = uint64_str.substr(0, 2);
+    if (prefix == "0x" || prefix == "0X") {
+      uint64_str.remove_prefix(2);
+    }
+  }
+
+  if (uint64_str.empty()) {
+    return false;
+  }
+
+  // Check all remaining digits in the string for anything other than zero.
+  for (char c : uint64_str) {
+    if (c != '0') {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+
+DebuggerCmd::DebuggerCmd(Simulator* sim,
+                         std::string cmd_word,
+                         std::string cmd_alias,
+                         std::string args_str,
+                         std::string description)
+    : sim_(sim),
+      ostream_(sim->GetOutputStream()),
+      command_word_(cmd_word),
+      command_alias_(cmd_alias),
+      args_str_(args_str),
+      description_(description) {}
+
+
+DebugReturn HelpCmd::Action(const std::vector<std::string>& args) {
+  USE(args);
+  sim_->GetDebugger()->PrintUsage();
+  return DebugContinue;
+}
+
+
+DebugReturn BreakCmd::Action(const std::vector<std::string>& args) {
+  if (args.size() != 1) {
+    fprintf(ostream_, "Error: Use `break <address>` to set a breakpoint\n");
+    return DebugContinue;
+  }
+
+  std::string arg = args.front();
+  auto break_addr = Debugger::ParseUint64String(arg);
+  if (!break_addr) {
+    fprintf(ostream_, "Error: Use `break <address>` to set a breakpoint\n");
+    return DebugContinue;
+  }
+
+  if (sim_->GetDebugger()->IsBreakpoint(*break_addr)) {
+    sim_->GetDebugger()->RemoveBreakpoint(*break_addr);
+    fprintf(ostream_,
+            "Breakpoint successfully removed at: 0x%" PRIx64 "\n",
+            *break_addr);
+  } else {
+    sim_->GetDebugger()->RegisterBreakpoint(*break_addr);
+    fprintf(ostream_,
+            "Breakpoint successfully added at: 0x%" PRIx64 "\n",
+            *break_addr);
+  }
+
+  return DebugContinue;
+}
+
+
+DebugReturn StepCmd::Action(const std::vector<std::string>& args) {
+  if (args.size() > 1) {
+    fprintf(ostream_,
+            "Error: use `step [number]` to step an optional number of"
+            " instructions\n");
+    return DebugContinue;
+  }
+
+  // Step 1 instruction by default.
+  std::optional<uint64_t> number_of_instructions_to_execute{1};
+
+  if (args.size() == 1) {
+    // Parse the argument to step that number of instructions.
+    std::string arg = args.front();
+    number_of_instructions_to_execute = Debugger::ParseUint64String(arg);
+    if (!number_of_instructions_to_execute) {
+      fprintf(ostream_,
+              "Error: use `step [number]` to step an optional number of"
+              " instructions\n");
+      return DebugContinue;
+    }
+  }
+
+  while (!sim_->IsSimulationFinished() &&
+         *number_of_instructions_to_execute > 0) {
+    sim_->ExecuteInstruction();
+    (*number_of_instructions_to_execute)--;
+
+    // The first instruction has already been printed by Debug() so only
+    // enable instruction tracing after the first instruction has been
+    // executed.
+    sim_->SetTraceParameters(sim_->GetTraceParameters() | LOG_DISASM);
+  }
+
+  // Disable instruction tracing after all instructions have been executed.
+  sim_->SetTraceParameters(sim_->GetTraceParameters() & ~LOG_DISASM);
+
+  if (sim_->IsSimulationFinished()) {
+    fprintf(ostream_,
+            "Debugger at the end of simulation, leaving simulator...\n");
+    return DebugExit;
+  }
+
+  return DebugContinue;
+}
+
+
+DebugReturn ContinueCmd::Action(const std::vector<std::string>& args) {
+  USE(args);
+
+  fprintf(ostream_, "Continuing...\n");
+
+  if (sim_->GetDebugger()->IsAtBreakpoint()) {
+    // This breakpoint has already been hit, so execute it before continuing.
+    sim_->ExecuteInstruction();
+  }
+
+  return DebugExit;
+}
+
+
+DebugReturn PrintCmd::Action(const std::vector<std::string>& args) {
+  if (args.size() != 1) {
+    fprintf(ostream_,
+            "Error: use `print <register|all>` to print the contents of a"
+            " specific register or all registers.\n");
+    return DebugContinue;
+  }
+
+  if (args.front() == "all") {
+    sim_->PrintRegisters();
+    sim_->PrintZRegisters();
+  } else if (args.front() == "system") {
+    sim_->PrintSystemRegisters();
+  } else if (args.front() == "ffr") {
+    sim_->PrintFFR();
+  } else {
+    auto reg = Debugger::ParseRegString(args.front());
+    if (!reg) {
+      fprintf(ostream_,
+              "Error: incorrect register format, use e.g: X0, x0, etc...\n");
+      return DebugContinue;
+    }
+
+    // Ensure the stack pointer is printed instead of the zero register.
+    if ((*reg).second == kSpRegCode) {
+      (*reg).second = kSPRegInternalCode;
+    }
+
+    // Registers are printed in different ways depending on their type.
+    switch ((*reg).first) {
+      case 'W':
+        sim_->PrintRegister(
+            (*reg).second,
+            static_cast<Simulator::PrintRegisterFormat>(
+                Simulator::PrintRegisterFormat::kPrintWReg |
+                Simulator::PrintRegisterFormat::kPrintRegPartial));
+        break;
+      case 'X':
+        sim_->PrintRegister((*reg).second,
+                            Simulator::PrintRegisterFormat::kPrintXReg);
+        break;
+      case 'V':
+        sim_->PrintVRegister((*reg).second);
+        break;
+      case 'Z':
+        sim_->PrintZRegister((*reg).second);
+        break;
+      case 'P':
+        sim_->PrintPRegister((*reg).second);
+        break;
+      default:
+        // ParseRegString should only allow valid register characters.
+        VIXL_UNREACHABLE();
+    }
+  }
+
+  return DebugContinue;
+}
+
+
+DebugReturn TraceCmd::Action(const std::vector<std::string>& args) {
+  if (args.size() != 0) {
+    fprintf(ostream_, "Error: use `trace` to toggle tracing of registers.\n");
+    return DebugContinue;
+  }
+
+  int trace_params = sim_->GetTraceParameters();
+  if ((trace_params & LOG_ALL) != LOG_ALL) {
+    fprintf(ostream_,
+            "Enabling disassembly, registers and memory write tracing\n");
+    sim_->SetTraceParameters(trace_params | LOG_ALL);
+  } else {
+    fprintf(ostream_,
+            "Disabling disassembly, registers and memory write tracing\n");
+    sim_->SetTraceParameters(trace_params & ~LOG_ALL);
+  }
+
+  return DebugContinue;
+}
+
+
+DebugReturn GdbCmd::Action(const std::vector<std::string>& args) {
+  if (args.size() != 0) {
+    fprintf(ostream_,
+            "Error: use `gdb` to enter GDB from the simulator debugger.\n");
+    return DebugContinue;
+  }
+
+  HostBreakpoint();
+  return DebugContinue;
+}
+
+
+}  // namespace aarch64
+}  // namespace vixl
+
+#endif  // VIXL_INCLUDE_SIMULATOR_AARCH64
@@ -138,11 +138,854 @@ void Decoder::VisitNamedInstruction(const Instruction* instr,
                                    const std::string& name) {
  std::list<DecoderVisitor*>::iterator it;
  Metadata m = {{"form", name}};
+  uint32_t form_hash = Hash(name.c_str());
+
+  // If an encoding is unallocated for this form, add the information to the
+  // metadata.
+  auto range = form_to_unalloc_.equal_range(form_hash);
+  for (auto itu = range.first; itu != range.second; ++itu) {
+    uint32_t mask = itu->second >> 32;
+    uint32_t value = itu->second & 0xffffffff;
+    if (instr->Mask(mask) == value) {
+      m.insert({"unallocated", ""});
+      break;
+    }
+  }
+
  for (it = visitors_.begin(); it != visitors_.end(); it++) {
    (*it)->Visit(&m, instr);
  }
 }

+void Decoder::PopulatePerInstructionUnallocatedMap(FormToUnallocMap* ftm) {
+  using UnallocToFormMap =
+      std::unordered_map<uint64_t, std::unordered_set<uint32_t>>;
+
+  // Map from mask/value (as uint64) to instruction form. Given an encoding,
+  // if, after applying the bitmask (top 32 bits), the resulting encoding equals
+  // bottom 32 bits, then the encoding is unallocated for the instructions
+  // indexed by the mask/value. On object construction, this is used to build a
+  // map from instruction to mask/value, allowing fast lookup during
+  // disassembly.
+  static const UnallocToFormMap forms =
+      {{0x00000001'00000001,
+        {"casp_cp32_ldstexcl"_h,
+         "caspa_cp32_ldstexcl"_h,
+         "caspl_cp32_ldstexcl"_h,
+         "caspal_cp32_ldstexcl"_h,
+         "casp_cp64_ldstexcl"_h,
+         "caspa_cp64_ldstexcl"_h,
+         "caspl_cp64_ldstexcl"_h,
+         "caspal_cp64_ldstexcl"_h}},
+       {0x0000001f'0000001f,
+        {"cpyen_cpy_memcms"_h,   "cpyern_cpy_memcms"_h,  "cpyewn_cpy_memcms"_h,
+         "cpye_cpy_memcms"_h,    "cpyfen_cpy_memcms"_h,  "cpyfern_cpy_memcms"_h,
+         "cpyfewn_cpy_memcms"_h, "cpyfe_cpy_memcms"_h,   "cpyfmn_cpy_memcms"_h,
+         "cpyfmrn_cpy_memcms"_h, "cpyfmwn_cpy_memcms"_h, "cpyfm_cpy_memcms"_h,
+         "cpyfpn_cpy_memcms"_h,  "cpyfprn_cpy_memcms"_h, "cpyfpwn_cpy_memcms"_h,
+         "cpyfp_cpy_memcms"_h,   "cpymn_cpy_memcms"_h,   "cpymrn_cpy_memcms"_h,
+         "cpymwn_cpy_memcms"_h,  "cpym_cpy_memcms"_h,    "cpypn_cpy_memcms"_h,
+         "cpyprn_cpy_memcms"_h,  "cpypwn_cpy_memcms"_h,  "cpyp_cpy_memcms"_h,
+         "seten_set_memcms"_h,   "sete_set_memcms"_h,    "setgen_set_memcms"_h,
+         "setge_set_memcms"_h,   "setgmn_set_memcms"_h,  "setgm_set_memcms"_h,
+         "setgpn_set_memcms"_h,  "setgp_set_memcms"_h,   "setmn_set_memcms"_h,
+         "setm_set_memcms"_h,    "setpn_set_memcms"_h,   "setp_set_memcms"_h}},
+       {0x000003e0'000003e0,
+        {"cpyen_cpy_memcms"_h,   "cpyern_cpy_memcms"_h,  "cpyewn_cpy_memcms"_h,
+         "cpye_cpy_memcms"_h,    "cpyfen_cpy_memcms"_h,  "cpyfern_cpy_memcms"_h,
+         "cpyfewn_cpy_memcms"_h, "cpyfe_cpy_memcms"_h,   "cpyfmn_cpy_memcms"_h,
+         "cpyfmrn_cpy_memcms"_h, "cpyfmwn_cpy_memcms"_h, "cpyfm_cpy_memcms"_h,
+         "cpyfpn_cpy_memcms"_h,  "cpyfprn_cpy_memcms"_h, "cpyfpwn_cpy_memcms"_h,
+         "cpyfp_cpy_memcms"_h,   "cpymn_cpy_memcms"_h,   "cpymrn_cpy_memcms"_h,
+         "cpymwn_cpy_memcms"_h,  "cpym_cpy_memcms"_h,    "cpypn_cpy_memcms"_h,
+         "cpyprn_cpy_memcms"_h,  "cpypwn_cpy_memcms"_h,  "cpyp_cpy_memcms"_h,
+         "seten_set_memcms"_h,   "sete_set_memcms"_h,    "setgen_set_memcms"_h,
+         "setge_set_memcms"_h,   "setgmn_set_memcms"_h,  "setgm_set_memcms"_h,
+         "setgpn_set_memcms"_h,  "setgp_set_memcms"_h,   "setmn_set_memcms"_h,
+         "setm_set_memcms"_h,    "setpn_set_memcms"_h,   "setp_set_memcms"_h}},
+       {0x00001c00'00001400,
+        {"add_32_addsub_ext"_h,
+         "add_64_addsub_ext"_h,
+         "subs_32s_addsub_ext"_h,
+         "subs_64s_addsub_ext"_h,
+         "sub_32_addsub_ext"_h,
+         "sub_64_addsub_ext"_h}},
+       {0x00001800'00001800,
+        {"add_32_addsub_ext"_h,
+         "add_64_addsub_ext"_h,
+         "subs_32s_addsub_ext"_h,
+         "subs_64s_addsub_ext"_h,
+         "sub_32_addsub_ext"_h,
+         "sub_64_addsub_ext"_h}},
+       {0x00010000'00010000,
+        {"casp_cp32_ldstexcl"_h,
+         "caspa_cp32_ldstexcl"_h,
+         "caspl_cp32_ldstexcl"_h,
+         "caspal_cp32_ldstexcl"_h,
+         "casp_cp64_ldstexcl"_h,
+         "caspa_cp64_ldstexcl"_h,
+         "caspl_cp64_ldstexcl"_h,
+         "caspal_cp64_ldstexcl"_h}},
+       {0x000207e0'000007c0, {"and_z_zi"_h, "eor_z_zi"_h, "orr_z_zi"_h}},
+       {0x000207e0'000007e0, {"and_z_zi"_h, "eor_z_zi"_h, "orr_z_zi"_h}},
+       {0x00030000'00000000, {"smov_asimdins_w_w"_h}},
+       {0x00070000'00000000, {"smov_asimdins_x_x"_h, "umov_asimdins_w_w"_h}},
+       {0x000f0000'00000000,
+        {"umov_asimdins_w_w"_h,
+         "umov_asimdins_x_x"_h,
+         "dup_asimdins_dv_v"_h,
+         "dup_asimdins_dr_r"_h,
+         "ins_asimdins_iv_v"_h,
+         "ins_asimdins_ir_r"_h}},
+       {0x001f0000'00000000, {"dup_z_zi"_h}},
+       {0x001f0000'001f0000,
+        {"prfb_i_p_br_s"_h,      "prfd_i_p_br_s"_h,      "prfh_i_p_br_s"_h,
+         "prfw_i_p_br_s"_h,      "cpyen_cpy_memcms"_h,   "cpyern_cpy_memcms"_h,
+         "cpyewn_cpy_memcms"_h,  "cpye_cpy_memcms"_h,    "cpyfen_cpy_memcms"_h,
+         "cpyfern_cpy_memcms"_h, "cpyfewn_cpy_memcms"_h, "cpyfe_cpy_memcms"_h,
+         "cpyfmn_cpy_memcms"_h,  "cpyfmrn_cpy_memcms"_h, "cpyfmwn_cpy_memcms"_h,
+         "cpyfm_cpy_memcms"_h,   "cpyfpn_cpy_memcms"_h,  "cpyfprn_cpy_memcms"_h,
+         "cpyfpwn_cpy_memcms"_h, "cpyfp_cpy_memcms"_h,   "cpymn_cpy_memcms"_h,
+         "cpymrn_cpy_memcms"_h,  "cpymwn_cpy_memcms"_h,  "cpym_cpy_memcms"_h,
+         "cpypn_cpy_memcms"_h,   "cpyprn_cpy_memcms"_h,  "cpypwn_cpy_memcms"_h,
+         "cpyp_cpy_memcms"_h}},
+       {0x0040f800'0000f800,
+        {"ands_32s_log_imm"_h,
+         "ands_64s_log_imm"_h,
+         "and_32_log_imm"_h,
+         "and_64_log_imm"_h,
+         "eor_32_log_imm"_h,
+         "eor_64_log_imm"_h,
+         "orr_32_log_imm"_h,
+         "orr_64_log_imm"_h}},
+       {0x0040fc00'00007c00,
+        {"ands_32s_log_imm"_h,
+         "ands_64s_log_imm"_h,
+         "and_32_log_imm"_h,
+         "and_64_log_imm"_h,
+         "eor_32_log_imm"_h,
+         "eor_64_log_imm"_h,
+         "orr_32_log_imm"_h,
+         "orr_64_log_imm"_h}},
+       {0x0040fc00'0000bc00,
+        {"ands_32s_log_imm"_h,
+         "ands_64s_log_imm"_h,
+         "and_32_log_imm"_h,
+         "and_64_log_imm"_h,
+         "eor_32_log_imm"_h,
+         "eor_64_log_imm"_h,
+         "orr_32_log_imm"_h,
+         "orr_64_log_imm"_h}},
+       {0x0040fc00'0000dc00,
+        {"ands_32s_log_imm"_h,
+         "ands_64s_log_imm"_h,
+         "and_32_log_imm"_h,
+         "and_64_log_imm"_h,
+         "eor_32_log_imm"_h,
+         "eor_64_log_imm"_h,
+         "orr_32_log_imm"_h,
+         "orr_64_log_imm"_h}},
+       {0x0040fc00'0000ec00,
+        {"ands_32s_log_imm"_h,
+         "ands_64s_log_imm"_h,
+         "and_32_log_imm"_h,
+         "and_64_log_imm"_h,
+         "eor_32_log_imm"_h,
+         "eor_64_log_imm"_h,
+         "orr_32_log_imm"_h,
+         "orr_64_log_imm"_h}},
+       {0x0040fc00'0000f400,
+        {"ands_32s_log_imm"_h,
+         "ands_64s_log_imm"_h,
+         "and_32_log_imm"_h,
+         "and_64_log_imm"_h,
+         "eor_32_log_imm"_h,
+         "eor_64_log_imm"_h,
+         "orr_32_log_imm"_h,
+         "orr_64_log_imm"_h}},
+       {0x00200000'00200000, {"fcmla_asimdelem_c_s"_h}},
+       {0x00400000'00000000,
+        {"shl_asisdshf_r"_h,
+         "sli_asisdshf_r"_h,
+         "sri_asisdshf_r"_h,
+         "srshr_asisdshf_r"_h,
+         "srsra_asisdshf_r"_h,
+         "sshr_asisdshf_r"_h,
+         "ssra_asisdshf_r"_h,
+         "urshr_asisdshf_r"_h,
+         "ursra_asisdshf_r"_h,
+         "ushr_asisdshf_r"_h,
+         "usra_asisdshf_r"_h,
+         "pmullb_z_zz"_h,
+         "pmullt_z_zz"_h,
+         "fcvtxn_asisdmisc_n"_h,
+         "fcvtxn_asimdmisc_n"_h}},
+       {0x00400000'00400000, {"urecpe_asimdmisc_r"_h, "ursqrte_asimdmisc_r"_h,
+                              "cnt_asimdmisc_r"_h,    "rev16_asimdmisc_r"_h,
+                              "shrn_asimdshf_n"_h,    "rshrn_asimdshf_n"_h,
+                              "sqshrn_asimdshf_n"_h,  "sqrshrn_asimdshf_n"_h,
+                              "sqshrun_asimdshf_n"_h, "sqrshrun_asimdshf_n"_h,
+                              "uqshrn_asimdshf_n"_h,  "uqrshrn_asimdshf_n"_h,
+                              "sshll_asimdshf_l"_h,   "ushll_asimdshf_l"_h,
+                              "sqrshrn_asisdshf_n"_h, "sqrshrun_asisdshf_n"_h,
+                              "sqshrn_asisdshf_n"_h,  "sqshrun_asisdshf_n"_h,
+                              "uqrshrn_asisdshf_n"_h, "uqshrn_asisdshf_n"_h}},
+       {0x00060000'00000000, {"flogb_z_p_z"_h}},
+       {0x00580000'00000000,
+        {"rshrnb_z_zi"_h,    "rshrnt_z_zi"_h,    "shrnb_z_zi"_h,
+         "shrnt_z_zi"_h,     "sqrshrnb_z_zi"_h,  "sqrshrnt_z_zi"_h,
+         "sqrshrunb_z_zi"_h, "sqrshrunt_z_zi"_h, "sqshrnb_z_zi"_h,
+         "sqshrnt_z_zi"_h,   "sqshrunb_z_zi"_h,  "sqshrunt_z_zi"_h,
+         "uqrshrnb_z_zi"_h,  "uqrshrnt_z_zi"_h,  "uqshrnb_z_zi"_h,
+         "uqshrnt_z_zi"_h,   "sshllb_z_zi"_h,    "sshllt_z_zi"_h,
+         "ushllb_z_zi"_h,    "ushllt_z_zi"_h,    "sqxtnb_z_zz"_h,
+         "sqxtnt_z_zz"_h,    "sqxtunb_z_zz"_h,   "sqxtunt_z_zz"_h,
+         "uqxtnb_z_zz"_h,    "uqxtnt_z_zz"_h}},
+       {0x00580000'00180000,
+        {"sqxtnb_z_zz"_h,
+         "sqxtnt_z_zz"_h,
+         "sqxtunb_z_zz"_h,
+         "sqxtunt_z_zz"_h,
+         "uqxtnb_z_zz"_h,
+         "uqxtnt_z_zz"_h}},
+       {0x00580000'00480000,
+        {"sqxtnb_z_zz"_h,
+         "sqxtnt_z_zz"_h,
+         "sqxtunb_z_zz"_h,
+         "sqxtunt_z_zz"_h,
+         "uqxtnb_z_zz"_h,
+         "uqxtnt_z_zz"_h}},
+       {0x00580000'00500000,
+        {"sqxtnb_z_zz"_h,
+         "sqxtnt_z_zz"_h,
+         "sqxtunb_z_zz"_h,
+         "sqxtunt_z_zz"_h,
+         "uqxtnb_z_zz"_h,
+         "uqxtnt_z_zz"_h}},
+       {0x00580000'00580000,
+        {"sqxtnb_z_zz"_h,
+         "sqxtnt_z_zz"_h,
+         "sqxtunb_z_zz"_h,
+         "sqxtunt_z_zz"_h,
+         "uqxtnb_z_zz"_h,
+         "uqxtnt_z_zz"_h}},
+       {0x00600000'00600000,
+        {"fmla_asimdelem_r_sd"_h,
+         "fmls_asimdelem_r_sd"_h,
+         "fmulx_asimdelem_r_sd"_h,
+         "fmul_asimdelem_r_sd"_h}},
+       {0x00700000'00000000,
+        {"fcvtzs_asisdshf_c"_h,
+         "fcvtzu_asisdshf_c"_h,
+         "scvtf_asisdshf_c"_h,
+         "ucvtf_asisdshf_c"_h}},
+       {0x00780000'00000000,
+        {"sqrshrn_asisdshf_n"_h,
+         "sqrshrun_asisdshf_n"_h,
+         "sqshrn_asisdshf_n"_h,
+         "sqshrun_asisdshf_n"_h,
+         "uqrshrn_asisdshf_n"_h,
+         "uqshrn_asisdshf_n"_h}},
+       {0x00780000'00080000,
+        {"scvtf_asimdshf_c"_h,
+         "ucvtf_asimdshf_c"_h,
+         "fcvtzs_asimdshf_c"_h,
+         "fcvtzu_asimdshf_c"_h}},
+       {0x00800000'00000000,
+        {"compact_z_p_z"_h, "sdot_z_zzz"_h, "udot_z_zzz"_h}},
+       {0x00800000'00800000,
+        {"cnt_asimdmisc_r"_h, "rev16_asimdmisc_r"_h, "rev32_asimdmisc_r"_h}},
+       {0x00c00000'00000000,
+        {"smlalb_z_zzz"_h,
+         "smlalt_z_zzz"_h,
+         "smlslb_z_zzz"_h,
+         "smlslt_z_zzz"_h,
+         "sqdmlalb_z_zzz"_h,
+         "sqdmlalbt_z_zzz"_h,
+         "sqdmlalt_z_zzz"_h,
+         "sqdmlslb_z_zzz"_h,
+         "sqdmlslbt_z_zzz"_h,
+         "sqdmlslt_z_zzz"_h,
+         "umlalb_z_zzz"_h,
+         "umlalt_z_zzz"_h,
+         "umlslb_z_zzz"_h,
+         "umlslt_z_zzz"_h,
+         "faddp_z_p_zz"_h,
+         "fmaxnmp_z_p_zz"_h,
+         "fmaxp_z_p_zz"_h,
+         "fminnmp_z_p_zz"_h,
+         "fminp_z_p_zz"_h,
+         "urecpe_z_p_z"_h,
+         "ursqrte_z_p_z"_h,
+         "saddwb_z_zz"_h,
+         "saddwt_z_zz"_h,
+         "ssubwb_z_zz"_h,
+         "ssubwt_z_zz"_h,
+         "uaddwb_z_zz"_h,
+         "uaddwt_z_zz"_h,
+         "usubwb_z_zz"_h,
+         "usubwt_z_zz"_h,
+         "sadalp_z_p_z"_h,
+         "uadalp_z_p_z"_h,
+         "sabalb_z_zzz"_h,
+         "sabalt_z_zzz"_h,
+         "sabdlb_z_zz"_h,
+         "sabdlt_z_zz"_h,
+         "saddlb_z_zz"_h,
+         "saddlbt_z_zz"_h,
+         "saddlt_z_zz"_h,
+         "smullb_z_zz"_h,
+         "smullt_z_zz"_h,
+         "sqdmullb_z_zz"_h,
+         "sqdmullt_z_zz"_h,
+         "ssublb_z_zz"_h,
+         "ssublbt_z_zz"_h,
+         "ssublt_z_zz"_h,
+         "ssubltb_z_zz"_h,
+         "uabalb_z_zzz"_h,
+         "uabalt_z_zzz"_h,
+         "uabdlb_z_zz"_h,
+         "uabdlt_z_zz"_h,
+         "uaddlb_z_zz"_h,
+         "uaddlt_z_zz"_h,
+         "umullb_z_zz"_h,
+         "umullt_z_zz"_h,
+         "usublb_z_zz"_h,
+         "usublt_z_zz"_h,
+         "addhnb_z_zz"_h,
+         "addhnt_z_zz"_h,
+         "raddhnb_z_zz"_h,
+         "raddhnt_z_zz"_h,
+         "rsubhnb_z_zz"_h,
+         "rsubhnt_z_zz"_h,
+         "subhnb_z_zz"_h,
+         "subhnt_z_zz"_h,
+         "cmeq_asisdsame_only"_h,
+         "cmge_asisdsame_only"_h,
+         "cmgt_asisdsame_only"_h,
+         "cmhi_asisdsame_only"_h,
+         "cmhs_asisdsame_only"_h,
+         "cmtst_asisdsame_only"_h,
+         "add_asisdsame_only"_h,
+         "sub_asisdsame_only"_h,
+         "addp_asisdpair_only"_h,
+         "frinta_z_p_z"_h,
+         "frinti_z_p_z"_h,
+         "frintm_z_p_z"_h,
+         "frintn_z_p_z"_h,
+         "frintp_z_p_z"_h,
+         "frintx_z_p_z"_h,
+         "frintz_z_p_z"_h,
+         "frecpx_z_p_z"_h,
+         "fsqrt_z_p_z"_h,
+         "frecpe_z_z"_h,
+         "frsqrte_z_z"_h,
+         "fmad_z_p_zzz"_h,
+         "fmla_z_p_zzz"_h,
+         "fmls_z_p_zzz"_h,
+         "fmsb_z_p_zzz"_h,
+         "fnmad_z_p_zzz"_h,
+         "fnmla_z_p_zzz"_h,
+         "fnmls_z_p_zzz"_h,
+         "fnmsb_z_p_zzz"_h,
+         "faddv_v_p_z"_h,
+         "fmaxnmv_v_p_z"_h,
+         "fmaxv_v_p_z"_h,
+         "fminnmv_v_p_z"_h,
+         "fminv_v_p_z"_h,
+         "fcmla_z_p_zzz"_h,
+         "fcadd_z_p_zz"_h,
+         "fcmeq_p_p_z0"_h,
+         "fcmge_p_p_z0"_h,
+         "fcmgt_p_p_z0"_h,
+         "fcmle_p_p_z0"_h,
+         "fcmlt_p_p_z0"_h,
+         "fcmne_p_p_z0"_h,
+         "facge_p_p_zz"_h,
+         "facgt_p_p_zz"_h,
+         "fcmeq_p_p_zz"_h,
+         "fcmge_p_p_zz"_h,
+         "fcmgt_p_p_zz"_h,
+         "fcmne_p_p_zz"_h,
+         "fcmuo_p_p_zz"_h,
+         "fadd_z_zz"_h,
+         "fmul_z_zz"_h,
+         "frecps_z_zz"_h,
+         "frsqrts_z_zz"_h,
+         "fsub_z_zz"_h,
+         "ftsmul_z_zz"_h,
+         "fadda_v_p_z"_h,
+         "sxtw_z_p_z"_h,
+         "uxtw_z_p_z"_h,
+         "sxth_z_p_z"_h,
+         "uxth_z_p_z"_h,
+         "sxtb_z_p_z"_h,
+         "uxtb_z_p_z"_h,
+         "fabs_z_p_z"_h,
+         "fneg_z_p_z"_h,
+         "sunpkhi_z_z"_h,
+         "sunpklo_z_z"_h,
+         "uunpkhi_z_z"_h,
+         "uunpklo_z_z"_h,
+         "revb_z_z"_h,
+         "revh_z_z"_h,
+         "revw_z_z"_h,
+         "ftssel_z_zz"_h,
+         "ftmad_z_zzi"_h,
+         "fexpa_z_z"_h,
+         "fabd_z_p_zz"_h,
+         "fadd_z_p_zz"_h,
+         "fdivr_z_p_zz"_h,
+         "fdiv_z_p_zz"_h,
+         "fmaxnm_z_p_zz"_h,
+         "fmax_z_p_zz"_h,
+         "fminnm_z_p_zz"_h,
+         "fmin_z_p_zz"_h,
+         "fmulx_z_p_zz"_h,
+         "fmul_z_p_zz"_h,
+         "fscale_z_p_zz"_h,
+         "fsubr_z_p_zz"_h,
+         "fsub_z_p_zz"_h,
+         "fadd_z_p_zs"_h,
+         "fmaxnm_z_p_zs"_h,
+         "fmax_z_p_zs"_h,
+         "fminnm_z_p_zs"_h,
+         "fmin_z_p_zs"_h,
+         "fmul_z_p_zs"_h,
+         "fsubr_z_p_zs"_h,
+         "fsub_z_p_zs"_h,
+         "abs_asisdmisc_r"_h,
+         "neg_asisdmisc_r"_h,
+         "cmeq_asisdmisc_z"_h,
+         "cmge_asisdmisc_z"_h,
+         "cmgt_asisdmisc_z"_h,
+         "cmle_asisdmisc_z"_h,
+         "cmlt_asisdmisc_z"_h,
+         "cdot_z_zzz"_h,
+         "histcnt_z_p_zz"_h,
+         "sdiv_z_p_zz"_h,
+         "sdivr_z_p_zz"_h,
+         "udiv_z_p_zz"_h,
+         "udivr_z_p_zz"_h,
+         "fdup_z_i"_h,
+         "fcpy_z_p_i"_h,
+         "sqdmulh_asimdsame_only"_h,
+         "sqrdmulh_asimdsame_only"_h,
+         "fcmla_asimdsame2_c"_h,
+         "fcadd_asimdsame2_c"_h,
+         "sqrdmlah_asimdsame2_only"_h,
+         "sqrdmlsh_asimdsame2_only"_h,
+         "sdot_asimdsame2_d"_h,
+         "udot_asimdsame2_d"_h,
+         "mla_asimdelem_r"_h,
+         "mls_asimdelem_r"_h,
+         "mul_asimdelem_r"_h,
+         "sqdmulh_asimdelem_r"_h,
+         "sqrdmlah_asimdelem_r"_h,
+         "sqrdmlsh_asimdelem_r"_h,
+         "sqrdmulh_asimdelem_r"_h,
+         "sqdmlal_asisddiff_only"_h,
+         "sqdmlsl_asisddiff_only"_h,
+         "sqdmull_asisddiff_only"_h,
+         "sqdmulh_asisdsame_only"_h,
+         "sqrdmulh_asisdsame_only"_h,
+         "sqrdmlah_asisdsame2_only"_h,
+         "sqrdmlsh_asisdsame2_only"_h,
+         "srshl_asisdsame_only"_h,
+         "urshl_asisdsame_only"_h,
+         "sshl_asisdsame_only"_h,
+         "ushl_asisdsame_only"_h,
+         "sqdmulh_asisdelem_r"_h,
+         "sqrdmlah_asisdelem_r"_h,
+         "sqrdmlsh_asisdelem_r"_h,
+         "sqrdmulh_asisdelem_r"_h,
+         "sqdmlal_asisdelem_l"_h,
+         "sqdmlsl_asisdelem_l"_h,
+         "sqdmull_asisdelem_l"_h,
+         "smlal_asimdelem_l"_h,
+         "smlsl_asimdelem_l"_h,
+         "smull_asimdelem_l"_h,
+         "umlal_asimdelem_l"_h,
+         "umlsl_asimdelem_l"_h,
+         "umull_asimdelem_l"_h,
+         "sqdmull_asimdelem_l"_h,
+         "sqdmlal_asimdelem_l"_h,
+         "sqdmlsl_asimdelem_l"_h,
+         "sqdmlal_asimddiff_l"_h,
+         "sqdmlsl_asimddiff_l"_h,
+         "sqdmull_asimddiff_l"_h}},
+       {0x00c00300'00000000,
+        {"asr_z_p_zi"_h,
+         "asrd_z_p_zi"_h,
+         "lsl_z_p_zi"_h,
+         "lsr_z_p_zi"_h,
+         "sqshl_z_p_zi"_h,
+         "sqshlu_z_p_zi"_h,
+         "srshr_z_p_zi"_h,
+         "uqshl_z_p_zi"_h,
+         "urshr_z_p_zi"_h}},
+       {0x00c00000'00400000,
+        {"urecpe_z_p_z"_h,
+         "ursqrte_z_p_z"_h,
+         "histseg_z_zz"_h,
+         "pmul_z_zz"_h,
+         "cmeq_asisdsame_only"_h,
+         "cmge_asisdsame_only"_h,
+         "cmgt_asisdsame_only"_h,
+         "cmhi_asisdsame_only"_h,
+         "cmhs_asisdsame_only"_h,
+         "cmtst_asisdsame_only"_h,
+         "add_asisdsame_only"_h,
+         "sub_asisdsame_only"_h,
+         "addp_asisdpair_only"_h,
+         "sxtw_z_p_z"_h,
+         "uxtw_z_p_z"_h,
+         "sxth_z_p_z"_h,
+         "uxth_z_p_z"_h,
+         "revh_z_z"_h,
+         "revw_z_z"_h,
+         "pmul_asimdsame_only"_h,
+         "abs_asisdmisc_r"_h,
+         "neg_asisdmisc_r"_h,
+         "cmeq_asisdmisc_z"_h,
+         "cmge_asisdmisc_z"_h,
+         "cmgt_asisdmisc_z"_h,
+         "cmle_asisdmisc_z"_h,
+         "cmlt_asisdmisc_z"_h,
+         "cdot_z_zzz"_h,
+         "histcnt_z_p_zz"_h,
+         "pmull_asimddiff_l"_h,
+         "sdot_asimdsame2_d"_h,
+         "udot_asimdsame2_d"_h,
+         "srshl_asisdsame_only"_h,
+         "urshl_asisdsame_only"_h,
+         "sshl_asisdsame_only"_h,
+         "ushl_asisdsame_only"_h}},
+       {0x00c00000'00800000,
+        {"histseg_z_zz"_h,         "pmul_z_zz"_h,
+         "cmeq_asisdsame_only"_h,  "cmge_asisdsame_only"_h,
+         "cmgt_asisdsame_only"_h,  "cmhi_asisdsame_only"_h,
+         "cmhs_asisdsame_only"_h,  "cmtst_asisdsame_only"_h,
+         "add_asisdsame_only"_h,   "sub_asisdsame_only"_h,
+         "addp_asisdpair_only"_h,  "sxtw_z_p_z"_h,
+         "uxtw_z_p_z"_h,           "revw_z_z"_h,
+         "pmul_asimdsame_only"_h,  "abs_asisdmisc_r"_h,
+         "neg_asisdmisc_r"_h,      "cmeq_asisdmisc_z"_h,
+         "cmge_asisdmisc_z"_h,     "cmgt_asisdmisc_z"_h,
+         "cmle_asisdmisc_z"_h,     "cmlt_asisdmisc_z"_h,
+         "match_p_p_zz"_h,         "nmatch_p_p_zz"_h,
+         "pmull_asimddiff_l"_h,    "srshl_asisdsame_only"_h,
+         "urshl_asisdsame_only"_h, "sshl_asisdsame_only"_h,
+         "ushl_asisdsame_only"_h}},
+       {0x00c00000'00c00000,
+        {"asr_z_p_zw"_h,
+         "lsl_z_p_zw"_h,
+         "lsr_z_p_zw"_h,
+         "urecpe_z_p_z"_h,
+         "ursqrte_z_p_z"_h,
+         "histseg_z_zz"_h,
+         "pmul_z_zz"_h,
+         "asr_z_zw"_h,
+         "lsl_z_zw"_h,
+         "lsr_z_zw"_h,
+         "pmul_asimdsame_only"_h,
+         "match_p_p_zz"_h,
+         "nmatch_p_p_zz"_h,
+         "adds_32_addsub_shift"_h,
+         "adds_64_addsub_shift"_h,
+         "add_32_addsub_shift"_h,
+         "add_64_addsub_shift"_h,
+         "subs_32_addsub_shift"_h,
+         "subs_64_addsub_shift"_h,
+         "sub_32_addsub_shift"_h,
+         "sub_64_addsub_shift"_h,
+         "mla_asimdsame_only"_h,
+         "mls_asimdsame_only"_h,
+         "mul_asimdsame_only"_h,
+         "saba_asimdsame_only"_h,
+         "sabd_asimdsame_only"_h,
+         "shadd_asimdsame_only"_h,
+         "shsub_asimdsame_only"_h,
+         "smaxp_asimdsame_only"_h,
+         "smax_asimdsame_only"_h,
+         "sminp_asimdsame_only"_h,
+         "smin_asimdsame_only"_h,
+         "srhadd_asimdsame_only"_h,
+         "uaba_asimdsame_only"_h,
+         "uabd_asimdsame_only"_h,
+         "uhadd_asimdsame_only"_h,
+         "uhsub_asimdsame_only"_h,
+         "umaxp_asimdsame_only"_h,
+         "umax_asimdsame_only"_h,
+         "uminp_asimdsame_only"_h,
+         "umin_asimdsame_only"_h,
+         "urhadd_asimdsame_only"_h,
+         "sqdmulh_asimdsame_only"_h,
+         "sqrdmulh_asimdsame_only"_h,
+         "sqrdmlah_asimdsame2_only"_h,
+         "sqrdmlsh_asimdsame2_only"_h,
+         "sdot_asimdsame2_d"_h,
+         "udot_asimdsame2_d"_h,
+         "clz_asimdmisc_r"_h,
+         "cls_asimdmisc_r"_h,
+         "rev64_asimdmisc_r"_h,
+         "mla_asimdelem_r"_h,
+         "mls_asimdelem_r"_h,
+         "mul_asimdelem_r"_h,
+         "sqdmulh_asimdelem_r"_h,
+         "sqrdmlah_asimdelem_r"_h,
+         "sqrdmlsh_asimdelem_r"_h,
+         "sqrdmulh_asimdelem_r"_h,
+         "sqxtn_asisdmisc_n"_h,
+         "sqxtun_asisdmisc_n"_h,
+         "uqxtn_asisdmisc_n"_h,
+         "sqdmlal_asisddiff_only"_h,
+         "sqdmlsl_asisddiff_only"_h,
+         "sqdmull_asisddiff_only"_h,
+         "sqdmulh_asisdsame_only"_h,
+         "sqrdmulh_asisdsame_only"_h,
+         "sqrdmlah_asisdsame2_only"_h,
+         "sqrdmlsh_asisdsame2_only"_h,
+         "sqdmulh_asisdelem_r"_h,
+         "sqrdmlah_asisdelem_r"_h,
+         "sqrdmlsh_asisdelem_r"_h,
+         "sqrdmulh_asisdelem_r"_h,
+         "sqdmlal_asisdelem_l"_h,
+         "sqdmlsl_asisdelem_l"_h,
+         "sqdmull_asisdelem_l"_h,
+         "shll_asimdmisc_s"_h,
+         "xtn_asimdmisc_n"_h,
+         "sqxtn_asimdmisc_n"_h,
+         "uqxtn_asimdmisc_n"_h,
+         "sqxtun_asimdmisc_n"_h,
+         "smlal_asimdelem_l"_h,
+         "smlsl_asimdelem_l"_h,
+         "smull_asimdelem_l"_h,
+         "umlal_asimdelem_l"_h,
+         "umlsl_asimdelem_l"_h,
+         "umull_asimdelem_l"_h,
+         "sqdmull_asimdelem_l"_h,
+         "sqdmlal_asimdelem_l"_h,
+         "sqdmlsl_asimdelem_l"_h,
+         "saddlv_asimdall_only"_h,
+         "uaddlv_asimdall_only"_h,
+         "addv_asimdall_only"_h,
+         "smaxv_asimdall_only"_h,
+         "sminv_asimdall_only"_h,
+         "umaxv_asimdall_only"_h,
+         "uminv_asimdall_only"_h,
+         "sabal_asimddiff_l"_h,
+         "sabdl_asimddiff_l"_h,
+         "saddl_asimddiff_l"_h,
+         "smlal_asimddiff_l"_h,
+         "smlsl_asimddiff_l"_h,
+         "smull_asimddiff_l"_h,
+         "ssubl_asimddiff_l"_h,
+         "uabal_asimddiff_l"_h,
+         "uabdl_asimddiff_l"_h,
+         "uaddl_asimddiff_l"_h,
+         "umlal_asimddiff_l"_h,
+         "umlsl_asimddiff_l"_h,
+         "umull_asimddiff_l"_h,
+         "usubl_asimddiff_l"_h,
+         "saddw_asimddiff_w"_h,
+         "ssubw_asimddiff_w"_h,
+         "uaddw_asimddiff_w"_h,
+         "usubw_asimddiff_w"_h
+         "addhn_asimddiff_n"_h,
+         "raddhn_asimddiff_n"_h,
+         "rsubhn_asimddiff_n"_h,
+         "subhn_asimddiff_n"_h,
+         "sqdmlal_asimddiff_l"_h,
+         "sqdmlsl_asimddiff_l"_h,
+         "sqdmull_asimddiff_l"_h}},
+       {0x00c02000'00002000, {"dup_z_i"_h}},
+       {0x00d80000'00000000,
+        {"xar_z_zzi"_h,
+         "asr_z_zi"_h,
+         "lsr_z_zi"_h,
+         "sri_z_zzi"_h,
+         "srsra_z_zi"_h,
+         "ssra_z_zi"_h,
+         "ursra_z_zi"_h,
+         "usra_z_zi"_h,
+         "lsl_z_zi"_h,
+         "sli_z_zzi"_h}},
+       {0x40000000'00000000, {"fcmla_asimdelem_c_s"_h}},
+       {0x40000800'00000800, {"fcmla_asimdelem_c_h"_h}},
+       {0x40000c00'00000c00,
+        {"ld2_asisdlse_r2"_h,
+         "ld2_asisdlsep_i2_i"_h,
+         "ld2_asisdlsep_r2_r"_h,
+         "st2_asisdlse_r2"_h,
+         "st2_asisdlsep_i2_i"_h,
+         "st2_asisdlsep_r2_r"_h,
+         "ld3_asisdlse_r3"_h,
+         "ld3_asisdlsep_i3_i"_h,
+         "ld3_asisdlsep_r3_r"_h,
+         "st3_asisdlse_r3"_h,
+         "st3_asisdlsep_i3_i"_h,
+         "st3_asisdlsep_r3_r"_h,
+         "ld4_asisdlse_r4"_h,
+         "ld4_asisdlsep_i4_i"_h,
+         "ld4_asisdlsep_r4_r"_h,
+         "st4_asisdlse_r4"_h,
+         "st4_asisdlsep_i4_i"_h,
+         "st4_asisdlsep_r4_r"_h}},
+       {0x40004000'00004000, {"ext_asimdext_only"_h}},
+       {0x400f0000'00080000, {"dup_asimdins_dv_v"_h, "dup_asimdins_dr_d"_h}},
+       {0x40400000'00000000,
+        {"fmaxnmv_asimdall_only_sd"_h,
+         "fminnmv_asimdall_only_sd"_h,
+         "fmaxv_asimdall_only_sd"_h,
+         "fminv_asimdall_only_sd"_h}},
+       {0x40400000'00400000,
+        {"fabs_asimdmisc_r"_h,         "fcvtas_asimdmisc_r"_h,
+         "fcvtau_asimdmisc_r"_h,       "fcvtms_asimdmisc_r"_h,
+         "fcvtmu_asimdmisc_r"_h,       "fcvtns_asimdmisc_r"_h,
+         "fcvtnu_asimdmisc_r"_h,       "fcvtps_asimdmisc_r"_h,
+         "fcvtpu_asimdmisc_r"_h,       "fcvtzs_asimdmisc_r"_h,
+         "fcvtzu_asimdmisc_r"_h,       "fneg_asimdmisc_r"_h,
+         "frecpe_asimdmisc_r"_h,       "frint32x_asimdmisc_r"_h,
+         "frint32z_asimdmisc_r"_h,     "frint64x_asimdmisc_r"_h,
+         "frint64z_asimdmisc_r"_h,     "frinta_asimdmisc_r"_h,
+         "frinti_asimdmisc_r"_h,       "frintm_asimdmisc_r"_h,
+         "frintn_asimdmisc_r"_h,       "frintp_asimdmisc_r"_h,
+         "frintx_asimdmisc_r"_h,       "frintz_asimdmisc_r"_h,
+         "frsqrte_asimdmisc_r"_h,      "fsqrt_asimdmisc_r"_h,
+         "scvtf_asimdmisc_r"_h,        "ucvtf_asimdmisc_r"_h,
+         "fmaxnmv_asimdall_only_sd"_h, "fminnmv_asimdall_only_sd"_h,
+         "fmaxv_asimdall_only_sd"_h,   "fminv_asimdall_only_sd"_h,
+         "fcmeq_asimdmisc_fz"_h,       "fcmge_asimdmisc_fz"_h,
+         "fcmgt_asimdmisc_fz"_h,       "fcmle_asimdmisc_fz"_h,
+         "fcmlt_asimdmisc_fz"_h,       "fabd_asimdsame_only"_h,
+         "facge_asimdsame_only"_h,     "facgt_asimdsame_only"_h,
+         "faddp_asimdsame_only"_h,     "fadd_asimdsame_only"_h,
+         "fcmeq_asimdsame_only"_h,     "fcmge_asimdsame_only"_h,
+         "fcmgt_asimdsame_only"_h,     "fdiv_asimdsame_only"_h,
+         "fmaxnmp_asimdsame_only"_h,   "fmaxnm_asimdsame_only"_h,
+         "fmaxp_asimdsame_only"_h,     "fmax_asimdsame_only"_h,
+         "fminnmp_asimdsame_only"_h,   "fminnm_asimdsame_only"_h,
+         "fminp_asimdsame_only"_h,     "fmin_asimdsame_only"_h,
+         "fmla_asimdsame_only"_h,      "fmls_asimdsame_only"_h,
+         "fmulx_asimdsame_only"_h,     "fmul_asimdsame_only"_h,
+         "frecps_asimdsame_only"_h,    "frsqrts_asimdsame_only"_h,
+         "fsub_asimdsame_only"_h,      "fmla_asimdelem_r_sd"_h,
+         "fmls_asimdelem_r_sd"_h,      "fmulx_asimdelem_r_sd"_h,
+         "fmul_asimdelem_r_sd"_h,      "sri_asimdshf_r"_h,
+         "srshr_asimdshf_r"_h,         "srsra_asimdshf_r"_h,
+         "sshr_asimdshf_r"_h,          "ssra_asimdshf_r"_h,
+         "urshr_asimdshf_r"_h,         "ursra_asimdshf_r"_h,
+         "ushr_asimdshf_r"_h,          "usra_asimdshf_r"_h,
+         "scvtf_asimdshf_c"_h,         "ucvtf_asimdshf_c"_h,
+         "fcvtzs_asimdshf_c"_h,        "fcvtzu_asimdshf_c"_h}},
+       {0x40400000'40400000,
+        {"fmaxnmv_asimdall_only_sd"_h,
+         "fminnmv_asimdall_only_sd"_h,
+         "fmaxv_asimdall_only_sd"_h,
+         "fminv_asimdall_only_sd"_h}},
+       {0x40c00000'00800000,
+        {"saddlv_asimdall_only"_h,
+         "uaddlv_asimdall_only"_h,
+         "addv_asimdall_only"_h,
+         "smaxv_asimdall_only"_h,
+         "sminv_asimdall_only"_h,
+         "umaxv_asimdall_only"_h,
+         "uminv_asimdall_only"_h}},
+       {0x40c00000'00c00000,
+        {"cmeq_asimdmisc_z"_h,       "cmge_asimdmisc_z"_h,
+         "cmgt_asimdmisc_z"_h,       "cmle_asimdmisc_z"_h,
+         "cmlt_asimdmisc_z"_h,       "addp_asimdsame_only"_h,
+         "add_asimdsame_only"_h,     "cmeq_asimdsame_only"_h,
+         "cmge_asimdsame_only"_h,    "cmgt_asimdsame_only"_h,
+         "cmhi_asimdsame_only"_h,    "cmhs_asimdsame_only"_h,
+         "cmtst_asimdsame_only"_h,   "sqadd_asimdsame_only"_h,
+         "sqdmulh_asimdsame_only"_h, "sqrdmulh_asimdsame_only"_h,
+         "sqrshl_asimdsame_only"_h,  "sqshl_asimdsame_only"_h,
+         "sqsub_asimdsame_only"_h,   "srshl_asimdsame_only"_h,
+         "sshl_asimdsame_only"_h,    "sub_asimdsame_only"_h,
+         "uqadd_asimdsame_only"_h,   "uqrshl_asimdsame_only"_h,
+         "uqshl_asimdsame_only"_h,   "uqsub_asimdsame_only"_h,
+         "urshl_asimdsame_only"_h,   "ushl_asimdsame_only"_h,
+         "trn1_asimdperm_only"_h,    "trn2_asimdperm_only"_h,
+         "uzp1_asimdperm_only"_h,    "uzp2_asimdperm_only"_h,
+         "zip1_asimdperm_only"_h,    "zip2_asimdperm_only"_h,
+         "fcmla_asimdsame2_c"_h,     "fcadd_asimdsame2_c"_h}},
+       {0x80200000'00200000,
+        {"sbfm_64m_bitfield"_h,
+         "sbfm_32m_bitfield"_h,
+         "ubfm_32m_bitfield"_h,
+         "ubfm_64m_bitfield"_h,
+         "bfm_32m_bitfield"_h,
+         "bfm_64m_bitfield"_h}},
+       {0x80008000'00000000,
+        {"scvtf_d32_float2fix"_h,
+         "scvtf_d64_float2fix"_h,
+         "scvtf_h32_float2fix"_h,
+         "scvtf_h64_float2fix"_h,
+         "scvtf_s32_float2fix"_h,
+         "scvtf_s64_float2fix"_h,
+         "ucvtf_d32_float2fix"_h,
+         "ucvtf_d64_float2fix"_h,
+         "ucvtf_h32_float2fix"_h,
+         "ucvtf_h64_float2fix"_h,
+         "ucvtf_s32_float2fix"_h,
+         "ucvtf_s64_float2fix"_h}},
+       {0x80008000'00008000,
+        {"sbfm_64m_bitfield"_h,
+         "sbfm_32m_bitfield"_h,
+         "ubfm_32m_bitfield"_h,
+         "ubfm_64m_bitfield"_h,
+         "bfm_32m_bitfield"_h,
+         "bfm_64m_bitfield"_h}},
+       {0xc0000000'40000000,
+        {"cpyen_cpy_memcms"_h,   "cpyern_cpy_memcms"_h,  "cpyewn_cpy_memcms"_h,
+         "cpye_cpy_memcms"_h,    "cpyfen_cpy_memcms"_h,  "cpyfern_cpy_memcms"_h,
+         "cpyfewn_cpy_memcms"_h, "cpyfe_cpy_memcms"_h,   "cpyfmn_cpy_memcms"_h,
+         "cpyfmrn_cpy_memcms"_h, "cpyfmwn_cpy_memcms"_h, "cpyfm_cpy_memcms"_h,
+         "cpyfpn_cpy_memcms"_h,  "cpyfprn_cpy_memcms"_h, "cpyfpwn_cpy_memcms"_h,
+         "cpyfp_cpy_memcms"_h,   "cpymn_cpy_memcms"_h,   "cpymrn_cpy_memcms"_h,
+         "cpymwn_cpy_memcms"_h,  "cpym_cpy_memcms"_h,    "cpypn_cpy_memcms"_h,
+         "cpyprn_cpy_memcms"_h,  "cpypwn_cpy_memcms"_h,  "cpyp_cpy_memcms"_h,
+         "seten_set_memcms"_h,   "sete_set_memcms"_h,    "setgen_set_memcms"_h,
+         "setge_set_memcms"_h,   "setgmn_set_memcms"_h,  "setgm_set_memcms"_h,
+         "setgpn_set_memcms"_h,  "setgp_set_memcms"_h,   "setmn_set_memcms"_h,
+         "setm_set_memcms"_h,    "setpn_set_memcms"_h,   "setp_set_memcms"_h}},
+       {0xc0000000'80000000,
+        {"cpyen_cpy_memcms"_h,   "cpyern_cpy_memcms"_h,  "cpyewn_cpy_memcms"_h,
+         "cpye_cpy_memcms"_h,    "cpyfen_cpy_memcms"_h,  "cpyfern_cpy_memcms"_h,
+         "cpyfewn_cpy_memcms"_h, "cpyfe_cpy_memcms"_h,   "cpyfmn_cpy_memcms"_h,
+         "cpyfmrn_cpy_memcms"_h, "cpyfmwn_cpy_memcms"_h, "cpyfm_cpy_memcms"_h,
+         "cpyfpn_cpy_memcms"_h,  "cpyfprn_cpy_memcms"_h, "cpyfpwn_cpy_memcms"_h,
+         "cpyfp_cpy_memcms"_h,   "cpymn_cpy_memcms"_h,   "cpymrn_cpy_memcms"_h,
+         "cpymwn_cpy_memcms"_h,  "cpym_cpy_memcms"_h,    "cpypn_cpy_memcms"_h,
+         "cpyprn_cpy_memcms"_h,  "cpypwn_cpy_memcms"_h,  "cpyp_cpy_memcms"_h,
+         "seten_set_memcms"_h,   "sete_set_memcms"_h,    "setgen_set_memcms"_h,
+         "setge_set_memcms"_h,   "setgmn_set_memcms"_h,  "setgm_set_memcms"_h,
+         "setgpn_set_memcms"_h,  "setgp_set_memcms"_h,   "setmn_set_memcms"_h,
+         "setm_set_memcms"_h,    "setpn_set_memcms"_h,   "setp_set_memcms"_h}},
+       {0xc0000000'c0000000,
+        {"cpyen_cpy_memcms"_h,   "cpyern_cpy_memcms"_h,  "cpyewn_cpy_memcms"_h,
+         "cpye_cpy_memcms"_h,    "cpyfen_cpy_memcms"_h,  "cpyfern_cpy_memcms"_h,
+         "cpyfewn_cpy_memcms"_h, "cpyfe_cpy_memcms"_h,   "cpyfmn_cpy_memcms"_h,
+         "cpyfmrn_cpy_memcms"_h, "cpyfmwn_cpy_memcms"_h, "cpyfm_cpy_memcms"_h,
+         "cpyfpn_cpy_memcms"_h,  "cpyfprn_cpy_memcms"_h, "cpyfpwn_cpy_memcms"_h,
+         "cpyfp_cpy_memcms"_h,   "cpymn_cpy_memcms"_h,   "cpymrn_cpy_memcms"_h,
+         "cpymwn_cpy_memcms"_h,  "cpym_cpy_memcms"_h,    "cpypn_cpy_memcms"_h,
+         "cpyprn_cpy_memcms"_h,  "cpypwn_cpy_memcms"_h,  "cpyp_cpy_memcms"_h,
+         "seten_set_memcms"_h,   "sete_set_memcms"_h,    "setgen_set_memcms"_h,
+         "setge_set_memcms"_h,   "setgmn_set_memcms"_h,  "setgm_set_memcms"_h,
+         "setgpn_set_memcms"_h,  "setgp_set_memcms"_h,   "setmn_set_memcms"_h,
+         "setm_set_memcms"_h,    "setpn_set_memcms"_h,   "setp_set_memcms"_h}}};
+
+  for (auto& itm : forms) {
+    const std::unordered_set<uint32_t>& s = forms.at(itm.first);
+    for (const uint32_t& its : s) {
+      ftm->insert(std::make_pair(its, itm.first));
+    }
+  }
+}
+
 // Initialise empty vectors for sampled bits and pattern table.
 const std::vector<uint8_t> DecodeNode::kEmptySampledBits;
 const std::vector<DecodePattern> DecodeNode::kEmptyPatternTable;
@@ -190,6 +1033,7 @@ BitExtractFn DecodeNode::GetBitExtractFunctionHelper(uint32_t x, uint32_t y) {
    INSTANTIATE_TEMPLATE_M(00000800);
    INSTANTIATE_TEMPLATE_M(00000c00);
    INSTANTIATE_TEMPLATE_M(00000c10);
+    INSTANTIATE_TEMPLATE_M(00000f00);
    INSTANTIATE_TEMPLATE_M(00000fc0);
    INSTANTIATE_TEMPLATE_M(00001000);
    INSTANTIATE_TEMPLATE_M(00001400);
@@ -324,11 +1168,13 @@ BitExtractFn DecodeNode::GetBitExtractFunctionHelper(uint32_t x, uint32_t y) {
    INSTANTIATE_TEMPLATE_MV(00003000, 00002000);
    INSTANTIATE_TEMPLATE_MV(00003000, 00003000);
    INSTANTIATE_TEMPLATE_MV(00003010, 00000000);
+    INSTANTIATE_TEMPLATE_MV(0000301f, 0000001f);
    INSTANTIATE_TEMPLATE_MV(00003c00, 00003c00);
    INSTANTIATE_TEMPLATE_MV(00040010, 00000000);
    INSTANTIATE_TEMPLATE_MV(00060000, 00000000);
    INSTANTIATE_TEMPLATE_MV(00061000, 00000000);
    INSTANTIATE_TEMPLATE_MV(00070000, 00030000);
+    INSTANTIATE_TEMPLATE_MV(0007309f, 0000001f);
    INSTANTIATE_TEMPLATE_MV(00073ee0, 00033060);
    INSTANTIATE_TEMPLATE_MV(00073f9f, 0000001f);
    INSTANTIATE_TEMPLATE_MV(000f0000, 00000000);
@@ -603,6 +603,28 @@ std::pair<int, int> Instruction::GetSVEMulLongZmAndIndex() const {
  return std::make_pair(reg_code, index);
 }

+// Get the register and index for NEON indexed multiplies.
+std::pair<int, int> Instruction::GetNEONMulRmAndIndex() const {
+  int reg_code = GetRm();
+  int index = (GetNEONH() << 2) | (GetNEONL() << 1) | GetNEONM();
+  switch (GetNEONSize()) {
+    case 0:  // FP H-sized elements.
+    case 1:  // Integer H-sized elements.
+      // 4-bit Rm, 3-bit index.
+      reg_code &= 0xf;
+      break;
+    case 2:  // S-sized elements.
+      // 5-bit Rm, 2-bit index.
+      index >>= 1;
+      break;
+    case 3:  // FP D-sized elements.
+      // 5-bit Rm, 1-bit index.
+      index >>= 2;
+      break;
+  }
+  return std::make_pair(reg_code, index);
+}
+
 // Logical immediates can't encode zero, so a return value of zero is used to
 // indicate a failure case. Specifically, where the constraints on imm_s are
 // not met.
@@ -1025,6 +1047,8 @@ VectorFormat VectorFormatHalfWidth(VectorFormat vform) {
      return kFormatVnH;
    case kFormatVnD:
      return kFormatVnS;
+    case kFormatVnQ:
+      return kFormatVnD;
    default:
      VIXL_UNREACHABLE();
      return kFormatUndefined;
@@ -6118,6 +6118,42 @@ LogicVRegister Simulator::fcvtxn2(VectorFormat vform,
  return dst;
 }

+LogicVRegister Simulator::bfcvtn(VectorFormat vform,
+                                 LogicVRegister dst,
+                                 const LogicVRegister& src) {
+  SimVRegister tmp;
+  LogicVRegister srctmp = mov(kFormatVnD, tmp, src);
+  int input_lane_count = LaneCountFromFormat(vform);
+  if (IsSVEFormat(vform)) {
+    input_lane_count /= 2;
+  }
+
+  dst.ClearForWrite(vform);
+  VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kHRegSize);
+
+  for (int i = 0; i < input_lane_count; i++) {
+    dst.SetFloat(i,
+                 BFloat16ToRawbits(FPToBFloat16(srctmp.Float<float>(i),
+                                                FPTieEven,
+                                                ReadDN())));
+  }
+  return dst;
+}
+
+LogicVRegister Simulator::bfcvtn2(VectorFormat vform,
+                                  LogicVRegister dst,
+                                  const LogicVRegister& src) {
+  VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kHRegSize);
+  dst.ClearForWrite(vform);
+  int lane_count = LaneCountFromFormat(vform) / 2;
+  for (int i = lane_count - 1; i >= 0; i--) {
+    dst.SetFloat(i + lane_count,
+                 BFloat16ToRawbits(
+                     FPToBFloat16(src.Float<float>(i), FPTieEven, ReadDN())));
+  }
+  return dst;
+}
+

 // Based on reference C function recip_sqrt_estimate from ARM ARM.
 double Simulator::recip_sqrt_estimate(double a) {
@@ -7853,16 +7889,26 @@ LogicVRegister Simulator::matmul(VectorFormat vform_dst,
 //
 // Are stored in the input vector registers as:
 //
-//           3   2   1   0
-//  src1 = [ d | c | b | a ]
-//  src2 = [ D | B | C | A ]
+//             3   2   1   0
+//    src1 = [ d | c | b | a ]
+//    src2 = [ D | B | C | A ]  nb. transposition
 //
+// Giving:
+//             3   2   1   0
+//  result = [ w | z | y | x ]
+//
+// Where:
+//
+//       x = (a * A) + (b * C) + a
+//       y = (a * B) + (b * D) + b
+//       z = (c * A) + (d * C) + c
+//       w = (c * B) + (d * D) + d
 template <typename T>
 LogicVRegister Simulator::fmatmul(VectorFormat vform,
                                  LogicVRegister srcdst,
                                  const LogicVRegister& src1,
                                  const LogicVRegister& src2) {
-  T result[kZRegMaxSizeInBytes / sizeof(T)];
+  T result[kZRegMaxSizeInBytes / sizeof(T)] = {};
  int T_per_segment = 4;
  int segment_count = GetVectorLengthInBytes() / (T_per_segment * sizeof(T));
  for (int seg = 0; seg < segment_count; seg++) {
@@ -7879,12 +7925,9 @@ LogicVRegister Simulator::fmatmul(VectorFormat vform,
    }
  }
  for (int i = 0; i < LaneCountFromFormat(vform); i++) {
-    // Elements outside a multiple of 4T are set to zero. This happens only
-    // for double precision operations, when the VL is a multiple of 128 bits,
-    // but not a multiple of 256 bits.
-    T value = (i < (T_per_segment * segment_count)) ? result[i] : 0;
-    srcdst.SetFloat<T>(vform, i, value);
+    srcdst.SetFloat<T>(vform, i, result[i]);
  }
+
  return srcdst;
 }

@@ -8477,6 +8520,77 @@ LogicVRegister Simulator::sm3tt2(LogicVRegister srcdst,
  return srcdst;
 }

+static uint64_t SM4SBox(uint64_t x) {
+  static const uint8_t sbox[256] = {
+      0x48, 0x39, 0xcb, 0xd7, 0x3e, 0x5f, 0xee, 0x79, 0x20, 0x4d, 0xdc, 0x3a,
+      0xec, 0x7d, 0xf0, 0x18, 0x84, 0xc6, 0x6e, 0xc5, 0x09, 0xf1, 0xb9, 0x65,
+      0x7e, 0x77, 0x96, 0x0c, 0x4a, 0x97, 0x69, 0x89, 0xb0, 0xb4, 0xe5, 0xb8,
+      0x12, 0xd0, 0x74, 0x2d, 0xbd, 0x7b, 0xcd, 0xa5, 0x88, 0x31, 0xc1, 0x0a,
+      0xd8, 0x5a, 0x10, 0x1f, 0x41, 0x5c, 0xd9, 0x11, 0x7f, 0xbc, 0xdd, 0xbb,
+      0x92, 0xaf, 0x1b, 0x8d, 0x51, 0x5b, 0x6c, 0x6d, 0x72, 0x6a, 0xff, 0x03,
+      0x2f, 0x8e, 0xfd, 0xde, 0x45, 0x37, 0xdb, 0xd5, 0x6f, 0x4e, 0x53, 0x0d,
+      0xab, 0x23, 0x29, 0xc0, 0x60, 0xca, 0x66, 0x82, 0x2e, 0xe2, 0xf6, 0x1d,
+      0xe3, 0xb1, 0x8c, 0xf5, 0x30, 0x32, 0x93, 0xad, 0x55, 0x1a, 0x34, 0x9b,
+      0xa4, 0x5d, 0xae, 0xe0, 0xa1, 0x15, 0x61, 0xf9, 0xce, 0xf2, 0xf7, 0xa3,
+      0xb5, 0x38, 0xc7, 0x40, 0xd2, 0x8a, 0xbf, 0xea, 0x9e, 0xc8, 0xc4, 0xa0,
+      0xe7, 0x02, 0x36, 0x4c, 0x52, 0x27, 0xd3, 0x9f, 0x57, 0x46, 0x00, 0xd4,
+      0x87, 0x78, 0x21, 0x01, 0x3b, 0x7c, 0x22, 0x25, 0xa2, 0xd1, 0x58, 0x63,
+      0x5e, 0x0e, 0x24, 0x1e, 0x35, 0x9d, 0x56, 0x70, 0x4b, 0x0f, 0xeb, 0xf8,
+      0x8b, 0xda, 0x64, 0x71, 0xb2, 0x81, 0x6b, 0x68, 0xa8, 0x4f, 0x85, 0xe6,
+      0x19, 0x3c, 0x59, 0x83, 0xba, 0x17, 0x73, 0xf3, 0xfc, 0xa7, 0x07, 0x47,
+      0xa6, 0x3f, 0x8f, 0x75, 0xfa, 0x94, 0xdf, 0x80, 0x95, 0xe8, 0x08, 0xc9,
+      0xa9, 0x1c, 0xb3, 0xe4, 0x62, 0xac, 0xcf, 0xed, 0x43, 0x0b, 0x54, 0x33,
+      0x7a, 0x98, 0xef, 0x91, 0xf4, 0x50, 0x42, 0x9c, 0x99, 0x06, 0x86, 0x49,
+      0x26, 0x13, 0x44, 0xaa, 0xc3, 0x04, 0xbe, 0x2a, 0x76, 0x9a, 0x67, 0x2b,
+      0x05, 0x2c, 0xfb, 0x28, 0xc2, 0x14, 0xb6, 0x16, 0xb7, 0x3d, 0xe1, 0xcc,
+      0xfe, 0xe9, 0x90, 0xd6,
+  };
+  uint64_t result = 0;
+  for (int j = 24; j >= 0; j -= 8) {
+    uint8_t s = 255 - ((x >> j) & 0xff);
+    result = (result << 8) | sbox[s];
+  }
+  return result;
+}
+
+LogicVRegister Simulator::sm4(LogicVRegister srcdst,
+                              const LogicVRegister& src1,
+                              const LogicVRegister& src2,
+                              bool is_key) {
+  using namespace std::placeholders;
+  auto ROL = std::bind(RotateLeft, _1, _2, kSRegSize);
+
+  VectorFormat vf = kFormat4S;
+  uint64_t result[4] = {};
+  if (is_key) {
+    src1.UintArray(vf, result);
+  } else {
+    srcdst.UintArray(vf, result);
+  }
+
+  for (int i = 0; i < 4; i++) {
+    uint64_t k = is_key ? src2.Uint(vf, i) : src1.Uint(vf, i);
+    uint64_t intval = result[3] ^ result[2] ^ result[1] ^ k;
+    intval = SM4SBox(intval);
+
+    if (is_key) {
+      intval ^= ROL(intval, 13) ^ ROL(intval, 23);
+    } else {
+      intval ^=
+          ROL(intval, 2) ^ ROL(intval, 10) ^ ROL(intval, 18) ^ ROL(intval, 24);
+    }
+
+    intval ^= result[0];
+
+    result[0] = result[1];
+    result[1] = result[2];
+    result[2] = result[3];
+    result[3] = intval;
+  }
+  srcdst.SetUintArray(vf, result);
+  return srcdst;
+}
+
 }  // namespace aarch64
 }  // namespace vixl

@@ -550,8 +550,7 @@ void MacroAssembler::B(Label* label) {
  b(label);
 }

-
-void MacroAssembler::B(Label* label, Condition cond) {
+void MacroAssembler::Bcommon(Label* label, Condition cond, bool use_bc) {
  // We don't need to check the size of the literal pool, because the size of
  // the literal pool is already bounded by the literal range, which is smaller
  // than the range of this branch.
@@ -563,7 +562,11 @@ void MacroAssembler::B(Label* label, Condition cond) {

  if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) {
    Label done;
-    b(&done, InvertCondition(cond));
+    if (use_bc) {
+      bc(&done, InvertCondition(cond));
+    } else {
+      b(&done, InvertCondition(cond));
+    }
    b(label);
    bind(&done);
  } else {
@@ -572,7 +575,11 @@ void MacroAssembler::B(Label* label, Condition cond) {
                                            label,
                                            CondBranchType);
    }
-    b(label, cond);
+    if (use_bc) {
+      bc(label, cond);
+    } else {
+      b(label, cond);
+    }
  }
 }

@@ -34,14 +34,19 @@ namespace vixl {
 const double kFP64DefaultNaN = RawbitsToDouble(UINT64_C(0x7ff8000000000000));
 const float kFP32DefaultNaN = RawbitsToFloat(0x7fc00000);
 const Float16 kFP16DefaultNaN = RawbitsToFloat16(0x7e00);
+const BFloat16 kBFP16DefaultNaN = RawbitsToBFloat16(0x7fc0);

 // Floating-point zero values.
 const Float16 kFP16PositiveZero = RawbitsToFloat16(0x0);
 const Float16 kFP16NegativeZero = RawbitsToFloat16(0x8000);
+const BFloat16 kBFP16PositiveZero = RawbitsToBFloat16(0x0);
+const BFloat16 kBFP16NegativeZero = RawbitsToBFloat16(0x8000);

 // Floating-point infinity values.
 const Float16 kFP16PositiveInfinity = RawbitsToFloat16(0x7c00);
 const Float16 kFP16NegativeInfinity = RawbitsToFloat16(0xfc00);
+const BFloat16 kBFP16PositiveInfinity = RawbitsToBFloat16(0x7f80);
+const BFloat16 kBFP16NegativeInfinity = RawbitsToBFloat16(0xff80);
 const float kFP32PositiveInfinity = RawbitsToFloat(0x7f800000);
 const float kFP32NegativeInfinity = RawbitsToFloat(0xff800000);
 const double kFP64PositiveInfinity =
@@ -57,6 +62,14 @@ bool IsZero(Float16 value) {

 uint16_t Float16ToRawbits(Float16 value) { return value.rawbits_; }

+bool IsZero(BFloat16 value) {
+  uint16_t bits = BFloat16ToRawbits(value);
+  return (bits == BFloat16ToRawbits(kBFP16PositiveZero) ||
+          bits == BFloat16ToRawbits(kBFP16NegativeZero));
+}
+
+uint16_t BFloat16ToRawbits(BFloat16 value) { return value.rawbits_; }
+
 uint32_t FloatToRawbits(float value) {
  uint32_t bits = 0;
  memcpy(&bits, &value, 4);
@@ -78,6 +91,13 @@ Float16 RawbitsToFloat16(uint16_t bits) {
 }


+BFloat16 RawbitsToBFloat16(uint16_t bits) {
+  BFloat16 f;
+  f.rawbits_ = bits;
+  return f;
+}
+
+
 float RawbitsToFloat(uint32_t bits) {
  float value = 0.0;
  memcpy(&value, &bits, 4);
@@ -552,4 +572,76 @@ Float16 FPToFloat16(double value,
  return kFP16PositiveZero;
 }

+BFloat16 FPToBFloat16(float value,
+                      FPRounding round_mode,
+                      UseDefaultNaN DN,
+                      bool* exception) {
+  // Only the FPTieEven rounding mode is implemented.
+  VIXL_ASSERT(round_mode == FPTieEven);
+  USE(round_mode);
+
+  uint32_t raw = FloatToRawbits(value);
+  int32_t sign = raw >> 31;
+  int32_t exponent =
+      static_cast<int32_t>(ExtractUnsignedBitfield32(30, 23, raw)) - 127;
+  uint32_t mantissa = ExtractUnsignedBitfield32(22, 0, raw);
+
+  switch (std::fpclassify(value)) {
+    case FP_NAN: {
+      if (IsSignallingNaN(value)) {
+        if (exception != NULL) {
+          *exception = true;
+        }
+      }
+      if (DN == kUseDefaultNaN) return kBFP16DefaultNaN;
+
+      // Convert NaNs as the processor would:
+      //  - The sign is propagated.
+      //  - The payload (mantissa) is transferred as much as possible, except
+      //    that the top bit is forced to '1', making the result a quiet NaN.
+      uint16_t result = (sign == 0) ? BFloat16ToRawbits(kBFP16PositiveInfinity)
+                                    : BFloat16ToRawbits(kBFP16NegativeInfinity);
+      result |= mantissa >> (kFloatMantissaBits - kBFloat16MantissaBits);
+      result |= (1 << 6);  // Force a quiet NaN;
+      return RawbitsToBFloat16(result);
+    }
+
+    case FP_ZERO:
+      return (sign == 0) ? kBFP16PositiveZero : kBFP16NegativeZero;
+
+    case FP_INFINITE:
+      return (sign == 0) ? kBFP16PositiveInfinity : kBFP16NegativeInfinity;
+
+    case FP_NORMAL:
+      // Add the implicit '1' bit to the mantissa.
+      mantissa += (1 << 23);
+      break;
+
+    case FP_SUBNORMAL:
+      // Reduce exponent to account for MSB of mantissa.
+      int32_t leading_mantissa_bits =
+          CountLeadingZeros(mantissa) - (32 - kFloatMantissaBits);
+      exponent -= leading_mantissa_bits;
+      break;
+  }
+
+  // Convert float-to-half as the processor would, assuming that FPCR.FZ
+  // (flush-to-zero) is not set.
+  return FPRoundToBFloat16(sign, exponent, mantissa, round_mode);
+}
+
+BFloat16 FPToBFloat16(double value,
+                      FPRounding round_mode,
+                      UseDefaultNaN DN,
+                      bool* exception) {
+  USE(value);
+  USE(round_mode);
+  USE(DN);
+  USE(exception);
+  // TODO: Implement this for correct conversion of doubles to BFloat (without
+  // implicit NaN silencing.)
+  VIXL_UNIMPLEMENTED();
+  return kBFP16PositiveZero;
+}
+
 }  // namespace vixl
@@ -76,6 +76,7 @@
    <ClCompile Include="src\aarch64\cpu-aarch64.cc" />
    <ClCompile Include="src\aarch64\cpu-features-auditor-aarch64.cc" />
    <ClCompile Include="src\aarch64\decoder-aarch64.cc" />
+    <ClCompile Include="src\aarch64\debugger-aarch64.cc" />
    <ClCompile Include="src\aarch64\disasm-aarch64.cc" />
    <ClCompile Include="src\aarch64\instructions-aarch64.cc" />
    <ClCompile Include="src\aarch64\logic-aarch64.cc" />
@@ -84,6 +85,7 @@
    <ClCompile Include="src\aarch64\operands-aarch64.cc" />
    <ClCompile Include="src\aarch64\pointer-auth-aarch64.cc" />
    <ClCompile Include="src\aarch64\registers-aarch64.cc" />
+    <ClCompile Include="src\aarch64\simulator-aarch64.cc" />
    <ClCompile Include="src\code-buffer-vixl.cc" />
    <ClCompile Include="src\compiler-intrinsics-vixl.cc" />
    <ClCompile Include="src\cpu-features.cc" />
@@ -65,6 +65,9 @@
    <ClCompile Include="src\aarch64\decoder-aarch64.cc">
      <Filter>aarch64</Filter>
    </ClCompile>
+    <ClCompile Include="src\aarch64\debugger-aarch64.cc">
+      <Filter>aarch64</Filter>
+    </ClCompile>
    <ClCompile Include="src\aarch64\disasm-aarch64.cc">
      <Filter>aarch64</Filter>
    </ClCompile>
@@ -89,6 +92,9 @@
    <ClCompile Include="src\aarch64\registers-aarch64.cc">
      <Filter>aarch64</Filter>
    </ClCompile>
+    <ClCompile Include="src\aarch64\simulator-aarch64.cc">
+      <Filter>aarch64</Filter>
+    </ClCompile>
    <ClCompile Include="src\aarch64\assembler-aarch64.cc">
      <Filter>aarch64</Filter>
    </ClCompile>
@@ -15,7 +15,7 @@

 #include <stdio.h> // for debug print
 #include <assert.h>
-#include <list>
+#include <vector>
 #include <string>
 #include <algorithm>
 #ifndef NDEBUG
@@ -116,10 +116,12 @@
 	#undef XBYAK_USE_MEMFD
 #endif

-#if defined(_WIN64) || defined(__MINGW64__) || (defined(__CYGWIN__) && defined(__x86_64__))
-	#define XBYAK64_WIN
-#elif defined(__x86_64__)
-	#define XBYAK64_GCC
+#if !defined(XBYAK64_WIN) && !defined(XBYAK64_GCC)
+	#if defined(_WIN64) || defined(__MINGW64__) || (defined(__CYGWIN__) && defined(__x86_64__))
+		#define XBYAK64_WIN
+	#elif defined(__x86_64__)
+		#define XBYAK64_GCC
+	#endif
 #endif
 #if !defined(XBYAK64) && !defined(XBYAK32)
 	#if defined(XBYAK64_GCC) || defined(XBYAK64_WIN)
@@ -174,7 +176,7 @@ namespace Xbyak {

 enum {
 	DEFAULT_MAX_CODE_SIZE = 4096,
-	VERSION = 0x7352 /* 0xABCD = A.BC(.D) */
+	VERSION = 0x7370 /* 0xABCD = A.BC(.D) */
 };

 #ifndef MIE_INTEGER_TYPE_DEFINED
@@ -1074,7 +1076,7 @@ public:
 		}
 	}
 	friend RegExp operator+(const RegExp& a, const RegExp& b);
-	friend RegExp operator+(const RegExp& e, size_t disp);
+	friend RegExp operator+(const RegExp& e, unsigned long long disp);
 	friend RegExp operator-(const RegExp& e, size_t disp);
 private:
 	/*
@@ -1128,15 +1130,19 @@ inline RegExp operator*(int scale, const Reg& r)
 // backward compatibility for eax+&x (pointer address)
 inline RegExp operator+(const RegExp& a, const void* b) { return a + RegExp(b); }

-// overload for integer literals (e.g. eax+0) to avoid ambiguity with the void* overload
-inline RegExp operator+(const RegExp& e, int disp) { return e + size_t(disp); }
-
-inline RegExp operator+(const RegExp& e, size_t disp)
+// since what size_t is typedef'd to depends on the implementation, use unsigned long long (assume u64) for the implementation.
+inline RegExp operator+(const RegExp& e, unsigned long long disp)
 {
 	RegExp ret = e;
-	ret.disp_ += disp;
+	ret.disp_ += static_cast<size_t>(disp);
 	return ret;
 }
+// overload for integer literals (e.g. eax+0) to avoid ambiguity with the void* overload
+inline RegExp operator+(const RegExp& e, int disp) { return e + static_cast<unsigned long long>(disp); }
+inline RegExp operator+(const RegExp& e, long disp) { return e + static_cast<unsigned long long>(disp); }
+inline RegExp operator+(const RegExp& e, long long disp) { return e + static_cast<unsigned long long>(disp); }
+inline RegExp operator+(const RegExp& e, unsigned int disp) { return e + static_cast<unsigned long long>(disp); }
+inline RegExp operator+(const RegExp& e, unsigned long disp) { return e + static_cast<unsigned long long>(disp); }

 inline RegExp operator-(const RegExp& e, size_t disp)
 {
@@ -1172,7 +1178,7 @@ class CodeArray {
 			return disp;
 		}
 	};
-	typedef std::list<AddrInfo> AddrInfoList;
+	typedef std::vector<AddrInfo> AddrInfoList;
 	AddrInfoList addrInfoList_;
 	const Type type_;
 #ifdef XBYAK_USE_MMAP_ALLOCATOR
@@ -1319,7 +1325,7 @@ public:
 	*/
 	void rewrite(size_t offset, uint64_t disp, size_t size)
 	{
-		assert(offset < maxSize_);
+		if (offset >= maxSize_ || size > maxSize_ - offset) XBYAK_THROW(ERR_OFFSET_IS_TOO_BIG)
 		if (size != 1 && size != 2 && size != 4 && size != 8) XBYAK_THROW(ERR_BAD_PARAMETER)
 		uint8_t *const data = top_ + offset;
 		for (size_t i = 0; i < size; i++) {
@@ -1363,7 +1369,7 @@ public:
 		DWORD oldProtect;
 		return VirtualProtect(const_cast<void*>(addr), size, mode, &oldProtect) != 0;
 #elif defined(__GNUC__)
-		size_t pageSize = sysconf(_SC_PAGESIZE);
+		size_t pageSize = inner::getPageSize();
 		size_t iaddr = reinterpret_cast<size_t>(addr);
 		size_t roundAddr = iaddr & ~(pageSize - static_cast<size_t>(1));
 		return mprotect(reinterpret_cast<void*>(roundAddr), size + (iaddr - roundAddr), mode) == 0;
@@ -1548,7 +1554,8 @@ class LabelManager {
 		SlabelDefList defList;
 		SlabelUndefList undefList;
 	};
-	typedef std::list<SlabelState> StateList;
+	// SlabelState is cheap to move, so std::vector is preferred over std::list.
+	typedef std::vector<SlabelState> StateList;
 	// for Label class
 	struct ClabelVal {
 		ClabelVal(size_t offset = 0) : offset(offset), refCount(1) {}
@@ -1,4 +1,4 @@
-const char *getVersionString() const { return "7.35.2"; }
+const char *getVersionString() const { return "7.37"; }
 void aadd(const Address& addr, const Reg32e &reg) { opMR(addr, reg, T_0F38, 0x0FC, T_APX); }
 void aand(const Address& addr, const Reg32e &reg) { opMR(addr, reg, T_0F38|T_66, 0x0FC, T_APX|T_66); }
 void adc(const Operand& op, uint32_t imm) { opOI(op, imm, 0x10, 2); }
@@ -853,7 +853,6 @@ void prefetcht0(const Address& addr) { opMR(addr, Reg32(1), T_0F, 0x18); }
 void prefetcht1(const Address& addr) { opMR(addr, Reg32(2), T_0F, 0x18); }
 void prefetcht2(const Address& addr) { opMR(addr, Reg32(3), T_0F, 0x18); }
 void prefetchw(const Address& addr) { opMR(addr, Reg32(1), T_0F, 0x0D); }
-void prefetchwt1(const Address& addr) { opMR(addr, Reg32(2), T_0F, 0x0D); }
 void psadbw(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0xF6); }
 void pshufb(const Mmx& mmx, const Operand& op) { opMMX(mmx, op, 0x00, T_0F38, T_66); }
 void pshufd(const Mmx& mmx, const Operand& op, uint8_t imm8) { opMMX(mmx, op, 0x70, T_0F, T_66, imm8); }
@@ -2036,10 +2035,6 @@ void kxorb(const Opmask& r1, const Opmask& r2, const Opmask& r3) { opVex(r1, &r2
 void kxord(const Opmask& r1, const Opmask& r2, const Opmask& r3) { opVex(r1, &r2, r3, T_L1 | T_0F | T_66 | T_W1, 0x47); }
 void kxorq(const Opmask& r1, const Opmask& r2, const Opmask& r3) { opVex(r1, &r2, r3, T_L1 | T_0F | T_W1, 0x47); }
 void kxorw(const Opmask& r1, const Opmask& r2, const Opmask& r3) { opVex(r1, &r2, r3, T_L1 | T_0F | T_W0, 0x47); }
-void v4fmaddps(const Zmm& z1, const Zmm& z2, const Address& addr) { opAVX_X_X_XM(z1, z2, addr, T_0F38 | T_F2 | T_W0 | T_YMM | T_MUST_EVEX | T_N16, 0x9A); }
-void v4fmaddss(const Xmm& x1, const Xmm& x2, const Address& addr) { opAVX_X_X_XM(x1, x2, addr, T_0F38 | T_F2 | T_W0 | T_MUST_EVEX | T_N16, 0x9B); }
-void v4fnmaddps(const Zmm& z1, const Zmm& z2, const Address& addr) { opAVX_X_X_XM(z1, z2, addr, T_0F38 | T_F2 | T_W0 | T_YMM | T_MUST_EVEX | T_N16, 0xAA); }
-void v4fnmaddss(const Xmm& x1, const Xmm& x2, const Address& addr) { opAVX_X_X_XM(x1, x2, addr, T_0F38 | T_F2 | T_W0 | T_MUST_EVEX | T_N16, 0xAB); }
 void vaddbf16(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x58); }
 void vaddph(const Xmm& xmm, const Operand& op1, const Operand& op2 = Operand()) { opAVX_X_X_XM(xmm, op1, op2, T_MAP5 | T_W0 | T_YMM | T_MUST_EVEX | T_ER_Z | T_B16, 0x58); }
 void vaddsh(const Xmm& xmm, const Operand& op1, const Operand& op2 = Operand()) { opAVX_X_X_XM(xmm, op1, op2, T_MAP5 | T_F3 | T_W0 | T_MUST_EVEX | T_ER_X | T_N2, 0x58); }
@@ -2199,10 +2194,10 @@ void vcompressps(const Operand& op, const Xmm& x) { opAVX_X_XM_IMM(x, op, T_N4|T
 void vcomxsd(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N8|T_F2|T_0F|T_EW1|T_SAE_X|T_MUST_EVEX, 0x2F); }
 void vcomxsh(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N2|T_F3|T_MAP5|T_W0|T_SAE_X|T_MUST_EVEX, 0x2F); }
 void vcomxss(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_N4|T_F3|T_0F|T_W0|T_SAE_X|T_MUST_EVEX, 0x2F); }
-void vcvt2ph2bf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N1|T_F2|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); }
-void vcvt2ph2bf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N1|T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); }
-void vcvt2ph2hf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N1|T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x18); }
-void vcvt2ph2hf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N1|T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x1B); }
+void vcvt2ph2bf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_N_VL|T_F2|T_0F38|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); }
+void vcvt2ph2bf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_N_VL|T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x74); }
+void vcvt2ph2hf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_N_VL|T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x18); }
+void vcvt2ph2hf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_N16|T_N_VL|T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x1B); }
 void vcvt2ps2phx(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F38|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x67); }
 void vcvtbf162ibs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x69); }
 void vcvtbf162iubs(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_F2|T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x6B); }
@@ -2211,7 +2206,7 @@ void vcvtbiasph2bf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt6(x
 void vcvtbiasph2hf8(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt6(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x18); }
 void vcvtbiasph2hf8s(const Xmm& x1, const Xmm& x2, const Operand& op) { opCvt6(x1, x2, op, T_MAP5|T_W0|T_YMM|T_MUST_EVEX|T_B16, 0x1B); }
 void vcvtdq2ph(const Xmm& x, const Operand& op) { checkCvt4(x, op); opCvt(x, op, T_N16|T_N_VL|T_MAP5|T_W0|T_YMM|T_ER_Z|T_MUST_EVEX|T_B32, 0x5B); }
-void vcvthf82ph(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_MUST_EVEX | T_F2 | T_MAP5 | T_W0 | T_YMM | T_N1, 0x1E); }
+void vcvthf82ph(const Xmm& x, const Operand& op) { checkCvt1(x, op); opVex(x, 0, op, T_MUST_EVEX|T_F2|T_MAP5|T_W0|T_YMM|T_N8|T_N_VL, 0x1E); }
 void vcvtne2ps2bf16(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_F2|T_0F38|T_W0|T_YMM|T_SAE_Z|T_MUST_EVEX|T_B32, 0x72); }
 void vcvtpd2ph(const Xmm& x, const Operand& op) { opCvt5(x, op, T_N16|T_N_VL|T_66|T_MAP5|T_EW1|T_ER_Z|T_MUST_EVEX|T_B64, 0x5A); }
 void vcvtpd2qq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66|T_0F|T_EW1|T_YMM|T_ER_Z|T_MUST_EVEX|T_B64, 0x7B); }
@@ -2437,8 +2432,6 @@ void vmulph(const Xmm& xmm, const Operand& op1, const Operand& op2 = Operand())
 void vmulsh(const Xmm& xmm, const Operand& op1, const Operand& op2 = Operand()) { opAVX_X_X_XM(xmm, op1, op2, T_MAP5 | T_F3 | T_W0 | T_MUST_EVEX | T_ER_X | T_N2, 0x59); }
 void vp2intersectd(const Opmask& k, const Xmm& x, const Operand& op) { if (k.getOpmaskIdx() != 0) XBYAK_THROW(ERR_OPMASK_IS_ALREADY_SET) opAVX_K_X_XM(k, x, op, T_F2 | T_0F38 | T_YMM | T_EVEX | T_W0 | T_B32, 0x68); }
 void vp2intersectq(const Opmask& k, const Xmm& x, const Operand& op) { if (k.getOpmaskIdx() != 0) XBYAK_THROW(ERR_OPMASK_IS_ALREADY_SET) opAVX_K_X_XM(k, x, op, T_F2 | T_0F38 | T_YMM | T_EVEX | T_EW1 | T_B64, 0x68); }
-void vp4dpwssd(const Zmm& z1, const Zmm& z2, const Address& addr) { opAVX_X_X_XM(z1, z2, addr, T_0F38 | T_F2 | T_W0 | T_YMM | T_MUST_EVEX | T_N16, 0x52); }
-void vp4dpwssds(const Zmm& z1, const Zmm& z2, const Address& addr) { opAVX_X_X_XM(z1, z2, addr, T_0F38 | T_F2 | T_W0 | T_YMM | T_MUST_EVEX | T_N16, 0x53); }
 void vpabsq(const Xmm& x, const Operand& op) { opAVX_X_XM_IMM(x, op, T_66 | T_0F38 | T_MUST_EVEX | T_EW1 | T_B64 | T_YMM, 0x1F); }
 void vpandd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0xDB); }
 void vpandnd(const Xmm& x1, const Xmm& x2, const Operand& op) { opAVX_X_X_XM(x1, x2, op, T_66|T_0F|T_W0|T_YMM|T_MUST_EVEX|T_B32, 0xDF); }
@@ -101,6 +101,8 @@
 #endif
 #ifdef _WIN32
 #include <windows.h>
+#else
+#include <sched.h>
 #endif
 namespace Xbyak { namespace util {
 class CpuTopology;
@@ -526,16 +528,16 @@ public:
 	XBYAK_DEFINE_TYPE(36, tAVX512DQ);
 	XBYAK_DEFINE_TYPE(37, tAVX512_IFMA);
 	XBYAK_DEFINE_TYPE(37, tAVX512IFMA);// = tAVX512_IFMA;
-	XBYAK_DEFINE_TYPE(38, tAVX512PF);
-	XBYAK_DEFINE_TYPE(39, tAVX512ER);
+//	XBYAK_DEFINE_TYPE(38, tAVX512PF); // Xeon Phi only
+//	XBYAK_DEFINE_TYPE(39, tAVX512ER);
 	XBYAK_DEFINE_TYPE(40, tAVX512CD);
 	XBYAK_DEFINE_TYPE(41, tAVX512BW);
 	XBYAK_DEFINE_TYPE(42, tAVX512VL);
 	XBYAK_DEFINE_TYPE(43, tAVX512_VBMI);
 	XBYAK_DEFINE_TYPE(43, tAVX512VBMI); // = tAVX512_VBMI; // changed by Intel's manual
-	XBYAK_DEFINE_TYPE(44, tAVX512_4VNNIW);
-	XBYAK_DEFINE_TYPE(45, tAVX512_4FMAPS);
-	XBYAK_DEFINE_TYPE(46, tPREFETCHWT1);
+//	XBYAK_DEFINE_TYPE(44, tAVX512_4VNNIW);
+//	XBYAK_DEFINE_TYPE(45, tAVX512_4FMAPS);
+//	XBYAK_DEFINE_TYPE(46, tPREFETCHWT1);
 	XBYAK_DEFINE_TYPE(47, tPREFETCHW);
 	XBYAK_DEFINE_TYPE(48, tSHA);
 	XBYAK_DEFINE_TYPE(49, tMPX);
@@ -587,6 +589,7 @@ public:
 	XBYAK_DEFINE_TYPE(95, tAMX_FP8);
 	XBYAK_DEFINE_TYPE(96, tMOVRS);
 	XBYAK_DEFINE_TYPE(97, tHYBRID);
+	XBYAK_DEFINE_TYPE(98, tAMX_COMPLEX);

 #undef XBYAK_SPLIT_ID
 #undef XBYAK_DEFINE_TYPE
@@ -679,8 +682,6 @@ public:
 					if (type_ & tAVX512F) {
 						if (ebx & (1U << 17)) type_ |= tAVX512DQ;
 						if (ebx & (1U << 21)) type_ |= tAVX512_IFMA;
-						if (ebx & (1U << 26)) type_ |= tAVX512PF;
-						if (ebx & (1U << 27)) type_ |= tAVX512ER;
 						if (ebx & (1U << 28)) type_ |= tAVX512CD;
 						if (ebx & (1U << 30)) type_ |= tAVX512BW;
 						if (ebx & (1U << 31)) type_ |= tAVX512VL;
@@ -689,8 +690,6 @@ public:
 						if (ecx & (1U << 11)) type_ |= tAVX512_VNNI;
 						if (ecx & (1U << 12)) type_ |= tAVX512_BITALG;
 						if (ecx & (1U << 14)) type_ |= tAVX512_VPOPCNTDQ;
-						if (edx & (1U << 2)) type_ |= tAVX512_4VNNIW;
-						if (edx & (1U << 3)) type_ |= tAVX512_4FMAPS;
 						if (edx & (1U << 8)) type_ |= tAVX512_VP2INTERSECT;
 						if ((type_ & tAVX512BW) && (edx & (1U << 23))) type_ |= tAVX512_FP16;
 					}
@@ -713,7 +712,6 @@ public:
 			if (ebx & (1U << 23)) type_ |= tCLFLUSHOPT;
 			if (ebx & (1U << 24)) type_ |= tCLWB;
 			if (ebx & (1U << 29)) type_ |= tSHA;
-			if (ecx & (1U << 0)) type_ |= tPREFETCHWT1;
 			if (ecx & (1U << 5)) type_ |= tWAITPKG;
 			if (ecx & (1U << 8)) type_ |= tGFNI;
 			if (ecx & (1U << 9)) type_ |= tVAES;
@@ -745,6 +743,7 @@ public:
 				if (eax & (1U << 31)) type_ |= tMOVRS;
 				if (edx & (1U << 4)) type_ |= tAVX_VNNI_INT8;
 				if (edx & (1U << 5)) type_ |= tAVX_NE_CONVERT;
+				if (edx & (1U << 8)) type_ |= tAMX_COMPLEX;
 				if (edx & (1U << 10)) type_ |= tAVX_VNNI_INT16;
 				if (edx & (1U << 14)) type_ |= tPREFETCHITI;
 				if (edx & (1U << 19)) type_ |= tAVX10;
@@ -1296,11 +1295,57 @@ inline uint32_t popcnt(uint64_t mask)
 #endif
 }

+// fall back to CPUID leaf 0x1A
+inline CoreType getCoreType()
+{
+	uint32_t data[4] = {};
+	Cpu::getCpuidEx(0x1A, 0, data);
+	const uint32_t coreTypeField = (data[0] >> 24) & 0xFF;
+	if (coreTypeField == 0x40) return Performance; // P-core
+	if (coreTypeField == 0x20) return Efficient; // E-core
+	return Standard;
+}
+
 #ifdef _WIN32

 typedef std::vector<uint32_t> U32Vec;
+
+#if (defined(NTDDI_VERSION) && NTDDI_VERSION >= 0x06010000) || (defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0601)
+	#define XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY 1
+#else
+	#define XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY 0
+#endif
+
+#if (defined(NTDDI_VERSION) && NTDDI_VERSION >= 0x0A000000) || (defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0A00)
+	#define XBYAK_WINSDK_HAS_EFFICIENCY_CLASS 1
+#else
+	#define XBYAK_WINSDK_HAS_EFFICIENCY_CLASS 0
+#endif
+
+// GroupMasks[] / GroupCount on CACHE_RELATIONSHIP added in Win10 20H1 (SDK 10.0.19041, NTDDI_WIN10_VB)
+// NOTE: _WIN32_WINNT has no sub-version granularity for Win10, so only
+// NTDDI_VERSION can distinguish 20H1 (0x0A00000C) from earlier Win10 builds.
+// If NTDDI_VERSION is not set, this macro will be 0 (safe/conservative fallback).
+#if defined(NTDDI_VERSION) && NTDDI_VERSION >= 0x0A00000C
+	#define XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS 1
+#else
+	#define XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS 0
+#endif
+
+#if XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY
 typedef SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX ProcInfo;

+inline CoreType getCoreTypeForAffinity(const GROUP_AFFINITY& affinity)
+{
+	GROUP_AFFINITY previousMask = {};
+	if (!SetThreadGroupAffinity(GetCurrentThread(), &affinity, &previousMask)) {
+		return Standard;
+	}
+	CoreType type = impl::getCoreType();
+	SetThreadGroupAffinity(GetCurrentThread(), &previousMask, NULL);
+	return type;
+}
+
 // return total logical cpus if sucessful, 0 if failed
 inline uint32_t getGroupAcc(U32Vec& v)
 {
@@ -1346,10 +1391,12 @@ static inline uint32_t getCores(std::vector<LogicalCpu>& cpus, bool isHybrid, co
 			cpu.coreId = coreIdx++;
 			if (!isHybrid) {
 				cpu.coreType = Standard;
-			} else if (core.EfficiencyClass > 0) {
-				cpu.coreType = Performance;
 			} else {
-				cpu.coreType = Efficient;
+#if XBYAK_WINSDK_HAS_EFFICIENCY_CLASS
+				cpu.coreType = core.EfficiencyClass > 0 ? Performance : Efficient;
+#else
+				cpu.coreType = getCoreTypeForAffinity(core.GroupMask[0]);
+#endif
 			}

 			const GROUP_AFFINITY* masks = core.GroupMask;
@@ -1374,13 +1421,19 @@ static inline uint32_t getCores(std::vector<LogicalCpu>& cpus, bool isHybrid, co

 inline bool convertMask(CpuMask& mask, const U32Vec& groupAcc, const CACHE_RELATIONSHIP& cache)
 {
-	const GROUP_AFFINITY* masks = cache.GroupMasks;
-
-	for (WORD i = 0; i < cache.GroupCount; i++) {
-		const WORD group = masks[i].Group;
-		const KAFFINITY m = masks[i].Mask;
-		const uint32_t base = groupAcc[group];
-
+#if XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS
+	const WORD count = cache.GroupCount;
+#else
+	const WORD count = 1;
+#endif
+	for (WORD i = 0; i < count; i++) {
+#if XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS
+		const GROUP_AFFINITY& cg = cache.GroupMasks[i];
+#else
+		const GROUP_AFFINITY& cg = cache.GroupMask;
+#endif
+		const KAFFINITY m = cg.Mask;
+		const uint32_t base = groupAcc[cg.Group];
 		for (uint32_t b = 0; b < sizeof(KAFFINITY) * 8; b++) {
 			if (m & (KAFFINITY(1) << b)) {
 				if (!mask.append(base + b)) return false;
@@ -1441,7 +1494,17 @@ inline bool initCpuTopology(CpuTopology& cpuTopo)
 	}
 	return true;
 }
-
+#else
+inline bool initCpuTopology(CpuTopology& cpuTopo)
+{
+	(void)cpuTopo;
+	return false;
+}
+#endif
+// unset WinSDK version macros to avoid Macro pollution
+#undef XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY
+#undef XBYAK_WINSDK_HAS_EFFICIENCY_CLASS
+#undef XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS
 #elif defined(__linux__) // Linux

 struct WrapFILE {
@@ -1471,6 +1534,15 @@ inline bool parseCpuList(CpuMask& mask, const char* path) {
 	return setStr(mask, buf);
 }

+inline CoreType setAffinityAndGetCoreType(uint32_t cpu)
+{
+	cpu_set_t cpuMask;
+	CPU_ZERO(&cpuMask);
+	CPU_SET(cpu, &cpuMask);
+	if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuMask)) return Standard;
+	return impl::getCoreType();
+}
+
 inline bool initCpuTopology(CpuTopology& cpuTopo)
 {
 	const uint32_t logicalCpuNum = sysconf(_SC_NPROCESSORS_ONLN);
@@ -1564,9 +1636,10 @@ inline bool initCpuTopology(CpuTopology& cpuTopo)
 	// Assign core types for hybrid architectures
 	const bool isHybrid = cpuTopo.isHybrid();
 	if (isHybrid) {
-		// For hybrid systems, read P-core and E-core lists from sysfs
+		// For hybrid systems, try toread P-core and E-core lists from sysfs first
 		CpuMask pCoreMask;
-		if (parseCpuList(pCoreMask, "/sys/devices/cpu_core/cpus")) {
+		const bool hasPCoreSysfs = parseCpuList(pCoreMask, "/sys/devices/cpu_core/cpus");
+		if (hasPCoreSysfs) {
 			// Set Performance core types
 			for (CpuMask::const_iterator it = pCoreMask.begin(); it != pCoreMask.end(); ++it) {
 				uint32_t cpuIdx = *it;
@@ -1576,7 +1649,8 @@ inline bool initCpuTopology(CpuTopology& cpuTopo)
 			}
 		}
 		CpuMask eCoreMask;
-		if (parseCpuList(eCoreMask, "/sys/devices/cpu_atom/cpus")) {
+		const bool hasECoreSysfs = parseCpuList(eCoreMask, "/sys/devices/cpu_atom/cpus");
+		if (hasECoreSysfs) {
 			// Set Efficient core types
 			for (CpuMask::const_iterator it = eCoreMask.begin(); it != eCoreMask.end(); ++it) {
 				uint32_t cpuIdx = *it;
@@ -1585,6 +1659,17 @@ inline bool initCpuTopology(CpuTopology& cpuTopo)
 				}
 			}
 		}
+		// Fallback: if either sysfs paths are unavailable, detect both core type per-CPU
+		if (!hasPCoreSysfs || !hasECoreSysfs) {
+			cpu_set_t originalMask;
+			CPU_ZERO(&originalMask);
+			if (sched_getaffinity(0, sizeof(cpu_set_t), &originalMask) == 0) {
+				for (uint32_t cpu = 0; cpu < logicalCpuNum; cpu++) {
+					cpuTopo.logicalCpus_[cpu].coreType = impl::setAffinityAndGetCoreType(cpu);
+				}
+				sched_setaffinity(0, sizeof(cpu_set_t), &originalMask);
+			}
+		}
 	}

 	// Read coherency line size
@@ -1645,8 +1730,6 @@ private:
 };

 #ifdef XBYAK64
-const int UseRCX = 1 << 6;
-const int UseRDX = 1 << 7;

 class Pack {
 	static const size_t maxTblNum = 15;
@@ -1745,28 +1828,35 @@ public:
 	}
 };

+// start from a bit position larger than the number of GPRs
+const int UseRBP = 1 << 5;
+const int UseRCX = 1 << 6;
+const int UseRDX = 1 << 7;
+const int UseRSI = 1 << 8;
+const int UseRDI = 1 << 9;
+const int UseRBPAsFramePointer = UseRBP | (1 << 10);
+
 class StackFrame {
 #ifdef XBYAK64_WIN
 	static const int noSaveNum = 6;
-	static const int rcxPos = 0;
-	static const int rdxPos = 1;
 #else
 	static const int noSaveNum = 8;
-	static const int rcxPos = 3;
-	static const int rdxPos = 2;
 #endif
+	static const int maxPnum = 4;
 	static const int maxRegNum = 14; // maxRegNum = 16 - rsp - rax
+	static const int calleeSaveNum = maxRegNum - noSaveNum;
+	static const int UseMASK = UseRCX|UseRDX|UseRSI|UseRDI|UseRBP;
 	Xbyak::CodeGenerator *code_;
-	Xbyak::Reg64 pTbl_[4];
+	Xbyak::Reg64 pTbl_[maxPnum];
 	Xbyak::Reg64 tTbl_[maxRegNum];
 	Pack p_;
 	Pack t_;
 	int pNum_;
 	int tNum_;
+	int useRegs_;
 	int saveNum_;
+	int saveRegs_[calleeSaveNum];
 	int P_;
-	bool useRcx_;
-	bool useRdx_;
 	bool makeEpilog_;
 	StackFrame(const StackFrame&);
 	void operator=(const StackFrame&);
@@ -1776,45 +1866,69 @@ public:
 	/*
 		make stack frame
 		@param sf [in] this
-		@param pNum [in] num of function parameter(0 <= pNum <= 4)
-		@param tNum [in] num of temporary register(0 <= tNum, with UseRCX, UseRDX) #{pNum + tNum [+rcx] + [rdx]} <= 14
+		@param pNum [in] number of function parameters(0 <= pNum <= 4)
+		@param tNum [in] number of temporary registers(0 <= tNum, can be OR-ed with Use{RCX,RDX,RSI,RDI,RBP}, e.g., 3|UseRCX)
 		@param stackSizeByte [in] local stack size
 		@param makeEpilog [in] automatically call close() if true

+		pNum + tNum + #Use must be <= 14
+
 		you can use
 		rax
-		gp0, ..., gp(pNum - 1)
-		gt0, ..., gt(tNum-1)
-		rcx if tNum & UseRCX
-		rdx if tNum & UseRDX
-		rsp[0..stackSizeByte - 1]
+		p[0], ..., p[pNum-1] as function parameters
+		t[0], ..., t[tNum-1] as temporary registers
+		{rcx,rdx,rsi,rdi,rbp} are explicitly available by specifying Use{RCX,RDX,RSI,RDI,RBP} in tNum
+		rsp[0..stackSizeByte-1] if stackSizeByte > 0
 	*/
 	StackFrame(Xbyak::CodeGenerator *code, int pNum, int tNum = 0, int stackSizeByte = 0, bool makeEpilog = true)
 		: code_(code)
 		, pNum_(pNum)
-		, tNum_(tNum & ~(UseRCX | UseRDX))
+		, tNum_(tNum & ~(UseMASK|UseRBPAsFramePointer))
+		, useRegs_(tNum & UseMASK) // drop UseRBPAsFramePointer bit
 		, saveNum_(0)
 		, P_(0)
-		, useRcx_((tNum & UseRCX) != 0)
-		, useRdx_((tNum & UseRDX) != 0)
 		, makeEpilog_(makeEpilog)
 		, p(p_)
 		, t(t_)
 	{
-		using namespace Xbyak;
 		if (pNum < 0 || pNum > 4) XBYAK_THROW(ERR_BAD_PNUM)
-		const int allRegNum = pNum + tNum_ + (useRcx_ ? 1 : 0) + (useRdx_ ? 1 : 0);
-		if (tNum_ < 0 || allRegNum > maxRegNum) XBYAK_THROW(ERR_BAD_TNUM)
-		const Reg64& _rsp = code->rsp;
-		saveNum_ = local::max_(0, allRegNum - noSaveNum);
-		const int *tbl = getOrderTbl() + noSaveNum;
-		for (int i = 0; i < saveNum_; i++) {
-			code->push(Reg64(tbl[i]));
+		if (tNum < 0) XBYAK_THROW(ERR_BAD_TNUM)
+		const int *const fullTbl = getRegEntryTbl();
+		const int *const calleeTbl = fullTbl + noSaveNum;
+		int callerUseNum = 0;
+		int calleeUseNum = 0;
+		for (int i = 0; i < maxRegNum; i++) {
+			if (useRegs_ & useFlagOf(fullTbl[i])) {
+				if (i < noSaveNum) {
+					callerUseNum++;
+				} else {
+					calleeUseNum++;
+				}
+			}
+		}
+		const int useNum = callerUseNum + calleeUseNum;
+		if (pNum + tNum_ + useNum > maxRegNum) XBYAK_THROW(ERR_BAD_TNUM)
+		const int baseSaveNum = local::max_(0, pNum + tNum_ + useNum - noSaveNum);
+		bool pushedRbp = false;
+		if (useRegs_ & UseRBP) {
+			code->push(rbp);
+			saveRegs_[saveNum_++] = Operand::RBP;
+			pushedRbp = true;
+			if ((tNum & UseRBPAsFramePointer) == UseRBPAsFramePointer) code->mov(rbp, rsp);
+		}
+		for (int i = 0; i < calleeSaveNum; i++) {
+			int r = calleeTbl[i];
+			if (i < baseSaveNum || isUseReg(r)) {
+				if (pushedRbp && r == Operand::RBP) continue;
+				saveRegs_[saveNum_++] = r;
+				code->push(Reg64(r));
+			}
 		}
 		P_ = (stackSizeByte + 7) / 8;
-		if (P_ > 0 && (P_ & 1) == (saveNum_ & 1)) P_++; // (rsp % 16) == 8, then increment P_ for 16 byte alignment
+		// (rsp % 16) == 8, then increment P_ for 16 byte alignment
+		if (P_ > 0 && (P_ & 1) == (saveNum_ & 1)) P_++;
 		P_ *= 8;
-		if (P_ > 0) code->sub(_rsp, P_);
+		if (P_ > 0) code->sub(rsp, P_);
 		int pos = 0;
 		for (int i = 0; i < pNum; i++) {
 			pTbl_[i] = Xbyak::Reg64(getRegIdx(pos));
@@ -1822,8 +1936,13 @@ public:
 		for (int i = 0; i < tNum_; i++) {
 			tTbl_[i] = Xbyak::Reg64(getRegIdx(pos));
 		}
-		if (useRcx_ && rcxPos < pNum) code_->mov(code_->r10, code_->rcx);
-		if (useRdx_ && rdxPos < pNum) code_->mov(code_->r11, code_->rdx);
+		// replace reserved reg with backup reg if needed
+		for (size_t i = 0; i < maxPnum; i++) {
+			const RegSlot& rp = getRegSlotTbl()[i];
+			if (isUseReg(rp.target) && rp.pos < pNum && rp.alt >= 0) {
+				code->mov(Xbyak::Reg64(rp.alt), Xbyak::Reg64(rp.target));
+			}
+		}
 		p_.init(pTbl_, pNum);
 		t_.init(tTbl_, tNum_);
 	}
@@ -1833,14 +1952,10 @@ public:
 	*/
 	void close(bool callRet = true)
 	{
-		using namespace Xbyak;
-		const Reg64& _rsp = code_->rsp;
-		const int *tbl = getOrderTbl() + noSaveNum;
-		if (P_ > 0) code_->add(_rsp, P_);
-		for (int i = 0; i < saveNum_; i++) {
-			code_->pop(Reg64(tbl[saveNum_ - 1 - i]));
+		if (P_ > 0) code_->add(code_->rsp, P_);
+		for (int i = saveNum_ - 1; i >= 0; i--) {
+			code_->pop(Reg64(saveRegs_[i]));
 		}
-
 		if (callRet) code_->ret();
 	}
 	~StackFrame()
@@ -1849,10 +1964,48 @@ public:
 		close();
 	}
 private:
-	const int *getOrderTbl() const
+	static int useFlagOf(int r)
 	{
-		using namespace Xbyak;
-		static const int tbl[] = {
+		switch (r) {
+		case Operand::RCX: return UseRCX;
+		case Operand::RDX: return UseRDX;
+		case Operand::RSI: return UseRSI;
+		case Operand::RDI: return UseRDI;
+		case Operand::RBP: return UseRBP;
+		default: return 0;
+		}
+	}
+	bool isUseReg(int r) const { return (useRegs_ & useFlagOf(r)) != 0; }
+	// Register allocation for the first 4 function parameters
+	struct RegSlot {
+		int target;
+		int pos; // position of target in getRegEntryTbl()
+		int alt; // alternative if target is used for parameter. -1 means no alternative.
+	};
+	const RegSlot *getRegSlotTbl() const
+	{
+		// Win: p[] = rcx(r10), rdx(r11), r8, r9:
+		// Linux: p[] = rdi(r8), rsi(r9), rdx(r11), rcx(r10)
+		// reg(alt) means a reserved reg if Use<reg> is used.
+
+		static const RegSlot tbl[maxPnum] = {
+#ifdef XBYAK64_WIN
+			{ Operand::RCX, 0, Operand::R10 },
+			{ Operand::RDX, 1, Operand::R11 },
+			{ Operand::RDI, 6, -1 },
+			{ Operand::RSI, 7, -1 },
+#else
+			{ Operand::RCX, 3, Operand::R10 },
+			{ Operand::RDX, 2, Operand::R11 },
+			{ Operand::RDI, 0, Operand::R8 },
+			{ Operand::RSI, 1, Operand::R9 },
+#endif
+		};
+		return tbl;
+	}
+	const int *getRegEntryTbl() const
+	{
+		static const int tbl[maxRegNum] = {
 #ifdef XBYAK64_WIN
 			Operand::RCX, Operand::RDX, Operand::R8, Operand::R9, Operand::R10, Operand::R11, Operand::RDI, Operand::RSI,
 #else
@@ -1862,21 +2015,28 @@ private:
 		};
 		return &tbl[0];
 	}
+	// get an available register index from tbl, skipping reserved registers
 	int getRegIdx(int& pos) const
 	{
-		assert(pos < maxRegNum);
-		using namespace Xbyak;
-		const int *tbl = getOrderTbl();
-		int r = tbl[pos++];
-		if (useRcx_) {
-			if (r == Operand::RCX) { return Operand::R10; }
-			if (r == Operand::R10) { r = tbl[pos++]; }
+		const int *tbl = getRegEntryTbl();
+		const RegSlot *slotTbl = getRegSlotTbl();
+		for (;;) {
+		NEXT:;
+			assert(pos < maxRegNum);
+			int r = tbl[pos++];
+			// if r is a Use*** target with alt, return alt as backup
+			// otherwise skip Use*** targets, their alts, and UseRBP's rbp
+			for (size_t i = 0; i < maxPnum; i++) {
+				const RegSlot& slot = slotTbl[i];
+				if (!isUseReg(slot.target)) continue;
+				if (r == slot.alt) goto NEXT;
+				if (r == slot.target) {
+					if (slot.alt >= 0) return slot.alt;
+					goto NEXT;
+				}
+			}
+			if (!isUseReg(r)) return r;
 		}
-		if (useRdx_) {
-			if (r == Operand::RDX) { return Operand::R11; }
-			if (r == Operand::R11) { return tbl[pos++]; }
-		}
-		return r;
 	}
 };
 #endif
@@ -31969,6 +31969,9 @@ SLKA-25176:
  name: "펌프 잇 업 익시드"
  name-en: "Pump It Up - Exceed"
  region: "NTSC-K"
+  gsHWFixes:
+    cpuSpriteRenderBW: 4 # Fixes graphical corruption in musical notes.
+    cpuSpriteRenderLevel: 2 # Needed for above.
 SLKA-25180:
  name: "스플린터 셀 - 판도라 투머로우"
  name-en: "Tom Clancy's Splinter Cell - Pandora Tomorrow"
@@ -44039,7 +44042,7 @@ SLPM-65495:
 SLPM-65496:
  name: "ハイパーストリートファイターⅡ アニバーサリーエディション"
  name-sort: "はいぱーすとりーとふぁいたー2 あにばーさりーえでぃしょん"
-  name-en: "Hyper Street Fighter"
+  name-en: "Hyper Street Fighter II - The Anniversary Edition"
  region: "NTSC-J"
 SLPM-65497:
  name: "ワールドサッカーウイニングイレブン7 インターナショナル プレミアムパッケージ supported by アディダス"
@@ -54506,6 +54509,7 @@ SLPM-74405:
  region: "NTSC-J"
  gameFixes:
    - EETimingHack # Stops crash after intro movie.
+    - InstantDMAHack # Stops hang when entering Sword Select screen.
 SLPM-74406:
  name: "WRC ～WORLD RALLY CHAMPIONSHIP～ [PlayStation2 the Best]"
  name-sort: "WRC ～わーるど らりー ちゃんぴおんしっぷ～ [PlayStation2 the Best]"
@@ -70115,6 +70119,9 @@ SLUS-21131:
  name: "Pump It Up - Exceed"
  region: "NTSC-U"
  compat: 5
+  gsHWFixes:
+    cpuSpriteRenderBW: 4 # Fixes graphical corruption in musical notes.
+    cpuSpriteRenderLevel: 2 # Needed for above.
 SLUS-21132:
  name: "Stella Deus - The Gate of Eternity"
  region: "NTSC-U"
@@ -1538,6 +1538,8 @@ xinput,XInput Controller,a:b0,b:b1,back:b6,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,
 05000000853200000503000000010000,Nacon MGX Pro,a:b0,b:b1,back:b10,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,guide:b12,leftshoulder:b6,leftstick:b13,lefttrigger:a5,leftx:a0,lefty:a1,rightshoulder:b7,rightstick:b14,righttrigger:a4,rightx:a2,righty:a3,start:b11,x:b3,y:b4,platform:Linux,
 0300000085320000170d000011010000,Nacon Revolution 5 Pro,a:b1,b:b2,back:b8,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,guide:b12,leftshoulder:b4,leftstick:b10,lefttrigger:a3,leftx:a0,lefty:a1,rightshoulder:b5,rightstick:b11,righttrigger:a4,rightx:a2,righty:a5,start:b9,touchpad:b13,x:b0,y:b3,platform:Linux,
 0300000085320000190d000011010000,Nacon Revolution 5 Pro,a:b1,b:b2,back:b8,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,guide:b12,leftshoulder:b4,leftstick:b10,lefttrigger:a3,leftx:a0,lefty:a1,rightshoulder:b5,rightstick:b11,righttrigger:a4,rightx:a2,righty:a5,start:b9,touchpad:b13,x:b0,y:b3,platform:Linux,
+0300000085320000180d000011010000,Nacon Revolution 5 Pro,a:b1,b:b2,back:b8,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,guide:b12,leftshoulder:b4,leftstick:b10,lefttrigger:a3,leftx:a0,lefty:a1,rightshoulder:b5,rightstick:b11,righttrigger:a4,rightx:a2,righty:a5,start:b9,x:b0,y:b3,platform:Linux,
+0300000085320000160d000011010000,Nacon Revolution 5 Pro,a:b1,b:b2,back:b8,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,guide:b12,leftshoulder:b4,leftstick:b10,lefttrigger:a3,leftx:a0,lefty:a1,rightshoulder:b5,rightstick:b11,righttrigger:a4,rightx:a2,righty:a5,start:b9,touchpad:b13,x:b0,y:b3,platform:Linux,
 030000000d0f00000900000010010000,Natec Genesis P44,a:b1,b:b2,back:b8,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,guide:b12,leftshoulder:b4,leftstick:b10,lefttrigger:b6,leftx:a0,lefty:a1,rightshoulder:b5,rightstick:b11,righttrigger:b7,rightx:a2,righty:a3,start:b9,x:b0,y:b3,platform:Linux,
 030000004f1f00000800000011010000,NeoGeo PS3,a:b1,b:b2,back:b8,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,guide:b12,leftshoulder:b4,lefttrigger:b6,leftx:a0,lefty:a1,rightshoulder:b5,righttrigger:b7,start:b9,x:b0,y:b3,platform:Linux,
 0300000092120000474e000000010000,NeoGeo X Arcade Stick,a:b0,b:b1,back:b8,dpdown:h0.4,dpleft:h0.8,dpright:h0.2,dpup:h0.1,start:b9,x:b3,y:b2,platform:Linux,
@@ -1729,6 +1729,11 @@ VS_OUTPUT vs_main_expand(uint vid : SV_VertexID)
 		}

 		vtx.interior = 0;
+
+		#if !VS_IIP
+			// Get the provoking vertex color (first vertex in DX)
+			vtx.c = i0 == 0 ? vtx.c : (i1 == 0 ? other.c : opposite.c);
+		#endif
 	}

 	return vtx;
@@ -308,6 +308,11 @@ void main()
 		}

 		VSout.interior = 0;
+
+		#if !VS_IIP
+			// Get the provoking vertex color (last vertex in GL)
+			vtx.c = i0 == 2 ? vtx.c : (i1 == 2 ? other.c : opposite.c);
+		#endif
 	}

 #endif
@@ -316,6 +316,11 @@ void main()
 		}

 		vsOut.interior = 0;
+
+		#if !VS_IIP
+			// Get the provoking vertex color (last vertex in VK)
+			vtx.c = i0 == 2 ? vtx.c : (i1 == 2 ? other.c : opposite.c);
+		#endif
 	}

 #endif
@@ -90,17 +90,15 @@ AchievementSettingsWidget::AchievementSettingsWidget(SettingsWindow* settings_di
 	}
 	else
 	{
-		// remove login and game info, not relevant for per-game
+		// hide login and game info, not relevant for per-game
 		m_ui.verticalLayout->removeWidget(m_ui.gameInfoBox);
-		m_ui.gameInfoBox->deleteLater();
-		m_ui.gameInfoBox = nullptr;
+		m_ui.gameInfoBox->hide();
 		m_ui.verticalLayout->removeWidget(m_ui.loginBox);
-		m_ui.loginBox->deleteLater();
-		m_ui.loginBox = nullptr;
+		m_ui.loginBox->hide();

 		// sound effects
 		m_ui.verticalLayout->removeWidget(m_ui.soundEffectsBox);
-		m_ui.soundEffectsBox->deleteLater();
+		m_ui.soundEffectsBox->hide();
 		m_ui.soundEffectsBox = nullptr;
 	}

@@ -121,9 +119,6 @@ void AchievementSettingsWidget::updateEnableState()
 	const bool unlock = enabled && sound && dialog()->getEffectiveBoolValue("Achievements", "UnlockSound", true);
 	const bool lbsound = enabled && sound && dialog()->getEffectiveBoolValue("Achievements", "LBSubmitSound", true);

-	m_ui.viewProfile->setEnabled(enabled);
-	m_ui.loginButton->setEnabled(enabled);
-
 	m_ui.hardcoreMode->setEnabled(enabled);
 	m_ui.achievementNotifications->setEnabled(enabled);
 	m_ui.leaderboardNotifications->setEnabled(enabled);
@@ -11,6 +11,28 @@
   </rect>
  </property>
  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <widget class="QLabel" name="rcheevosDisclaimer">
+     <property name="text">
+      <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p align=&quot;justify&quot;&gt;PCSX2 uses RetroAchievements as an achievement database and for tracking progress. To use achievements, please sign up for an account at &lt;a href=&quot;https://retroachievements.org/&quot;&gt;retroachievements.org&lt;/a&gt;.&lt;/p&gt;&lt;p align=&quot;justify&quot;&gt;To view the achievement list in-game, press the hotkey for &lt;span style=&quot; font-weight:600;&quot;&gt;Open Pause Menu&lt;/span&gt; and select &lt;span style=&quot; font-weight:600;&quot;&gt;Achievements&lt;/span&gt; from the menu.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+     </property>
+     <property name="textFormat">
+      <enum>Qt::TextFormat::RichText</enum>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+     <property name="margin">
+      <number>8</number>
+     </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
+     <property name="buddy">
+      <cstring>enable</cstring>
+     </property>
+    </widget>
+   </item>
   <item>
    <widget class="QGroupBox" name="loginBox">
     <property name="title">
@@ -18,32 +40,21 @@
     </property>
     <layout class="QHBoxLayout" name="horizontalLayout" stretch="0,0">
      <item>
-       <layout class="QGridLayout" name="acccountLayout">
-        <item row="0" column="0">
-         <widget class="QCheckBox" name="enable">
-          <property name="text">
-           <string>Enable Achievements</string>
-          </property>
-         </widget>
-        </item>
-        <item row="1" column="0">
-         <widget class="QLabel" name="loginStatus">
-          <property name="text">
-           <string>Username:
+       <widget class="QLabel" name="loginStatus">
+        <property name="text">
+         <string>Username:
 Login token generated at:</string>
-          </property>
-          <property name="alignment">
-           <set>Qt::AlignmentFlag::AlignLeading|Qt::AlignmentFlag::AlignLeft|Qt::AlignmentFlag::AlignTop</set>
-          </property>
-          <property name="textInteractionFlags">
-           <set>Qt::TextInteractionFlag::TextBrowserInteraction</set>
-          </property>
-          <property name="buddy">
-           <cstring>viewProfile</cstring>
-          </property>
-         </widget>
-        </item>
-       </layout>
+        </property>
+        <property name="alignment">
+         <set>Qt::AlignmentFlag::AlignLeading|Qt::AlignmentFlag::AlignLeft|Qt::AlignmentFlag::AlignTop</set>
+        </property>
+        <property name="textInteractionFlags">
+         <set>Qt::TextInteractionFlag::TextBrowserInteraction</set>
+        </property>
+        <property name="buddy">
+         <cstring>viewProfile</cstring>
+        </property>
+       </widget>
      </item>
      <item>
       <layout class="QHBoxLayout" name="achievementButtons">
@@ -94,62 +105,47 @@ Login token generated at:</string>
     </layout>
    </widget>
   </item>
-   <item>
-    <widget class="QLabel" name="rcheevosDisclaimer">
-     <property name="text">
-      <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p align=&quot;justify&quot;&gt;PCSX2 uses RetroAchievements as an achievement database and for tracking progress. To use achievements, please sign up for an account at &lt;a href=&quot;https://retroachievements.org/&quot;&gt;retroachievements.org&lt;/a&gt;.&lt;/p&gt;&lt;p align=&quot;justify&quot;&gt;To view the achievement list in-game, press the hotkey for &lt;span style=&quot; font-weight:600;&quot;&gt;Open Pause Menu&lt;/span&gt; and select &lt;span style=&quot; font-weight:600;&quot;&gt;Achievements&lt;/span&gt; from the menu.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
-     </property>
-     <property name="textFormat">
-      <enum>Qt::TextFormat::RichText</enum>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
-     </property>
-     <property name="margin">
-      <number>8</number>
-     </property>
-     <property name="openExternalLinks">
-      <bool>true</bool>
-     </property>
-     <property name="buddy">
-      <cstring>enable</cstring>
-     </property>
-    </widget>
-   </item>
   <item>
    <widget class="QGroupBox" name="settingsBox">
     <property name="title">
      <string>Settings</string>
     </property>
     <layout class="QGridLayout" name="gridLayout_5">
-      <item row="1" column="1">
-       <widget class="QCheckBox" name="unofficialAchievements">
+      <item row="0" column="0">
+       <widget class="QCheckBox" name="enable">
        <property name="text">
-         <string>Test Unofficial Achievements</string>
+         <string>Enable Achievements</string>
        </property>
       </widget>
      </item>
      <item row="0" column="1">
-       <widget class="QCheckBox" name="spectatorMode">
-        <property name="text">
-         <string>Enable Spectator Mode</string>
-        </property>
-       </widget>
-      </item>
-      <item row="0" column="0">
       <widget class="QCheckBox" name="hardcoreMode">
        <property name="text">
         <string>Enable Hardcore Mode</string>
        </property>
       </widget>
      </item>
-      <item row="1" column="0">
+      <item row="2" column="0">
+       <widget class="QCheckBox" name="spectatorMode">
+        <property name="text">
+         <string>Enable Spectator Mode</string>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
       <widget class="QCheckBox" name="encoreMode">
        <property name="text">
         <string>Enable Encore Mode</string>
        </property>
       </widget>
      </item>
+      <item row="3" column="0">
+       <widget class="QCheckBox" name="unofficialAchievements">
+        <property name="text">
+         <string>Test Unofficial Achievements</string>
+        </property>
+       </widget>
+      </item>
     </layout>
    </widget>
   </item>
@@ -537,9 +533,9 @@ Login token generated at:</string>
  </layout>
 </widget>
 <tabstops>
-  <tabstop>enable</tabstop>
  <tabstop>viewProfile</tabstop>
  <tabstop>loginButton</tabstop>
+  <tabstop>enable</tabstop>
  <tabstop>hardcoreMode</tabstop>
  <tabstop>spectatorMode</tabstop>
  <tabstop>encoreMode</tabstop>
@@ -876,7 +876,7 @@ static __fi void rcntEndGate(bool isVblank, u64 sCycle)
 					isVblank ? "vblank" : "hblank", counters[i].mode.GateMode, counters[i].count);
 				counters[i].count = 0;
 				counters[i].target &= 0xffff;
-				counters[i].startCycle = sCycle & ~(counters[i].rate - 1);
+				counters[i].startCycle = sCycle & ~(static_cast<u64>(counters[i].rate - 1));
 				break;
 		}
 	}
@@ -118,30 +118,12 @@ bool PCAPAdapter::recv(NetPacket* pkt)
 	if (!blocking && NetAdapter::recv(pkt))
 		return true;

-	pcap_pkthdr* header;
-	const u_char* pkt_data;
-
-	// pcap bridged will pick up packets not intended for us, returning false on those packets will incur a 1ms wait.
-	// This delays getting packets we need, so instead loop untill a valid packet, or no packet, is returned from pcap_next_ex.
-	while (pcap_next_ex(hpcap, &header, &pkt_data) > 0)
+	while (RecvPCAPPacket(pkt))
 	{
-		// 1518 is the largest Ethernet frame we can get using an MTU of 1500 (assuming no VLAN tagging).
-		// This includes the FCS, which should be trimmed (PS2 SDK dosn't allow extra space for this).
-		if (header->len > 1518)
-		{
-			Console.Error("DEV9: Dropped jumbo frame of size: %u", header->len);
-			continue;
-		}
-
-		pxAssert(header->len == header->caplen);
-
-		memcpy(pkt->buffer, pkt_data, header->len);
-		pkt->size = static_cast<int>(header->len);
-
 		if (!switched)
 			SetMACBridgedRecv(pkt);

-		if (VerifyPkt(pkt, header->len))
+		if (VerifyPkt(pkt, pkt->size))
 		{
 			HandleFrameCheckSequence(pkt);

@@ -173,10 +155,39 @@ bool PCAPAdapter::send(NetPacket* pkt)
 	if (!switched)
 		SetMACBridgedSend(pkt);

-	if (pcap_sendpacket(hpcap, (u_char*)pkt->buffer, pkt->size))
-		return false;
-	else
+	return SendPCAPPacket(pkt);
+}
+
+bool PCAPAdapter::RecvPCAPPacket(NetPacket* pkt)
+{
+	pcap_pkthdr* header;
+	const u_char* pkt_data;
+
+	// pcap bridged will pick up packets not intended for us, returning false on those packets will incur a 1ms wait.
+	// This delays getting packets we need, so instead loop untill a valid packet, or no packet, is returned from pcap_next_ex.
+	while (pcap_next_ex(hpcap, &header, &pkt_data) > 0)
+	{
+		// 1518 is the largest Ethernet frame we can get using an MTU of 1500 (assuming no VLAN tagging).
+		// This includes the FCS, which should be trimmed (PS2 SDK dosn't allow extra space for this).
+		if (header->len > 1518)
+		{
+			Console.Error("DEV9: Dropped jumbo frame of size: %u", header->len);
+			continue;
+		}
+
+		pxAssert(header->len == header->caplen);
+
+		memcpy(pkt->buffer, pkt_data, header->len);
+		pkt->size = static_cast<int>(header->len);
 		return true;
+	}
+
+	return false;
+}
+
+bool PCAPAdapter::SendPCAPPacket(const NetPacket* pkt)
+{
+	return pcap_sendpacket(hpcap, reinterpret_cast<const u_char*>(pkt->buffer), pkt->size) == 0;
 }

 void PCAPAdapter::reloadSettings()
@@ -39,6 +39,9 @@ private:
 	bool InitPCAP(const std::string& adapter, bool promiscuous);
 	bool SetMACSwitchedFilter(PacketReader::MAC_Address mac);

+	bool RecvPCAPPacket(NetPacket* pkt);
+	bool SendPCAPPacket(const NetPacket* pkt);
+
 	void SetMACBridgedRecv(NetPacket* pkt);
 	void SetMACBridgedSend(NetPacket* pkt);

@@ -91,11 +91,11 @@ void GSDrawingContext::UpdateScissor()
 	scissor.in = rscissor + GSVector4i::cxpr(0, 0, 1, 1);

 	// Fixed-point scissor min/max, used for rejecting primitives which are entirely outside.
-	scissor.cull = rscissor.sll32<4>();
+	// Add half a pixel around the edges for upscaling and lines/points.
+	scissor.cull = rscissor.sll32<4>() + GSVector4i(-8, -8, 8, 8);

-	// Offset applied to vertices for culling, zw is for native resolution culling
-	// We want to round subpixels down, because at least one pixel gets filled per scanline.
-	scissor.xyof = GSVector4i::loadl(&XYOFFSET.U64).xyxy().sub32(GSVector4i::cxpr(0, 0, 15, 15));
+	// Offset applied to vertices for culling.
+	scissor.xyof = GSVector4i::loadl(&XYOFFSET.U64).xyxy();
 }

 GIFRegTEX0 GSDrawingContext::GetSizeFixedTEX0(const GSVector4& st, bool linear, bool mipmap) const
@@ -1279,12 +1279,12 @@ void GSState::DumpTransferList(const std::string& filename)
 			(*file) << std::endl;

 		// clear, EE->GS, or GS->GS
-		(*file) << LIST_ITEM << "type: " << (transfer.zero_clear ? "clear" : ((transfer.transfer_type == EEGS_TransferType::EE_to_GS) ? "EE_to_GS" : "GS_to_GS")) << std::endl;
+		(*file) << LIST_ITEM << "type: " << ((transfer.transfer_type == EEGS_TransferType::Clear) ? "clear" : ((transfer.transfer_type == EEGS_TransferType::EE_to_GS) ? "EE_to_GS" : "GS_to_GS")) << std::endl;

 		// Dump BITBLTBUF
 		(*file) << INDENT << "BITBLTBUF: " << OPEN_MAP;

-		const bool gs_to_gs = (transfer.transfer_type == EEGS_TransferType::GS_to_GS) && !transfer.zero_clear;
+		const bool gs_to_gs = (transfer.transfer_type == EEGS_TransferType::GS_to_GS) && transfer.transfer_type != EEGS_TransferType::Clear;

 		if (gs_to_gs)
 		{
@@ -1330,11 +1330,11 @@ void GSState::DumpTransferImages()
 		const GSUploadQueue& transfer = m_draw_transfers[i];

 		std::string filename;
-		if ((transfer.transfer_type == EEGS_TransferType::EE_to_GS) || transfer.zero_clear)
+		if ((transfer.transfer_type == EEGS_TransferType::EE_to_GS) || transfer.transfer_type == EEGS_TransferType::Clear)
 		{
 			// clear or EE->GS: only the destination info is relevant.
 			filename = GetDrawDumpPath("%05lld_transfer%02d_%s_%04x_%d_%s_%d_%d_%d_%d.png",
-				s_n, transfer_n++, (transfer.zero_clear ? "clear" : "EE_to_GS"), transfer.blit.DBP, transfer.blit.DBW,
+				s_n, transfer_n++, ((transfer.transfer_type == EEGS_TransferType::Clear) ? "clear" : "EE_to_GS"), transfer.blit.DBP, transfer.blit.DBW,
 				GSUtil::GetPSMName(transfer.blit.DPSM), transfer.rect.x, transfer.rect.y, transfer.rect.z, transfer.rect.w);
 		}
 		else
@@ -2666,7 +2666,6 @@ void GSState::FlushPrim()
 					GSVector4i* RESTRICT vert_ptr = (GSVector4i*)&m_vertex->buff[i];
 					GSVector4i v = vert_ptr[1];
 					v = v.xxxx().u16to32().sub32(m_xyof);
-					v = v.blend32<12>(v.sra32<4>());
 					m_vertex->xy[i & 3] = v;
 					m_vertex->xy_tail = unused;
 				}
@@ -2912,12 +2911,11 @@ void GSState::Write(const u8* mem, int len)
 			m_draw_transfers.pop_back();
 			transfer.rect = transfer.rect.runion(r);
 			transfer.draw = s_n;
-			transfer.zero_clear = false;
 			m_draw_transfers.push_back(transfer);
 		}
 		else
 		{
-			const GSUploadQueue new_transfer = {blit, r, s_n, false, EEGS_TransferType::EE_to_GS};
+			const GSUploadQueue new_transfer = {blit, s_n, r, EEGS_TransferType::EE_to_GS};
 			m_draw_transfers.push_back(new_transfer);
 		}

@@ -3112,12 +3110,11 @@ void GSState::Move()
 		m_draw_transfers.pop_back();
 		transfer.rect = transfer.rect.runion(r);
 		transfer.draw = s_n;
-		transfer.zero_clear = false;
 		m_draw_transfers.push_back(transfer);
 	}
 	else
 	{
-		const GSUploadQueue new_transfer = {m_env.BITBLTBUF, r, s_n, false, EEGS_TransferType::GS_to_GS};
+		const GSUploadQueue new_transfer = {m_env.BITBLTBUF, s_n, r, EEGS_TransferType::GS_to_GS};
 		m_draw_transfers.push_back(new_transfer);
 	}

@@ -3790,10 +3787,8 @@ void GSState::UpdateContext()

 void GSState::UpdateScissor()
 {
-	m_scissor_cull_min = m_context->scissor.cull.xyxy();
-	m_scissor_cull_max = m_context->scissor.cull.zwzw();
 	m_xyof = m_context->scissor.xyof;
-	m_scissor_invalid = !m_context->scissor.in.gt32(m_context->scissor.in.zwzw()).allfalse();
+	m_scissor_invalid = m_context->scissor.in.rempty();
 }

 void GSState::UpdateVertexKick()
@@ -5841,6 +5836,7 @@ template <u32 prim, bool auto_flush>
 __forceinline void GSState::VertexKick(u32 skip)
 {
 	constexpr u32 n = NumIndicesForPrim(prim);
+	constexpr int primclass = GSUtil::GetPrimClass(prim);
 	static_assert(n > 0);

 	pxAssert(m_vertex->tail < m_vertex->maxcount + 3);
@@ -5884,11 +5880,7 @@ __forceinline void GSState::VertexKick(u32 skip)

 	// We maintain the X/Y coordinates for the last 4 vertices, as well as the head for triangle fans, so we can compute
 	// the min/max, and cull degenerate triangles, which saves draws in some cases. Why 4? Mod 4 is cheaper than Mod 3.
-	// These vertices are a full vector containing <X_Fixed_Point, Y_Fixed_Point, X_Integer, Y_Integer>. We use the
-	// integer coordinates for culling at native resolution, and the fixed point for all others. The XY offset has to be
-	// applied, then we split it into the fixed/integer portions.
-	const GSVector4i xy_ofs = new_v1.xxxx().u16to32().sub32(m_xyof);
-	const GSVector4i xy = xy_ofs.blend32<12>(xy_ofs.sra32<4>());
+	const GSVector4i xy = new_v1.xxxx().u16to32().sub32(m_xyof);
 	m_vertex->xy[xy_tail & 3] = xy;

 	// Backup head for triangle fans so we can read it later, otherwise it'll get lost after the 4th vertex.
@@ -5918,73 +5910,66 @@ __forceinline void GSState::VertexKick(u32 skip)
 	// Skip draws when scissor is out of range (i.e. bottom-right is less than top-left), since everything will get clipped.
 	skip |= static_cast<u32>(m_scissor_invalid);

-	GSVector4i pmin, pmax;
+	GSVector4i bbox;
 	if (skip == 0)
 	{
 		const GSVector4i v0 = m_vertex->xy[(xy_tail - 1) & 3];
 		const GSVector4i v1 = m_vertex->xy[(xy_tail - 2) & 3];
 		const GSVector4i v2 = (prim == GS_TRIANGLEFAN) ? m_vertex->xyhead : m_vertex->xy[(xy_tail - 3) & 3];

-		switch (prim)
+		if constexpr (n == 1)
 		{
-			case GS_POINTLIST:
-				pmin = v0;
-				pmax = v0;
-				break;
-			case GS_LINELIST:
-			case GS_LINESTRIP:
-			case GS_SPRITE:
-				pmin = v0.min_i32(v1);
-				pmax = v0.max_i32(v1);
-				break;
-			case GS_TRIANGLELIST:
-			case GS_TRIANGLESTRIP:
-			case GS_TRIANGLEFAN:
-				pmin = v0.min_i32(v1.min_i32(v2));
-				pmax = v0.max_i32(v1.max_i32(v2));
-				break;
-			default:
-				break;
+			bbox = v0;
+		}
+		else if constexpr (n == 2)
+		{
+			bbox = v0.runion(v1);
+		}
+		else if constexpr (n == 3)
+		{
+			bbox = v0.runion(v1).runion(v2);
 		}

-		GSVector4i test = pmax.lt32(m_scissor_cull_min) | pmin.gt32(m_scissor_cull_max);
-
-		switch (prim)
+		if constexpr (primclass == GS_TRIANGLE_CLASS || primclass == GS_SPRITE_CLASS)
 		{
-			case GS_TRIANGLELIST:
-			case GS_TRIANGLESTRIP:
-			case GS_TRIANGLEFAN:
-			case GS_SPRITE:
+			if (m_nativeres)
 			{
-				// Discard degenerate triangles which don't cover at least one pixel. Since the vertices are in native
-				// resolution space, we can use the integer locations. When upscaling, we can't, because a primitive which
-				// does not span a single pixel at 1x may span multiple pixels at higher resolutions.
-				const GSVector4i degen_test = pmin.eq32(pmax);
-				test |= m_nativeres ? degen_test.zwzw() : degen_test;
+				// For triangles and sprites at native res take the interior pixel centers.
+				const GSVector4i interior = (bbox + GSVector4i(0xF, 0xF, -1, -1)) & GSVector4i(~0xF);
+				bbox = interior + GSVector4i(0, 0, 1, 1); // +1 to bottom/right so empty test works correctly.
+			}
+			else
+			{
+				// For upscaling, remove bottom/right subtexels.
+				bbox -= ((bbox & GSVector4i(0xF)) == GSVector4i(0)) & GSVector4i(0, 0, 1, 1);
+			}
+
+			// For AA1 triangles and lines, expand the bounds by 1 pixel on all sides.
+			// Note: redundant check for the AA1 flag to avoid calling a function if not needed.
+			if (PRIM->AA1 && IsCoverageAlphaSupported())
+			{
+				bbox += GSVector4i(-0x10, -0x10, 0x10, 0x10);
 			}
-			break;
-			default:
-				break;
 		}

-		switch (prim)
+		// Do scissor test.
+		const GSVector4i bbox_ex = bbox + GSVector4i(0, 0, 1, 1); // Exclusive coords for the scissor test.
+		const GSVector4i& scissor = m_context->scissor.cull;
+		u32 test = static_cast<u32>(!bbox_ex.rintersects(scissor));
+
+		// Test for empty bbox.
+		if constexpr (primclass == GS_TRIANGLE_CLASS || primclass == GS_SPRITE_CLASS)
 		{
-			case GS_TRIANGLELIST:
-			case GS_TRIANGLESTRIP:
-			case GS_TRIANGLEFAN:
-				test = (test | v0.eq64(v1)) | (v1.eq64(v2) | v0.eq64(v2));
-				break;
-			default:
-				break;
+			test |= static_cast<u32>(bbox.rempty());
 		}

-#ifndef ARCH_ARM64
-		// We only care about the xy passing the skip test. zw is the offset coordinates for native culling.
-		skip |= test.mask() & 0xff;
-#else
-		// mask() is slow on ARM, so just pull the bits out instead, thankfully we only care about the first 4 bytes.
-		skip |= (static_cast<u64>(test.extract64<0>()) & UINT64_C(0x8080808080808080)) != 0;
-#endif
+		// Test for degenerate triangle.
+		if constexpr (primclass == GS_TRIANGLE_CLASS)
+		{
+			test |= static_cast<u32>(v0.eq(v1)) | static_cast<u32>(v1.eq(v2)) | static_cast<u32>(v0.eq(v2));
+		}
+
+		skip |= test;
 	}

 	if (skip != 0)
@@ -6094,13 +6079,12 @@ __forceinline void GSState::VertexKick(u32 skip)
 			ASSUME(0);
 	}

-	// Update rectangle for the current draw. We can use the re-integer coordinates from min/max here.
-	const GSVector4i draw_min = pmin.zwzw();
-	const GSVector4i draw_max = pmax;
-	if (m_index->tail != n)
-		temp_draw_rect = temp_draw_rect.min_i32(draw_min).blend32<12>(temp_draw_rect.max_i32(draw_max));
+	// Update rectangle for the current draw. Needs exclusive endpoints.
+	const GSVector4i draw_rect = bbox.sra32<4>() + GSVector4i(0, 0, 1, 1);
+	if (m_vertex->tail != n)
+		temp_draw_rect = temp_draw_rect.runion(draw_rect);
 	else
-		temp_draw_rect = draw_min.blend32<12>(draw_max);
+		temp_draw_rect = draw_rect;
 	temp_draw_rect = temp_draw_rect.rintersect(m_context->scissor.in);

 	constexpr u32 max_vertices = MaxVerticesForPrim(prim);
@@ -131,8 +131,6 @@ protected:

 	GSVertex m_v = {};
 	float m_q = 1.0f;
-	GSVector4i m_scissor_cull_min = {};
-	GSVector4i m_scissor_cull_max = {};
 	GSVector4i m_xyof = {};
 	int  m_used_buffers_idx = 0;
 	int m_current_buffer_idx = 0;
@@ -260,15 +258,15 @@ public:
 	{
 		EE_to_GS,
 		GS_to_GS,
-		GS_to_EE
+		GS_to_EE,
+		Clear
 	};

 	struct GSUploadQueue
 	{
 		GIFRegBITBLTBUF blit;
-		GSVector4i rect;
 		u64 draw;
-		bool zero_clear;
+		GSVector4i rect;
 		EEGS_TransferType transfer_type;
 	};

@@ -339,3 +339,27 @@ const char* GSUtil::GetPSMName(int psm)
 	}
 	return "BAD_PSM";
 }
+
+bool GSUtil::IsValidPSM(int psm)
+{
+	switch (psm)
+	{
+		case PSMCT32:
+		case PSMCT24:
+		case PSMCT16:
+		case PSMCT16S:
+		case PSMT8:
+		case PSMT4:
+		case PSMT8H:
+		case PSMT4HL:
+		case PSMT4HH:
+		case PSMZ32:
+		case PSMZ24:
+		case PSMZ16:
+		case PSMZ16S:
+		case PSGPU24:
+			return true;
+		default:
+			return false;
+	}
+}
@@ -6,6 +6,7 @@
 #include "GS.h"
 #include "GSRegs.h"
 #include "GSPerfMon.h"
+#include <climits>

 class GSUtil
 {
@@ -28,6 +29,8 @@ public:
 	static const char* GetACName(u32 ac);
 	static const char* GetPerfMonCounterName(GSPerfMon::counter_t counter, bool hw = true);

+	static bool IsValidPSM(int psm);
+
 	static const u32* HasSharedBitsPtr(u32 dpsm);
 	static bool HasSharedBits(u32 spsm, const u32* ptr);
 	static bool HasSharedBits(u32 spsm, u32 dpsm);
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0+

 #include "GSDirtyRect.h"
+#include <climits>
 #include <vector>

 GSDirtyRect::GSDirtyRect() :
@@ -4,6 +4,7 @@
 #pragma once

 #include "common/AlignedMalloc.h"
+#include <climits>

 template <class T>
 struct Element
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Florin9doi	469a3d60b2	USB: Additional commands for MSAC-US1	2026-05-13 12:17:55 -04:00
Sean Brar	bb35bed104	DEV9: Isolate raw PCAP packet IO	2026-05-13 12:17:27 -04:00
chaoticgd	624acf5533	Qt: Fix crash when interacting with per-game achievement settings	2026-05-13 12:17:02 -04:00
SternXD	f71f0b9cb0	GS/ImGui: Update ImGui when resizing the swap chain This allows for the OSD and BPM to stay in sync when resizing heavily Co-authored-by: GovanifY <me@govanify.com>	2026-05-13 17:03:02 +02:00
SternXD	07279568a6	3rdparty/Deps: Various updates (#14420 ) ### Description of Changes Updated a number of dependencies and third-party libraries to newer versions / commits. ## Deps: * libpng `v1.6.56` -> `v1.6.58` * harfbuzz `14.0.0` -> `14.2.0` * agilitysdk `1.619.1` -> `1.619.2` * rapidyaml `v0.11.1` -> `v0.12.1` * shaderc `2026.1` -> `2026.2` ## 3rdparty: * vixl `e6076e9` -> `3fe1686` * cubeb `e495bee` -> `6ad18e2` * xbyak `v7.35.2` -> `v7.37` * CPUInfo `7364b49` -> `3681f0c` * LZMA/7zipSDK `25.00` -> `26.01` * fast_float `v8.1.0` -> `v8.2.5` ### Rationale behind Changes Security fixes and improvements are good ### Suggested Testing Steps Test everything (yaml, png, vulkan, cpu information in osd, audio etc) ### Did you use AI to help find, test, or implement this issue or feature? Nah. --------- Signed-off-by: SternXD <stern@sidestore.io>	2026-05-12 12:14:40 -04:00
TJnotJT	4ef84fa51c	GS/HW: Fix flat shaded color in AA1 shader.	2026-05-12 14:46:49 +02:00
TheLastRar	38ee1e1e22	GS/DX: Specify shader model directly instead of via feature level	2026-05-11 22:29:45 +02:00
PCSX2 Bot	2b02a5dc78	[ci skip] PAD: Update to latest controller database.	2026-05-11 13:30:19 -04:00
descawed	dead00eb62	GameDB: Enable Instant DMA Hack for SLPM-74405 to fix hang	2026-05-10 19:07:21 +02:00
TJnotJT	4fa2b8e45d	GS/HW: Fixes/improvements to early scissor/bbox test.	2026-05-09 13:48:42 +02:00
$refractionpcsx2$ refractionpcsx2	fc90973509	GS/HW/COLCLIP: Check tex mapping is enabled when checking for recursion	2026-05-09 13:47:58 +02:00
Immersion95	ff9176f83e	GameDB: fixes wrong title for SLPM-65496 - Hyper Street Fighter II - The Anniversary Edition	2026-05-09 13:46:48 +02:00
$refractionpcsx2$ refractionpcsx2	63d3dd9978	GS/TC: Update dirty target linked to source when dirty overlaps the read	2026-05-09 13:46:25 +02:00
JordanTheToaster	b704c6caaa	GameDB: Pump It Up Exceed Fixes	2026-05-08 18:56:51 +02:00
lightningterror	a64465681c	GS/HW: Disable texture shuffle if there's no tex.	2026-05-08 18:29:05 +02:00
$refractionpcsx2$ refractionpcsx2	539357436c	Build: Clean up a few warnings	2026-05-08 18:17:31 +02:00
$refractionpcsx2$ refractionpcsx2	22cb6b0c9c	GS/HW: Remove zero clear from GSUploadQueue, use transfer_type instead	2026-05-08 18:15:18 +02:00
PCSX2 Bot	81df4e5def	[ci skip] Qt: Update Base Translation.	2026-05-07 02:15:53 +02:00
Ziemas	a3f3e8733d	GS: fix build with libstdc++ 16	2026-05-06 18:05:51 -04:00
SternXD	6fc84f76bb	FullscreenUI: Fix RA login state, and add dismiss dialog	2026-05-06 20:55:44 +02:00
TJnotJT	0949f193b8	GS/SW: Adjust bounds in assertion to present false positive. Account for the fact that bottom/right coordinates are exclusive.	2026-05-06 20:46:51 +02:00
TJnotJT	5f3dc1a76e	GS/SW: Early exit optimization for multithreaded horizontal lines.	2026-05-06 20:46:51 +02:00
TJnotJT	9afea55bb6	GS/SW: Adjust bbox to account for AA1 and points/line rounding.	2026-05-06 20:46:51 +02:00
TJnotJT	433614ae85	GS: Adjust primitive bbox in vertex kick to account for AA1.	2026-05-06 20:46:51 +02:00
TJnotJT	00c85c4a8f	GS/TC: Use temporary source on invalid TEX0.	2026-05-06 19:18:04 +02:00
PCSX2 Bot	f9885315bd	[ci skip] Qt: Update Base Translation.	2026-05-06 11:59:56 +02:00