PGD: Don't trust the header's data size, or a guest MAC key's pad size

- npdrmRead clamped the read with a signed compare against data_size, so a
  forged header with data_size >= 2^31 turned it into a ~2 GB copy past the
  guest's buffer. Clamp to what's left of the file, unsigned.
- sceDrmBBMacUpdate/Final take a MAC key in guest memory, and only checked
  pad_size > 16, so a negative one wrote at an arbitrary offset.
- Repeating the PGD ioctl leaked the previous descriptor, and pgd_open
  didn't check its allocation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Core/HLE/sceIo.cpp

@@ -1048,8 +1048,13 @@ static u32 npdrmRead(FileNode *f, u8 *data, int size) {
 	block  = pgd->file_offset/pgd->block_size;
 	offset = pgd->file_offset%pgd->block_size;
 
-	if (size > (int)pgd->data_size)
-		size = (int)pgd->data_size;
+	// Unsigned: data_size comes from the (forgeable) header, and a value over 2^31 turned the
+	// clamp into a huge read. Only what's left of the file can be read, too.
+	const u32 available = pgd->file_offset < pgd->data_size ? pgd->data_size - pgd->file_offset : 0;
+	if (size < 0)
+		size = 0;
+	if ((u32)size > available)
+		size = (int)available;
 	remain_size = size;
 
 	KirkState *kirk = __ChnnlsvKirkState();
@@ -3093,6 +3098,11 @@ int __IoIoctl(u32 id, u32 cmd, u32 indataPtr, u32 inlen, u32 outdataPtr, u32 out
 		pspFileSystem.SeekFile(f->handle, (s32)f->pgd_offset, FILEMOVE_BEGIN);
 		pspFileSystem.ReadFile(f->handle, pgd_header, 0x90);
 		KirkState *kirk = __ChnnlsvKirkState();
+		// Calling this again would otherwise leak the previous one.
+		if (f->pgdInfo) {
+			pgd_close(f->pgdInfo);
+			f->pgdInfo = nullptr;
+		}
 		f->pgdInfo = pgd_open(kirk, pgd_header, 2, key_ptr);
 		if (!f->pgdInfo) {
 			f->npdrm = false;

ext/libkirk/amctrl.c

@@ -154,7 +154,8 @@ int sceDrmBBMacUpdate(KirkState *kirk, MAC_KEY *mkey, u8 *buf, int size)
 	int retv = 0, ksize, p, type;
 	u8 *kbuf;
 
-	if(mkey->pad_size>16){
+	// The key can live in guest memory (sceDrmBBMacUpdate), so pad_size can be anything.
+	if(mkey->pad_size<0 || mkey->pad_size>16){
 		retv = 0x80510302;
 		goto _exit;
 	}
@@ -204,7 +205,7 @@ int sceDrmBBMacFinal(KirkState *kirk, MAC_KEY *mkey, u8 *buf, u8 *vkey)
 	u8 *kbuf, tmp[16], tmp1[16];
 	u32 t0, v0, v1;
 
-	if(mkey->pad_size>16)
+	if(mkey->pad_size<0 || mkey->pad_size>16)
 		return 0x80510302;
 
 	code = (mkey->type==2)? 0x3A : 0x38;
@@ -609,6 +610,8 @@ PGD_DESC *pgd_open(KirkState *kirk, u8 *pgd_buf, int pgd_flag, u8 *pgd_vkey)
 	//DEBUG_LOG(Log::HLE, "Open PGD ...");
 
 	pgd = (PGD_DESC*)malloc(sizeof(PGD_DESC));
+	if (!pgd)
+		return NULL;
 	memset(pgd, 0, sizeof(PGD_DESC));
 
 	pgd->key_index = *(u32*)(pgd_buf+4);