Add 7z streaming support, so we can have a progress bar

Common/File/VFS/SevenZipFileReader.cpp

@@ -1,4 +1,5 @@
 #include <algorithm>
+#include <array>
 #include <cstdlib>
 #include <cstring>
 #include <set>
@@ -7,6 +8,8 @@
 #include "ext/lzma-sdk/7z.h"
 #include "ext/lzma-sdk/7zCrc.h"
 #include "ext/lzma-sdk/7zFile.h"
+#include "ext/lzma-sdk/Lzma2Dec.h"
+#include "ext/lzma-sdk/LzmaDec.h"
 
 #include "Common/Data/Encoding/Utf8.h"
 #include "Common/Log.h"
@@ -14,6 +17,12 @@
 #include "Common/File/VFS/SevenZipFileReader.h"
 
 static constexpr size_t SEVENZIP_LOOKBUF_SIZE = 1 << 14;
+static constexpr size_t SEVENZIP_STREAM_LOOKAHEAD = 1 << 18;
+static constexpr size_t SEVENZIP_SKIPBUF_SIZE = 1 << 15;
+
+static constexpr UInt32 SEVENZIP_METHOD_COPY = 0;
+static constexpr UInt32 SEVENZIP_METHOD_LZMA = 0x30101;
+static constexpr UInt32 SEVENZIP_METHOD_LZMA2 = 0x21;
 
 class SevenZipFileReference : public VFSFileReference {
 public:
@@ -22,15 +31,286 @@ public:
 
 class SevenZipOpenFile : public VFSOpenFile {
 public:
+	enum class Mode {
+		MEMORY,
+		STREAM_COPY,
+		STREAM_LZMA,
+		STREAM_LZMA2,
+	};
+
+	SevenZipOpenFile() {
+		LzmaDec_CONSTRUCT(&lzma_);
+		Lzma2Dec_CONSTRUCT(&lzma2_);
+		archiveStream_.wres = 0;
+		File_Construct(&archiveStream_.file);
+		LookToRead2_INIT(&lookStream_);
+	}
+
 	~SevenZipOpenFile() override {
+		Release();
 		delete[] data;
 	}
 
+	void Release() {
+		if (lookStreamBuf_) {
+			free(lookStreamBuf_);
+			lookStreamBuf_ = nullptr;
+		}
+		if (lzmaAllocated_ && alloc_) {
+			LzmaDec_Free(&lzma_, alloc_);
+			lzmaAllocated_ = false;
+		}
+		if (lzma2Allocated_ && alloc_) {
+			Lzma2Dec_Free(&lzma2_, alloc_);
+			lzma2Allocated_ = false;
+		}
+		if (streamOpened_) {
+			File_Close(&archiveStream_.file);
+			streamOpened_ = false;
+		}
+	}
+
+	Mode mode = Mode::MEMORY;
+
 	uint8_t *data = nullptr;
 	size_t size = 0;
 	size_t offset = 0;
+
+	UInt32 fileIndex = 0;
+	UInt64 bytesToSkip = 0;
+	UInt64 fileRemaining = 0;
+	UInt64 packRemaining = 0;
+	UInt64 folderRemaining = 0;
+	bool streamError = false;
+	bool hasFileCRC = false;
+	UInt32 expectedFileCRC = 0;
+	UInt32 runningCRC = CRC_INIT_VAL;
+	bool crcChecked = false;
+
+	ISzAlloc *alloc_ = nullptr;
+	CFileInStream archiveStream_;
+	CLookToRead2 lookStream_;
+	Byte *lookStreamBuf_ = nullptr;
+	bool streamOpened_ = false;
+	CLzmaDec lzma_;
+	CLzma2Dec lzma2_;
+	bool lzmaAllocated_ = false;
+	bool lzma2Allocated_ = false;
+	std::array<uint8_t, SEVENZIP_SKIPBUF_SIZE> skipBuf_{};
 };
 
+static bool InitStreamingOpenFile(
+	const Path &archivePath,
+	const CSzArEx &db,
+	ISzAlloc *alloc,
+	UInt32 fileIndex,
+	SevenZipOpenFile *openFile) {
+	openFile->Release();
+	openFile->mode = SevenZipOpenFile::Mode::MEMORY;
+	openFile->alloc_ = alloc;
+	openFile->fileIndex = fileIndex;
+
+	const UInt32 folderIndex = db.FileToFolder[fileIndex];
+	if (folderIndex == (UInt32)-1) {
+		openFile->mode = SevenZipOpenFile::Mode::STREAM_COPY;
+		openFile->bytesToSkip = 0;
+		openFile->fileRemaining = 0;
+		openFile->folderRemaining = 0;
+		openFile->packRemaining = 0;
+		openFile->size = 0;
+		openFile->offset = 0;
+		return true;
+	}
+
+	const Byte *folderData = db.db.CodersData + db.db.FoCodersOffsets[folderIndex];
+	CSzData sd;
+	sd.Data = folderData;
+	sd.Size = db.db.FoCodersOffsets[(size_t)folderIndex + 1] - db.db.FoCodersOffsets[folderIndex];
+	CSzFolder folder;
+	if (SzGetNextFolderItem(&folder, &sd) != SZ_OK || sd.Size != 0) {
+		return false;
+	}
+
+	if (folder.NumCoders != 1 || folder.NumPackStreams != 1 || folder.NumBonds != 0 || folder.PackStreams[0] != 0) {
+		return false;
+	}
+
+	const CSzCoderInfo &coder = folder.Coders[0];
+	if (coder.NumStreams != 1) {
+		return false;
+	}
+
+	SevenZipOpenFile::Mode mode = SevenZipOpenFile::Mode::MEMORY;
+	if (coder.MethodID == SEVENZIP_METHOD_COPY) {
+		mode = SevenZipOpenFile::Mode::STREAM_COPY;
+	} else if (coder.MethodID == SEVENZIP_METHOD_LZMA) {
+		mode = SevenZipOpenFile::Mode::STREAM_LZMA;
+	} else if (coder.MethodID == SEVENZIP_METHOD_LZMA2) {
+		mode = SevenZipOpenFile::Mode::STREAM_LZMA2;
+	} else {
+		return false;
+	}
+
+	UInt64 packSize = 0;
+	const UInt64 *packPositions = db.db.PackPositions + db.db.FoStartPackStreamIndex[folderIndex];
+	packSize = packPositions[1] - packPositions[0];
+
+	const UInt64 unpackPos = db.UnpackPositions[fileIndex];
+	const UInt64 folderStartUnpackPos = db.UnpackPositions[db.FolderToFile[folderIndex]];
+	const UInt64 fileSize64 = db.UnpackPositions[(size_t)fileIndex + 1] - unpackPos;
+	const UInt64 folderSize64 = SzAr_GetFolderUnpackSize(&db.db, folderIndex);
+	const UInt64 fileOffsetInFolder = unpackPos - folderStartUnpackPos;
+
+	openFile->size = (size_t)fileSize64;
+	openFile->offset = 0;
+	openFile->mode = mode;
+	openFile->bytesToSkip = fileOffsetInFolder;
+	openFile->fileRemaining = fileSize64;
+	openFile->folderRemaining = folderSize64;
+	openFile->packRemaining = packSize;
+	openFile->streamError = false;
+	openFile->runningCRC = CRC_INIT_VAL;
+	openFile->crcChecked = false;
+	openFile->hasFileCRC = SzBitWithVals_Check(&db.CRCs, fileIndex);
+	openFile->expectedFileCRC = openFile->hasFileCRC ? db.CRCs.Vals[fileIndex] : 0;
+
+#ifdef _WIN32
+	if (InFile_OpenW(&openFile->archiveStream_.file, archivePath.ToWString().c_str()) != 0) {
+#else
+	if (InFile_Open(&openFile->archiveStream_.file, archivePath.ToString().c_str()) != 0) {
+#endif
+		return false;
+	}
+	openFile->streamOpened_ = true;
+
+	openFile->lookStreamBuf_ = (Byte *)malloc(SEVENZIP_LOOKBUF_SIZE * sizeof(Byte));
+	if (!openFile->lookStreamBuf_) {
+		openFile->Release();
+		return false;
+	}
+
+	openFile->lookStream_.bufSize = SEVENZIP_LOOKBUF_SIZE;
+	openFile->lookStream_.buf = openFile->lookStreamBuf_;
+	openFile->lookStream_.realStream = &openFile->archiveStream_.vt;
+	FileInStream_CreateVTable(&openFile->archiveStream_);
+	LookToRead2_CreateVTable(&openFile->lookStream_, False);
+	LookToRead2_INIT(&openFile->lookStream_);
+
+	const UInt64 streamPos = db.dataPos + packPositions[0];
+	if (LookInStream_SeekTo(&openFile->lookStream_.vt, streamPos) != SZ_OK) {
+		openFile->Release();
+		return false;
+	}
+
+	const Byte *props = folderData + coder.PropsOffset;
+	if (mode == SevenZipOpenFile::Mode::STREAM_LZMA) {
+		if (LzmaDec_Allocate(&openFile->lzma_, props, coder.PropsSize, alloc) != SZ_OK) {
+			openFile->Release();
+			return false;
+		}
+		openFile->lzmaAllocated_ = true;
+		LzmaDec_Init(&openFile->lzma_);
+	} else if (mode == SevenZipOpenFile::Mode::STREAM_LZMA2) {
+		if (coder.PropsSize != 1 || Lzma2Dec_Allocate(&openFile->lzma2_, props[0], alloc) != SZ_OK) {
+			openFile->Release();
+			return false;
+		}
+		openFile->lzma2Allocated_ = true;
+		Lzma2Dec_Init(&openFile->lzma2_);
+	}
+
+	return true;
+}
+
+static size_t StreamCopyRead(SevenZipOpenFile *openFile, uint8_t *dest, size_t length) {
+	size_t produced = 0;
+	while (produced < length && openFile->packRemaining > 0) {
+		const size_t wanted = (size_t)std::min<UInt64>((UInt64)(length - produced), openFile->packRemaining);
+		size_t lookahead = wanted;
+		const void *inBuf = nullptr;
+		const SRes lookRes = ILookInStream_Look(&openFile->lookStream_.vt, &inBuf, &lookahead);
+		if (lookRes != SZ_OK || lookahead == 0) {
+			openFile->streamError = true;
+			break;
+		}
+
+		memcpy(dest + produced, inBuf, lookahead);
+		if (ILookInStream_Skip(&openFile->lookStream_.vt, lookahead) != SZ_OK) {
+			openFile->streamError = true;
+			break;
+		}
+
+		openFile->packRemaining -= lookahead;
+		produced += lookahead;
+	}
+	return produced;
+}
+
+static size_t StreamDecodeRead(SevenZipOpenFile *openFile, uint8_t *dest, size_t length) {
+	size_t produced = 0;
+	while (produced < length && openFile->packRemaining > 0) {
+		size_t lookahead = (size_t)std::min<UInt64>(SEVENZIP_STREAM_LOOKAHEAD, openFile->packRemaining);
+		const void *inBuf = nullptr;
+		const SRes lookRes = ILookInStream_Look(&openFile->lookStream_.vt, &inBuf, &lookahead);
+		if (lookRes != SZ_OK || lookahead == 0) {
+			openFile->streamError = true;
+			break;
+		}
+
+		SizeT inProcessed = (SizeT)lookahead;
+		SizeT outProcessed = (SizeT)(length - produced);
+		ELzmaStatus status = LZMA_STATUS_NOT_SPECIFIED;
+		SRes decodeRes = SZ_ERROR_UNSUPPORTED;
+		if (openFile->mode == SevenZipOpenFile::Mode::STREAM_LZMA) {
+			decodeRes = LzmaDec_DecodeToBuf(&openFile->lzma_, dest + produced, &outProcessed, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_ANY, &status);
+		} else {
+			decodeRes = Lzma2Dec_DecodeToBuf(&openFile->lzma2_, dest + produced, &outProcessed, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_ANY, &status);
+		}
+
+		if (decodeRes != SZ_OK) {
+			openFile->streamError = true;
+			break;
+		}
+
+		if (inProcessed > 0) {
+			if (ILookInStream_Skip(&openFile->lookStream_.vt, inProcessed) != SZ_OK) {
+				openFile->streamError = true;
+				break;
+			}
+			openFile->packRemaining -= inProcessed;
+		}
+
+		produced += outProcessed;
+		if (outProcessed > 0) {
+			if (openFile->folderRemaining < outProcessed) {
+				openFile->streamError = true;
+				break;
+			}
+			openFile->folderRemaining -= outProcessed;
+		}
+
+		if (inProcessed == 0 && outProcessed == 0) {
+			openFile->streamError = true;
+			break;
+		}
+
+		if (status == LZMA_STATUS_FINISHED_WITH_MARK || status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) {
+			break;
+		}
+	}
+	return produced;
+}
+
+static size_t StreamReadChunk(SevenZipOpenFile *openFile, uint8_t *dest, size_t length) {
+	if (length == 0 || openFile->streamError) {
+		return 0;
+	}
+	if (openFile->mode == SevenZipOpenFile::Mode::STREAM_COPY) {
+		return StreamCopyRead(openFile, dest, length);
+	}
+	return StreamDecodeRead(openFile, dest, length);
+}
+
 void *SevenZipFileReader::Alloc(ISzAllocPtr, size_t size) {
 	return size == 0 ? nullptr : malloc(size);
 }
@@ -291,11 +571,16 @@ VFSOpenFile *SevenZipFileReader::OpenFileForRead(VFSFileReference *vfsReference,
 	}
 
 	SevenZipOpenFile *openFile = new SevenZipOpenFile();
-	openFile->data = ExtractFile(reference->index, &openFile->size);
-	if (!openFile->data) {
-		delete openFile;
-		return nullptr;
+	if (!InitStreamingOpenFile(archivePath_, db_, &allocImp_, reference->index, openFile)) {
+		// Fallback for unsupported folder graphs (for example BCJ2).
+		openFile->mode = SevenZipOpenFile::Mode::MEMORY;
+		openFile->data = ExtractFile(reference->index, &openFile->size);
+		if (!openFile->data) {
+			delete openFile;
+			return nullptr;
+		}
 	}
+
 	openFile->offset = 0;
 	*size = openFile->size;
 	return openFile;
@@ -303,11 +588,60 @@ VFSOpenFile *SevenZipFileReader::OpenFileForRead(VFSFileReference *vfsReference,
 
 void SevenZipFileReader::Rewind(VFSOpenFile *vfsOpenFile) {
 	SevenZipOpenFile *openFile = (SevenZipOpenFile *)vfsOpenFile;
+	if (openFile->mode != SevenZipOpenFile::Mode::MEMORY) {
+		if (!InitStreamingOpenFile(archivePath_, db_, &allocImp_, openFile->fileIndex, openFile)) {
+			openFile->streamError = true;
+		}
+		return;
+	}
 	openFile->offset = 0;
 }
 
 size_t SevenZipFileReader::Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) {
 	SevenZipOpenFile *openFile = (SevenZipOpenFile *)vfsOpenFile;
+	if (openFile->mode != SevenZipOpenFile::Mode::MEMORY) {
+		size_t produced = 0;
+		uint8_t *dest = (uint8_t *)buffer;
+
+		while (produced < length && openFile->fileRemaining > 0 && !openFile->streamError) {
+			if (openFile->bytesToSkip > 0) {
+				const size_t skipReq = (size_t)std::min<UInt64>(openFile->bytesToSkip, openFile->skipBuf_.size());
+				const size_t skipped = StreamReadChunk(openFile, openFile->skipBuf_.data(), skipReq);
+				if (skipped == 0) {
+					openFile->streamError = true;
+					break;
+				}
+				openFile->bytesToSkip -= skipped;
+				continue;
+			}
+
+			const size_t wanted = (size_t)std::min<UInt64>((UInt64)(length - produced), openFile->fileRemaining);
+			const size_t got = StreamReadChunk(openFile, dest + produced, wanted);
+			if (got == 0) {
+				openFile->streamError = true;
+				break;
+			}
+
+			if (openFile->hasFileCRC) {
+				openFile->runningCRC = CrcUpdate(openFile->runningCRC, dest + produced, got);
+			}
+
+			produced += got;
+			openFile->fileRemaining -= got;
+			openFile->offset += got;
+		}
+
+		if (openFile->fileRemaining == 0 && openFile->hasFileCRC && !openFile->crcChecked) {
+			openFile->crcChecked = true;
+			if (CRC_GET_DIGEST(openFile->runningCRC) != openFile->expectedFileCRC) {
+				openFile->streamError = true;
+				ERROR_LOG(Log::IO, "CRC mismatch while streaming '%s' from 7z archive", entries_[openFile->fileIndex].path.c_str());
+			}
+		}
+
+		return produced;
+	}
+
 	if (openFile->offset >= openFile->size) {
 		return 0;
 	}

Core/Util/GameManager.cpp

@@ -263,17 +263,83 @@ void GameManager::InstallZipContents(ZipFileTask task) {
 	if (task.zipFileInfo->archiveType == ArchiveType::SevenZ) {
 		Path destPath = task.destination;
 		g_OSD.SetProgressBar("install", di->T("Installing..."), 0.0f, 0.0f, 0.0f, 0.1f);
-		VFSInterface *reader = SevenZipFileReader::Create(task.fileName, "", true);
-		// TODO: We need streaming reads here.
-		size_t size;
-		u8 *data = reader->ReadFile(task.zipFileInfo->isoFilename, &size);
-
 		Path fn(task.zipFileInfo->isoFilename);
 		Path destFile = destPath / fn.GetFilename();
+		const size_t blockSize = 1024 * 128;
 
-		File::WriteDataToFile(false, data, size, destFile);
-		g_OSD.RemoveProgressBar("install", true, 0.5f);
+		SevenZipFileReader *reader = SevenZipFileReader::Create(task.fileName, "", true);
+		VFSFileReference *vfsRef = nullptr;
+		VFSOpenFile *vfsFile = nullptr;
+		FILE *f = nullptr;
+		bool success = false;
+		bool writeFailed = false;
+		bool readFailed = false;
+		size_t totalSize = 0;
+		size_t bytesCopied = 0;
+
+		if (reader) {
+			vfsRef = reader->GetFile(task.zipFileInfo->isoFilename);
+		}
+		if (vfsRef) {
+			vfsFile = reader->OpenFileForRead(vfsRef, &totalSize);
+		}
+		if (vfsFile) {
+			f = File::OpenCFile(destFile, "wb");
+		}
+
+		if (f) {
+			std::vector<u8> buffer(blockSize);
+			while (bytesCopied < totalSize) {
+				size_t readSize = std::min(blockSize, totalSize - bytesCopied);
+				size_t bytesRead = reader->Read(vfsFile, buffer.data(), readSize);
+				if (bytesRead == 0) {
+					ERROR_LOG(Log::HLE, "Failed to stream extract 7z ISO '%s'", task.zipFileInfo->isoFilename.c_str());
+					readFailed = true;
+					break;
+				}
+
+				size_t written = fwrite(buffer.data(), 1, bytesRead, f);
+				if (written != bytesRead) {
+					ERROR_LOG(Log::HLE, "Wrote %d bytes out of %d - Disk full?", (int)written, (int)bytesRead);
+					writeFailed = true;
+					break;
+				}
+
+				bytesCopied += bytesRead;
+				installProgress_ = totalSize > 0 ? (float)bytesCopied / (float)totalSize : 1.0f;
+				g_OSD.SetProgressBar("install", di->T("Installing..."), 0.0f, 1.0f, installProgress_, 0.1f);
+			}
+
+			success = bytesCopied == totalSize;
+			fclose(f);
+			f = nullptr;
+		}
+
+		if (vfsFile) {
+			reader->CloseFile(vfsFile);
+		}
+		if (vfsRef) {
+			reader->ReleaseFile(vfsRef);
+		}
+		delete reader;
+
+		if (!success) {
+			File::Delete(destFile);
+			if (writeFailed) {
+				SetInstallError(sy->T("Storage full"));
+			} else if (readFailed) {
+				auto iz = GetI18NCategory(I18NCat::INSTALLZIP);
+				SetInstallError(iz->T("Zip archive corrupt"));
+			} else {
+				SetInstallError(sy->T("Unable to open zip file"));
+			}
+		}
+
+		g_OSD.RemoveProgressBar("install", success, 0.5f);
 		installProgress_ = 1.0f;
+		if (success) {
+			ResetInstallError();
+		}
 		InstallDone();
 		return;
 	}