diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 2ae4d7c73e..0a8a876406 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -173,8 +173,8 @@ static const CommandLineParam g_autoParams[] = { {POFF(appendConfig), CmdParamType::String, "appendconfig", '\0', "Merge config FILE into the current configuration"}, {POFF(root), CmdParamType::String, "root", 'r', "Mount root directory"}, {POFF(stateToLoad), CmdParamType::String, "state", '\0', "Load state from specified file"}, - {POFF(compare), CmdParamType::Bool, "compare", 'c', "Enable comparison mode"}, - {POFF(bench), CmdParamType::Bool, "bench", 'b', "Enable benchmark mode"}, + {POFF(compare), CmdParamType::Bool, "compare", 'c', "Enable comparison mode", CmdLineMode::Headless}, + {POFF(bench), CmdParamType::Bool, "bench", 'b', "Enable benchmark mode", CmdLineMode::Headless}, {POFF(oldAtrac), CmdParamType::Bool, "old-atrac", '\0', "Use old ATRAC decoder"}, {POFF(log), CmdParamType::String, "log", '\0', "Output log to FILE"}, {POFF(screenshotFilename), CmdParamType::String, "screenshot", '\0', "Take a screenshot and save to FILE"}, @@ -188,7 +188,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(breakAction), CmdParamType::Enum, "break", '\0', "Set the action for break exceptions", CmdLineMode::Both, g_ExceptionActionValues, ARRAY_SIZE(g_ExceptionActionValues)}, {POFF(logNativeCrashes), CmdParamType::Bool, "log-native-crashes", '\0', "Log a native stack trace (Windows only) on an otherwise-unhandled crash", CmdLineMode::Both}, {POFF(verbose), CmdParamType::Bool, "verbose", '\0', "Enable verbose output", CmdLineMode::Both}, - + {POFF(printEqualLines), CmdParamType::Bool, "print-equal-lines", '\0', "Print lines that are equal during comparison", CmdLineMode::Headless}, // TODO: At some point we should maybe simply expose all config settings to be set directly from the command line automatically? }; diff --git a/Core/CmdLine.h b/Core/CmdLine.h index 82a724288e..ddca876adc 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -91,6 +91,7 @@ struct CommandLineOptions { std::optional bench; std::optional verbose; std::optional timeout; + std::optional printEqualLines; std::optional screenshotFilename; std::optional screenshotFilenameSave; diff --git a/headless/Compare.cpp b/headless/Compare.cpp index e0f6f470d8..a91c5fbf8d 100644 --- a/headless/Compare.cpp +++ b/headless/Compare.cpp @@ -189,7 +189,7 @@ std::string GetTestName(const Path &bootFilename) return ChopEnd(ChopFront(ChopFront(bootFilename.ToString(), "tests/"), "pspautotests/tests/"), ".prx"); } -bool CompareOutput(const Path &bootFilename, const std::string &output, bool verbose) { +bool CompareOutput(const Path &bootFilename, const std::string &output, bool verbose, bool printEqualLines) { Path expect_filename = bootFilename.GetFileExtension() == ".prx" ? bootFilename.WithReplacedExtension(".prx", ".expected") : bootFilename.WithExtraExtension(".expected"); std::unique_ptr expect_loader(ConstructFileLoader(expect_filename)); @@ -203,8 +203,14 @@ bool CompareOutput(const Path &bootFilename, const std::string &output, bool ver bool failed = false; while (expected.HasLines()) { - if (expected.Compare(actual)) + std::string value = expected.Peek(0); + if (expected.Compare(actual)) { // note: Compare actually advances if equal. This is pretty ugly. + if (printEqualLines) { + printf("= %s\n", value.c_str()); + } + // Lines were equal. continue; + } if (!failed) { GitHubActionsPrint("error", "Incorrect output for %s", currentTestName.c_str()); @@ -274,8 +280,8 @@ bool CompareOutput(const Path &bootFilename, const std::string &output, bool ver } } -static inline double CompareChannel(int pix1, int pix2) { - double diff = pix1 - pix2; +static inline float CompareChannel(int pix1, int pix2) { + float diff = pix1 - pix2; return diff * diff; } @@ -284,7 +290,6 @@ static inline double ComparePixel(u32 pix1, u32 pix2) { double r = CompareChannel(pix1 & 0xFF, pix2 & 0xFF); double g = CompareChannel((pix1 >> 8) & 0xFF, (pix2 >> 8) & 0xFF); double b = CompareChannel((pix1 >> 16) & 0xFF, (pix2 >> 16) & 0xFF); - return r + g + b; } diff --git a/headless/Compare.h b/headless/Compare.h index d7792713be..04ab899568 100644 --- a/headless/Compare.h +++ b/headless/Compare.h @@ -30,7 +30,7 @@ Path ExpectedFromFilename(const Path &bootFilename); Path ExpectedScreenshotFromFilename(const Path &bootFilename); std::string GetTestName(const Path &bootFilename); -bool CompareOutput(const Path &bootFilename, const std::string &output, bool verbose); +bool CompareOutput(const Path &bootFilename, const std::string &output, bool verbose, bool printEqualLines); std::vector TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 stride, u32 h); class ScreenshotComparer { diff --git a/headless/Headless.cpp b/headless/Headless.cpp index a8c17c7e26..e20b6de537 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -251,6 +251,7 @@ struct AutoTestOptions { bool compare; bool verbose; bool bench; + bool printEqualLines; }; static bool RunAutoTest(HeadlessHost *headlessHost, CoreParameter &coreParameter, const AutoTestOptions &opt) { @@ -355,7 +356,7 @@ static bool RunAutoTest(HeadlessHost *headlessHost, CoreParameter &coreParameter } if (opt.compare && passed) { - passed = CompareOutput(coreParameter.fileToStart, output, opt.verbose); + passed = CompareOutput(coreParameter.fileToStart, output, opt.verbose, opt.printEqualLines); } return passed; @@ -453,6 +454,7 @@ int main(int argc, const char* argv[]) { testOptions.bench = cmdLineOptions.bench.value_or(false); testOptions.timeout = cmdLineOptions.timeout.value_or(std::numeric_limits::infinity()); testOptions.verbose = cmdLineOptions.verbose.value_or(false); + testOptions.printEqualLines = cmdLineOptions.printEqualLines.value_or(false); bool fullLog = false; const char *stateToLoad = 0; @@ -724,8 +726,7 @@ int main(int argc, const char* argv[]) { if (testOptions.compare) { printf("%d tests passed, %d tests failed, %d tests missing.\n", (int)passedTests.size(), (int)failedTests.size(), (int)missingTests.size()); - if (!failedTests.empty()) - { + if (!failedTests.empty()) { printf("Failed tests:\n"); for (size_t i = 0; i < failedTests.size(); ++i) { printf(" %s\n", failedTests[i].c_str()); diff --git a/headless/README.md b/headless/README.md index 46aa3ebcb3..4ce27eaffc 100644 --- a/headless/README.md +++ b/headless/README.md @@ -1,6 +1,6 @@ # PPSSPPHeadless -Non-interactive, headless build of PPSSPP. It boots a PSP executable, PRX, or GE frame dump (`.ppdmp`) without a GUI, outputs emulated debug text to the console, optionally captures and compares screenshots, and exits. +Non-interactive, headless build of PPSSPP. It boots a PSP executable, PRX, or GE frame dump (`.ppdmp`) without a GUI, outputs emulated debug text to the console, optionally captures and compares text output or screenshots, and exits. Primarily intended for: - Automated regression testing (via [pspautotests](https://github.com/hrydgard/pspautotests/)) @@ -120,7 +120,8 @@ When the MSE exceeds `--max-mse`, the following files are saved in the working d The `--compare` flag also compares emulated debug output (`printf` / `sceIoWrite` to the emulator channel) against a `.expected` text file: - For `.prx` files: same path with `.expected` extension. -- The comparison is line-based with a diff algorithm that highlights insertions/deletions. +- The comparison is line-based with a diff algorithm that highlights mismatches and insertions/deletions, although the support for the + latter is limited, it's not a full-on diff. If only a screenshot reference exists and no `.expected` file, the test passes as long as there is no unexpected text output. @@ -157,13 +158,14 @@ Example workflow step: The following configuration is hardcoded for headless mode (config file is never saved): +(For actual up-to-date hardcoded config, see the headless.cpp file). + | Setting | Value | Notes | |----------------------|------------|------------------------------------------| | Internal resolution | 1× (480×272) | Fixed. | | Hardware transform | Enabled | | | Vertex decoder JIT | Enabled | | | Software renderer JIT| Enabled | | -| Skip GPU readback | No skip | Full readback for accurate screenshots. | | Ignore bad mem access| Enabled | | | Firmware version | 6.60 | | | PSP model | Slim | |