diff --git a/.gitignore b/.gitignore index 89b166c2e2..d58ac5312a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,9 @@ ppsspp.ini imgui.ini PPSSPPControls.dat +# Frametest output (see frametests.py) +frametest-out/ + # Gradle/Android Studio .gradle .idea @@ -147,5 +150,5 @@ RAPrefs_PPSSPP.cfg # For CLion cmake-build-*/ - -/.vscode/ + +/.vscode/ diff --git a/AGENTS.md b/AGENTS.md index 510478deaf..0675509672 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -152,6 +152,16 @@ pspautotests are a large set of tests of the PSP OS's API surface, and thus test See docs/pspautotests.md for a workflow for running pspautotests and improving PPSSPP with the results. +## Framedump rendering tests (frametests) + +There is a rendering test system that replays GE frame dumps (`.ppdmp`) through PPSSPPHeadless and compares +the output against reference images, driven by the `frametests.py` script and a JSON config per test set. +When changing rendering code, consider running these tests. See docs/frametest.md for full documentation. + +Note: `headless/Compare.cpp` reads back framebuffers top-down; the flip to bottom-up is only applied when +writing BMPs (and when reading BMP references). `TranslateDebugBufferToCompare` also exists as a copy in +`libretro/LibretroGraphicsContext.cpp` - keep the two in sync. + ## Adding HLE modules HLE module implementations live in `Core/HLE/sce.cpp` / `.h` (e.g. `sceOpenPSID.cpp`, `scePauth.cpp` are good diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index bbbb672cd8..86ebb822f2 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -181,7 +181,8 @@ static const CommandLineParam g_autoParams[] = { {POFF(log), CmdParamType::String, "log", '\0', "Output log to FILE", CmdLineMode::Application}, {POFF(enableLogging), CmdParamType::Bool, "log", '\0', "Full log output, not just emulated printfs", CmdLineMode::Headless}, {POFF(screenshotFilename), CmdParamType::String, "screenshot", '\0', "Compare rendered output against a reference screenshot FILE", CmdLineMode::Headless}, - {POFF(screenshotFilenameSave), CmdParamType::String, "screenshot-save", '\0', "Save rendered screenshot to specified path", CmdLineMode::Headless}, + {POFF(screenshotFilenameSave), CmdParamType::String, "screenshot-save", '\0', "Save rendered screenshot to specified path (PNG if the path ends in .png, BMP otherwise)", CmdLineMode::Headless}, + {POFF(screenshotFilenameDiff), CmdParamType::String, "screenshot-diff", '\0', "Save a visual comparison image to FILE when comparing screenshots", CmdLineMode::Headless}, {POFF(timeout), CmdParamType::Double, "timeout", '\0', "Set the timeout value", CmdLineMode::Headless}, {POFF(maxScreenshotError), CmdParamType::Double, "max-mse", '\0', "Maximum allowed MSE error for screenshot comparison", CmdLineMode::Headless}, {POFF(mountIso), CmdParamType::String, "mount", 'm', "Mount ISO/CSO on umd1:", CmdLineMode::Headless}, diff --git a/Core/CmdLine.h b/Core/CmdLine.h index 4d94bbc5cb..211b5853f4 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -95,6 +95,7 @@ struct CommandLineOptions { std::optional screenshotFilename; std::optional screenshotFilenameSave; + std::optional screenshotFilenameDiff; // Headless: mount an ISO/CSO on umd1:. std::optional mountIso; diff --git a/docs/frametest.md b/docs/frametest.md new file mode 100644 index 0000000000..d481ef4177 --- /dev/null +++ b/docs/frametest.md @@ -0,0 +1,191 @@ +# Framedump rendering tests (frametests) + +PPSSPP has a test system that replays GE frame dumps (`.ppdmp`) through +`PPSSPPHeadless` with various rendering settings, comparing the rendered +output against stored reference images. It is designed to catch rendering +regressions across GPU backends and configurations. + +The system has three parts: + +- **The test runner**: `frametests.py` (repo root). Test-set agnostic - it + works against any test data tree pointed to by its configuration. +- **The configuration**: a JSON file describing where the test data lives and + which rendering variants to run (e.g. `frametests/frametests.json`). +- **The test set**: the frame dumps and reference images (e.g. + `frametests/dumps/` and `frametests/ref/`). Different environments can use + different test sets - GitHub CI uses the one in the repo, custom machines + can use their own (possibly much larger) ones. + +## Quick start + +```bash +# Build PPSSPPHeadless (see headless/README.md), then: +python3 frametests.py frametests/frametests.json +``` + +The first run generates reference images for any dumps that don't have them +yet (status `NEW`) and compares the rest (status `PASS`/`FAIL`). A summary is +printed and a self-contained HTML report is written to +`frametests/frametest-out/report.html` (default output dir; gitignored). + +Exit code is `0` if all tests passed, `1` if anything failed, `2` on +configuration/usage errors. + +## Configuration + +The configuration is a single JSON file. All relative paths are resolved +against the config file's directory, so the same script works with any test +set by just pointing `--config` at a different file. + +```json +{ + "testRoot": "dumps", + "refRoot": "ref", + "outputRoot": "frametest-out", + "headlessPath": "", + "timeout": 60, + "maxMse": 0.0, + "variants": { + "soft": "--graphics=software" + } +} +``` + +| Key | Description | +|-----------------|-----------------------------------------------------------------------------| +| `testRoot` | Directory tree containing frame dumps (`.ppdmp` files, possibly wrapped in `.zip`). | +| `refRoot` | Where reference images live; mirrors the `testRoot` tree. | +| `outputRoot` | Where logs, diff images, the report, and generated references go. | +| `headlessPath` | Path to the `PPSSPPHeadless` binary. Empty = auto-detect (see below). | +| `timeout` | Per-test timeout in seconds. | +| `maxMse` | Maximum allowed MSE for screenshot comparison (0 = exact match). | +| `variants` | Map of variant name to command line arguments for the headless binary. | + +### Variants + +Each test runs once per variant. The variant name is appended to the +reference image filename, so each variant gets its own reference set: + +``` +dumps/Depth/11578 Virtua Tennis pause menu ULES00126_0002.zip + → ref/Depth/11578 Virtua Tennis pause menu ULES00126_0002-soft.png +``` + +The variant value is an arbitrary command line argument string passed to +`PPSSPPHeadless`, so new rendering configurations are just new entries (or a +different config file on a machine with the right GPU): + +```json +"variants": { + "soft": "--graphics=software", + "gl": "--graphics=opengl", + "gl-4x": "--graphics=opengl --resolution-scale=4" +} +``` + +## How a test runs + +For each dump (recursively under `testRoot`) and each variant: + +- If the reference image `-.png` is **missing**: the dump is + rendered and the output saved as the new reference. Status: `NEW`. This is + how references are created - run locally, then commit the generated images + (also copied to `/generated/` for convenience). +- If the reference **exists**: the dump is rendered, the output saved to + `/actuals/`, and compared against the reference using MSE + (mean squared error over R, G, B per pixel, alpha ignored). A visual + comparison image (actual / reference+diff) is saved to + `/diffs/` whenever a comparison runs. Status: `PASS` or `FAIL` + (mismatch, crash, or timeout). + +A `FAIL` with no MSE reported usually means the headless binary crashed +before producing a screenshot, and a reference image that can't be loaded +(corrupt) is reported as `ERROR`. The full emulator log of every non-passing +test is kept in `/logs/`. + +### Reference images + +- PNG, 512×272 (480×272 display in a 512-wide framebuffer), stored top-down + (row 0 = top of screen). The BMP output format is bottom-up per the BMP + spec; the flip is applied only when writing BMPs. +- Generated with `--graphics=software` they are fully deterministic: a + subsequent run produces byte-identical output, so `maxMse` can be 0. +- If rendering code changes the output, existing references may need + regenerating: delete the affected reference images and re-run to regenerate + them. + +### Headless flags used + +The runner uses `--screenshot=` (compare), `--screenshot-save=` +(save output; PNG if the path ends in `.png`, else BMP) and +`--screenshot-diff=` (always write a visual comparison when comparing). +See `headless/README.md` for details. + +## Command line options + +``` +usage: frametests.py [OPTIONS] [CONFIG.json] +``` + +| Option | Description | +|--------------------|-----------------------------------------------------------------------| +| `CONFIG.json` | Path to the configuration file (default: `frametests/frametests.json`). | +| `--filter=SUBSTR` | Only run dumps whose relative path contains SUBSTR (case-insensitive). | +| `--strict` | Treat missing reference images as failures (a configuration error). | +| `--out-mode=all\|failures` | `all` keeps everything in the output dir; `failures` keeps only artifacts of non-passing tests (smaller CI artifacts). Default: `all`. | + +The headless binary is located, in order of preference: + +1. `headlessPath` from the config file. +2. The `PPSSPP_HEADLESS` environment variable. +3. Well-known paths relative to the current directory (e.g. + `Windows/x64/Debug/PPSSPPHeadless.exe`, `build/PPSSPPHeadless`), newest + by modification time. + +## CI integration + +In CI (`GITHUB_ACTIONS` is set) the runner behaves as if `--strict` was +passed and prints `::error` annotations for each failing test, so failures +show up inline in the GitHub Actions log. A typical CI job: + +```yaml +- name: Build headless + run: ./b.sh --headless +- name: Run frametests + run: python3 frametests.py frametests/frametests.json --out-mode=failures +- name: Upload report + uses: actions/upload-artifact@v4 + with: + name: frametest-report + path: frametests/frametest-out/ +``` + +A missing reference image in CI means the test set is incomplete - the run +fails and the generated reference is available in +`frametests/frametest-out/generated/` (part of the uploaded artifact) so it +can be committed. + +## Custom machines + +The script is designed to run on machines with different GPUs and bigger test +sets than CI. Copy the repo, point at a custom config: + +```bash +python3 frametests.py /path/to/my-config.json +``` + +The config can live anywhere and point at any test data tree; only the +script itself is shared. Add hardware-specific variants (`--graphics=vulkan`, +`--msaa=...`, etc.) to the machine's own config - no code changes needed for +new flag combinations, as long as the headless binary supports them. + +## Adding a new framedump + +1. Drop the dump into `frametests/dumps/` (either as a `.ppdmp` or zipped). +2. Run `python3 frametests.py frametests/frametests.json --filter=` - + the reference image(s) are generated and copied to `/generated/`. +3. Commit the dump and the reference images under `frametests/ref/`. + +The CI test set (dumps and references) is maintained separately from the +runner; the `frametests/` directory is just one of several possible test sets +and may become a git submodule at some point. diff --git a/frametests.py b/frametests.py new file mode 100644 index 0000000000..d075bd39e7 --- /dev/null +++ b/frametests.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Framedump rendering test runner for PPSSPPHeadless. + +Runs GE frame dumps (".ppdmp", possibly zipped) through PPSSPPHeadless with a +set of rendering variants (command line option sets), generating reference +images when missing and comparing against existing ones when present. Produces +a self-contained HTML report and returns a nonzero exit code on failure. + +This script is test-set agnostic: it reads its configuration from a JSON file +(similar to frametests/frametests.json) that points at the test data tree, +so it can be used against different test sets (the CI one, or bigger private +ones on custom machines). + +Usage: + frametests.py [OPTIONS] [CONFIG.json] + +Example: + python3 frametests.py frametests/frametests.json +""" + +import argparse +import base64 +import glob +import html +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import time +from pathlib import Path + +# test.py-style candidate paths for the headless binary, relative to the +# current working directory, in preference order. +HEADLESS_CANDIDATES = [ + "Windows/x64/Debug/PPSSPPHeadless.exe", + "Windows/Debug/PPSSPPHeadless.exe", + "Windows/x64/Release/PPSSPPHeadless.exe", + "Windows/Release/PPSSPPHeadless.exe", + "build/PPSSPPHeadless", + "build-headless/PPSSPPHeadless", + "build*/PPSSPPHeadless", + "PPSSPPHeadless", + "ppsspp/PPSSPPHeadless", +] + +MSE_RE = re.compile(r"Screenshot MSE: ([0-9.eE+-]+)") +LOG_EMBED_LIMIT = 64 * 1024 + +STATUS_PASS = "PASS" +STATUS_FAIL = "FAIL" +STATUS_NEW = "NEW" +STATUS_ERROR = "ERROR" + + +def find_headless(config_dir, config_path): + """Locate the PPSSPPHeadless binary. The config headlessPath wins, then + the PPSSPP_HEADLESS env var, then candidate paths (newest by mtime).""" + candidates = [] + if config_path: + p = Path(config_path) + candidates.append(p if p.is_absolute() else config_dir / p) + env_path = os.environ.get("PPSSPP_HEADLESS") + if env_path: + candidates.append(Path(env_path)) + for pattern in HEADLESS_CANDIDATES: + candidates.extend(Path(m) for m in glob.glob(pattern)) + found = [c for c in candidates if c.is_file()] + if not found: + return None + return max(found, key=lambda p: p.stat().st_mtime) + + +def strip_dump_extensions(name): + """Derive the base name for a dump, stripping .zip and .ppdmp extensions.""" + base = name + lower = base.lower() + if lower.endswith(".zip"): + base = base[:-4] + lower = base.lower() + if lower.endswith(".ppdmp"): + base = base[:-6] + return base + + +def collect_dumps(test_root): + """Recursively collect frame dumps (.ppdmp files and .zip wrappers).""" + dumps = [] + for dirpath, dirnames, filenames in os.walk(test_root): + dirnames[:] = [d for d in sorted(dirnames) if not d.startswith(".")] + for filename in sorted(filenames): + lower = filename.lower() + if lower.endswith(".ppdmp") or lower.endswith(".zip"): + dumps.append(Path(dirpath) / filename) + return sorted(dumps) + + +def run_test(headless, dump_path, variant_args, ref_path, actual_path, diff_path, max_mse, timeout, output_dir): + """Run one dump with one variant's args. Returns (status, mse, output, timed_out).""" + generate = not ref_path.exists() + args = [str(headless)] + list(variant_args) + if generate: + args.append("--screenshot-save=" + str(ref_path)) + else: + args.extend([ + "--screenshot-save=" + str(actual_path), + "--screenshot=" + str(ref_path), + "--max-mse=" + str(max_mse), + "--screenshot-diff=" + str(diff_path), + ]) + args.append(str(dump_path)) + + timed_out = False + try: + proc = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd=str(output_dir), text=True, encoding="utf-8", errors="replace") + output, _ = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + output, _ = proc.communicate() + timed_out = True + returncode = proc.returncode + + # Headless may drop these in the cwd on failure (outside GITHUB_ACTIONS). + for stray in ("__testfailure.bmp", "__testcompare.png"): + try: + (output_dir / stray).unlink() + except OSError: + pass + + match = MSE_RE.search(output) + mse = float(match.group(1)) if match else None + + if timed_out: + return STATUS_FAIL, mse, output, True + if generate: + if returncode != 0 or not ref_path.exists(): + return STATUS_ERROR, mse, output, False + return STATUS_NEW, mse, output, False + if returncode != 0: + return STATUS_FAIL, mse, output, False + return STATUS_PASS, mse, output, False + + +def image_data_uri(path): + """Read an image file as a PNG data URI, or None if missing.""" + try: + data = path.read_bytes() + except OSError: + return None + return "data:image/png;base64," + base64.b64encode(data).decode("ascii") + + +def write_report(report_path, results, variants): + rows = [] + for res in results: + cls = "failed" if res["status"] in (STATUS_FAIL, STATUS_ERROR) else ("new" if res["status"] == STATUS_NEW else "") + mse = ('MSE: %.6f' % res["mse"]) if res["mse"] is not None else "" + detail = '%s' % html.escape(res["detail"]) if res["detail"] else "" + ref_img = 'reference' % res["ref_uri"] if res["ref_uri"] else "" + actual_img = 'actual' % res["actual_uri"] if res["actual_uri"] else "" + diff_img = 'diff' % res["diff_uri"] if res["diff_uri"] else "" + log = ('
log
%s
' % html.escape(res["log"])) if res["log"] else "" + rows.append("""
+
+ {status} + {variant} + {name} + {mse}{detail} +
+
+ {ref_img}{actual_img}{diff_img} +
+ {log} +
""".format( + cls=cls, + status=html.escape(res["status"]), + variant=html.escape(res["variant"]), + name=html.escape(res["dump_name"]), + mse=mse, + detail=detail, + ref_img=ref_img, + actual_img=actual_img, + diff_img=diff_img, + log=log, + )) + + counts = {} + for res in results: + counts[res["status"]] = counts.get(res["status"], 0) + 1 + summary = ", ".join("%s: %d" % (k, v) for k, v in sorted(counts.items())) + + variant_rows = "" + for variant in variants: + var_counts = {} + for res in results: + if res["variant"] == variant: + var_counts[res["status"]] = var_counts.get(res["status"], 0) + 1 + variant_rows += "%s%s" % ( + html.escape(variant), html.escape(", ".join("%s: %d" % (k, v) for k, v in sorted(var_counts.items())))) + + report = """ + + + +PPSSPP frametests report + + + +

PPSSPP frametests report

+

{summary}

+

Variants

+ + +{variant_rows} +
variantresults
+

Results

+{rows} + +""".format(summary=html.escape(summary), variant_rows=variant_rows, rows="\n".join(rows)) + + report_path.write_text(report, encoding="utf-8") + return report_path + + +def main(): + parser = argparse.ArgumentParser(description="Run PPSSPP framedump rendering tests.") + parser.add_argument("config", nargs="?", default="frametests/frametests.json", + help="Path to the JSON configuration file (default: frametests/frametests.json)") + parser.add_argument("--filter", default="", help="Only run dumps whose path contains this substring (case-insensitive)") + parser.add_argument("--strict", action="store_true", + help="Treat missing reference images as failures (configuration error)") + parser.add_argument("--out-mode", choices=["all", "failures"], default="all", + help="What to keep in the output directory: everything, or only failures (default: all)") + args = parser.parse_args() + + strict = args.strict or bool(os.environ.get("GITHUB_ACTIONS")) + + config_path = Path(args.config) + if not config_path.is_file(): + print("ERROR: config file not found: %s" % config_path, file=sys.stderr) + return 2 + config_dir = config_path.parent + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + test_root = (config_dir / config["testRoot"]).resolve() + ref_root = (config_dir / config["refRoot"]).resolve() + output_root = (config_dir / config.get("outputRoot", "frametest-out")).resolve() + timeout = float(config.get("timeout", 60)) + max_mse = float(config.get("maxMse", 0.0)) + variants = config.get("variants", {}) + if not variants: + print("ERROR: no variants defined in %s" % config_path, file=sys.stderr) + return 2 + + headless = find_headless(config_dir, config.get("headlessPath", "")) + if headless is None: + print("ERROR: PPSSPPHeadless binary not found. Set 'headlessPath' in %s or PPSSPP_HEADLESS, or run from the repo root." % config_path, file=sys.stderr) + return 2 + + if not test_root.is_dir(): + print("ERROR: test root not found: %s (check 'testRoot' in %s)" % (test_root, config_path), file=sys.stderr) + return 2 + + dumps = collect_dumps(test_root) + if not dumps: + print("ERROR: no frame dumps found under %s" % test_root, file=sys.stderr) + return 2 + + if args.filter: + filter_lower = args.filter.lower() + dumps = [d for d in dumps if filter_lower in str(d.relative_to(test_root)).lower()] + + log_dir = output_root / "logs" + diff_dir = output_root / "diffs" + actual_dir = output_root / "actuals" + generated_dir = output_root / "generated" + for d in (log_dir, diff_dir, actual_dir, generated_dir): + d.mkdir(parents=True, exist_ok=True) + + print("Headless: %s" % headless) + print("Test root: %s" % test_root) + print("Variants: %s" % ", ".join(variants.keys())) + print("Running %d dumps..." % len(dumps)) + + results = [] + failures = 0 + errors = 0 + new_refs = 0 + start_time = time.time() + + for dump in dumps: + rel = dump.relative_to(test_root) + base = strip_dump_extensions(dump.name) + rel_dir = rel.parent + for variant, variant_args_str in variants.items(): + variant_args = shlex.split(variant_args_str) + ref_path = ref_root / rel_dir / ("%s-%s.png" % (base, variant)) + actual_path = actual_dir / rel_dir / ("%s-%s.png" % (base, variant)) + diff_path = diff_dir / rel_dir / ("%s-%s.png" % (base, variant)) + log_path = log_dir / rel_dir / ("%s-%s.log" % (base, variant)) + for p in (ref_path, actual_path, diff_path, log_path): + p.parent.mkdir(parents=True, exist_ok=True) + + status, mse, output, timed_out = run_test( + headless, dump, variant_args, ref_path, actual_path, diff_path, + max_mse, timeout, output_root) + + detail = "" + if status == STATUS_FAIL: + if timed_out: + detail = "timed out" + elif mse is None: + if "Unable to read screenshot" in output: + status = STATUS_ERROR + detail = "reference image could not be loaded (corrupt or unreadable)" + else: + detail = "no screenshot MSE reported" + else: + detail = "MSE %.6f exceeds maximum %.6f" % (mse, max_mse) + if status == STATUS_ERROR and not detail: + detail = "headless failed or produced no reference image" + if status == STATUS_NEW: + detail = "reference image generated" + + # Track counters and emit GitHub Actions annotations. + if status == STATUS_FAIL: + failures += 1 + if os.environ.get("GITHUB_ACTIONS"): + print("::error file=%s::%s failed (%s)" % (rel, variant, detail)) + elif status == STATUS_ERROR: + errors += 1 + if os.environ.get("GITHUB_ACTIONS"): + print("::error file=%s::%s errored (%s)" % (rel, variant, detail)) + elif status == STATUS_NEW: + new_refs += 1 + gen_path = generated_dir / rel_dir / ("%s-%s.png" % (base, variant)) + gen_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ref_path, gen_path) + if strict: + failures += 1 + if os.environ.get("GITHUB_ACTIONS"): + print("::error file=%s::%s missing reference image (config error; generated one saved to %s)" % (rel, variant, gen_path)) + + # Keep artifacts (log, actual, diff) for everything in "all" mode, + # or only for failures/errors otherwise. + keep_artifacts = args.out_mode == "all" or status in (STATUS_FAIL, STATUS_ERROR) + if keep_artifacts: + log_path.write_text(output, encoding="utf-8", errors="replace") + else: + for p in (log_path, actual_path, diff_path): + try: + p.unlink() + except OSError: + pass + + if status != STATUS_PASS: + print("[%s] %s (%s)%s" % (status, rel, variant, " - " + detail if detail else "")) + + # In "failures" mode, only embed images for non-passing tests to + # keep the report (and thus the CI artifact) small. + ref_uri = None + if args.out_mode == "all" or status != STATUS_PASS: + if "could not be loaded" not in detail: + ref_uri = image_data_uri(ref_path) + results.append({ + "dump_name": str(rel), + "variant": variant, + "status": status, + "mse": mse, + "detail": detail, + "log": output if keep_artifacts and len(output) < LOG_EMBED_LIMIT else "", + "ref_uri": ref_uri, + "actual_uri": image_data_uri(actual_path) if actual_path.exists() else None, + "diff_uri": image_data_uri(diff_path) if diff_path.exists() else None, + }) + + elapsed = time.time() - start_time + print("Done in %.1fs: %d tests, %d failures, %d errors, %d new references." % ( + elapsed, len(results), failures, errors, new_refs)) + + report_path = output_root / "report.html" + write_report(report_path, results, list(variants.keys())) + print("Report written to: %s" % report_path) + + if failures or errors: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/headless/Compare.cpp b/headless/Compare.cpp index a91c5fbf8d..eee48b3051 100644 --- a/headless/Compare.cpp +++ b/headless/Compare.cpp @@ -305,19 +305,17 @@ std::vector TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 const u32 *pixels32 = (const u32 *)buffer->GetData(); const u16 *pixels16 = (const u16 *)buffer->GetData(); int outStride = buffer->GetStride(); - if (!buffer->GetFlipped()) { - // Bitmaps are flipped, so we have to compare backwards in this case. + if (buffer->GetFlipped()) { + // The buffer is stored bottom-up (e.g. to fit a bitmap), so read backwards + // here to get top-down output. int toLastRow = outStride * (h > buffer->GetHeight() ? buffer->GetHeight() - 1 : h - 1); pixels32 += toLastRow; pixels16 += toLastRow; outStride = -outStride; } - // Skip the bottom of the image if the buffer was smaller. Remember, we're flipped. + // The output is top-down; if the buffer was smaller, the image starts at the top. u32 *dst = &data[0]; - if (safeH < h) { - dst += (h - safeH) * stride; - } for (u32 y = 0; y < safeH; ++y) { switch (buffer->GetFormat()) { @@ -414,17 +412,19 @@ double ScreenshotComparer::Compare(const Path &screenshotFilename) { double errors = 0; if (asBitmap_) { - // The reference is flipped and BGRA by default for the common BMP compare case. + // The reference is a BMP, stored bottom-up and BGRA by default for the + // common BMP compare case, so read it backwards (pixels_ is top-down). for (u32 y = 0; y < h_; ++y) { - u32 yoff = y * referenceStride_; + u32 yoff = (h_ - y - 1) * referenceStride_; for (u32 x = 0; x < w_; ++x) errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]); } } else { + // The reference is a PNG, stored top-down, so no flip needed. // Just convert to BGRA for simplicity. ConvertRGBA8888ToBGRA8888(reference_, reference_, h_ * referenceStride_); for (u32 y = 0; y < h_; ++y) { - u32 yoff = (h_ - y - 1) * referenceStride_; + u32 yoff = y * referenceStride_; for (u32 x = 0; x < w_; ++x) errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]); } @@ -448,7 +448,9 @@ bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) { FILE *saved = File::OpenCFile(resultFilename, "wb"); if (saved) { fwrite(&header, sizeof(header), 1, saved); - fwrite(pixels_.data(), sizeof(u32), stride_ * h_, saved); + // Bitmaps are stored bottom-up, so write the rows backwards (pixels_ is top-down). + for (u32 y = 0; y < h_; ++y) + fwrite(pixels_.data() + (h_ - y - 1) * stride_, sizeof(u32), stride_, saved); fclose(saved); return true; @@ -457,23 +459,36 @@ bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) { return false; } +bool ScreenshotComparer::SaveActualPNG(const Path &resultFilename) { + std::vector rgba((size_t)stride_ * h_ * 4); + const u32 *pixels = pixels_.data(); + for (size_t i = 0; i < (size_t)stride_ * h_; ++i) { + u32 p = pixels[i]; + rgba[i * 4 + 0] = (p >> 16) & 0xFF; + rgba[i * 4 + 1] = (p >> 8) & 0xFF; + rgba[i * 4 + 2] = p & 0xFF; + rgba[i * 4 + 3] = (p >> 24) & 0xFF; + } + return pngSave(resultFilename, rgba.data(), stride_, h_, 4); +} + bool ScreenshotComparer::SaveVisualComparisonPNG(const Path &resultFilename) { std::unique_ptr comparison(new u32[w_ * 2 * h_ * 2]); if (asBitmap_) { - // The reference is flipped and BGRA by default for the common BMP compare case. + // The reference is a BMP, stored bottom-up, so read it backwards. for (u32 y = 0; y < h_; ++y) { - u32 yoff = y * referenceStride_; - u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2; + u32 yoff = (h_ - y - 1) * referenceStride_; + u32 comparisonRow = y * 2 * w_ * 2; for (u32 x = 0; x < w_; ++x) { PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]); } } } else { - // Reference is already in BGRA either way. + // Reference is a PNG, stored top-down, so no flip needed. for (u32 y = 0; y < h_; ++y) { - u32 yoff = (h_ - y - 1) * referenceStride_; - u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2; + u32 yoff = y * referenceStride_; + u32 comparisonRow = y * 2 * w_ * 2; for (u32 x = 0; x < w_; ++x) { PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]); } diff --git a/headless/Compare.h b/headless/Compare.h index 04ab899568..a9240440a9 100644 --- a/headless/Compare.h +++ b/headless/Compare.h @@ -47,6 +47,7 @@ public: } bool SaveActualBitmap(const Path &filename); + bool SaveActualPNG(const Path &filename); bool SaveVisualComparisonPNG(const Path &filename); protected: diff --git a/headless/Headless.cpp b/headless/Headless.cpp index af52010547..c2bd6c09da 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -70,7 +70,9 @@ static Path g_comparisonScreenshot; static Path g_screenshotSavePath; +static Path g_screenshotDiffPath; static double g_maxScreenshotError = 0.0; +static bool g_screenshotFailed = false; static std::string g_debugOutputBuffer; static bool g_writeFailureScreenshot = true; static bool g_writeDebugOutput = true; @@ -200,7 +202,8 @@ void System_SendDebugScreenshot(const uint8_t *data, int width, int height) { // If a screenshot save path is set, save unconditionally. if (!g_screenshotSavePath.empty()) { ScreenshotComparer saver(pixels, FRAME_STRIDE, FRAME_WIDTH, FRAME_HEIGHT); - if (saver.SaveActualBitmap(g_screenshotSavePath)) + bool saved = g_screenshotSavePath.GetFileExtension() == ".png" ? saver.SaveActualPNG(g_screenshotSavePath) : saver.SaveActualBitmap(g_screenshotSavePath); + if (saved) SendAndCollectOutput("Screenshot saved to: " + g_screenshotSavePath.ToVisualString() + "\n"); } @@ -213,10 +216,12 @@ void System_SendDebugScreenshot(const uint8_t *data, int width, int height) { double errors = comparer.Compare(g_comparisonScreenshot); if (errors < 0) { SendAndCollectOutput(comparer.GetError() + "\n"); + g_screenshotFailed = true; } if (errors > g_maxScreenshotError) { SendAndCollectOutput(StringFromFormat("Screenshot MSE: %f\n", errors)); + g_screenshotFailed = true; } if (errors > g_maxScreenshotError && g_writeFailureScreenshot) { @@ -224,6 +229,12 @@ void System_SendDebugScreenshot(const uint8_t *data, int width, int height) { SendAndCollectOutput("Actual output written to: __testfailure.bmp\n"); comparer.SaveVisualComparisonPNG(Path("__testcompare.png")); } + + // If a diff path is set, always save the visual comparison (regardless of pass/fail). + if (!g_screenshotDiffPath.empty() && errors >= 0) { + if (comparer.SaveVisualComparisonPNG(g_screenshotDiffPath)) + SendAndCollectOutput("Screenshot comparison saved to: " + g_screenshotDiffPath.ToVisualString() + "\n"); + } } static GraphicsContext *CreateGraphicsContext(GPUCore gpuCore, std::string **deviceSetting) { @@ -286,6 +297,7 @@ static bool RunAutoTest(GraphicsContext *graphicsContext, CoreParameter &corePar // Kinda ugly, trying to guesstimate the test name from filename... currentTestName = GetTestName(coreParameter.fileToStart); + g_screenshotFailed = false; std::string output; if (opt.compare || opt.bench) @@ -384,6 +396,11 @@ static bool RunAutoTest(GraphicsContext *graphicsContext, CoreParameter &corePar passed = CompareOutput(coreParameter.fileToStart, output, opt.verbose, opt.printEqualLines); } + // Screenshot comparison failures are recorded in System_SendDebugScreenshot. + if (!g_comparisonScreenshot.empty() && g_screenshotFailed) { + passed = false; + } + return passed; } @@ -472,7 +489,7 @@ int RunTests(GraphicsContext *graphicsContext, CoreParameter &coreParameter, con std::string testName = GetTestName(coreParameter.fileToStart); printf(" %s - %f seconds average\n", testName.c_str(), (et - st) / runs); } - if (testOptions.compare) { + if (testOptions.compare || !g_comparisonScreenshot.empty()) { std::string testName = GetTestName(coreParameter.fileToStart); if (passed) { passedTests.push_back(testName); @@ -483,7 +500,7 @@ int RunTests(GraphicsContext *graphicsContext, CoreParameter &coreParameter, con } } - if (testOptions.compare) { + if (testOptions.compare || !g_comparisonScreenshot.empty()) { printf("%d tests passed, %d tests failed, %d tests missing.\n", (int)passedTests.size(), (int)failedTests.size(), (int)missingTests.size()); if (!failedTests.empty()) { printf("Failed tests:\n"); @@ -762,6 +779,9 @@ int main(int argc, const char* argv[]) { if (cmdLineOptions.screenshotFilenameSave.has_value()) { SetScreenshotSavePath(Path(std::string(cmdLineOptions.screenshotFilenameSave.value()))); } + if (cmdLineOptions.screenshotFilenameDiff.has_value()) { + g_screenshotDiffPath = Path(std::string(cmdLineOptions.screenshotFilenameDiff.value())); + } SetWriteFailureScreenshot(!getenv("GITHUB_ACTIONS") && !testOptions.bench); SetWriteDebugOutput(!testOptions.compare && !testOptions.bench); diff --git a/headless/README.md b/headless/README.md index 4ce27eaffc..8e13caed3b 100644 --- a/headless/README.md +++ b/headless/README.md @@ -43,7 +43,8 @@ PPSSPPHeadless file.elf|file.prx|file.ppdmp [...] [options] | `-o`, `--odslog` | Write log to `OutputDebugString` (Windows only). | | `--graphics=` | GPU backend: `software`, `gles`, `directx11`, `vulkan`. | | `--screenshot=` | Compare the rendered output against a reference screenshot. | -| `--screenshot-save=` | Save the rendered output to a BMP file (no comparison). | +| `--screenshot-save=` | Save the rendered output to a file (PNG if the path ends in `.png`, BMP otherwise). | +| `--screenshot-diff=` | When comparing screenshots, save a visual comparison image to this file (always, regardless of pass/fail). | | `--max-mse=` | Maximum allowed Mean Squared Error for screenshot comparison (default: 0 = exact). | | `--compare` / `-c` | Compare test output with `.expected` text file and/or screenshot (see below). | | `--timeout=` | Abort test if it takes longer than this. | @@ -80,13 +81,18 @@ GE frame dumps are recordings of a single frame's GE graphics commands. When a ` 3. The framebuffer is captured automatically (512×272 stride, 480×272 visible). 4. A screenshot is sent for comparison/saving via `--compare`, `--screenshot`, or `--screenshot-save`. +For batch rendering tests over sets of frame dumps, see `docs/frametest.md` (the `frametests.py` runner). + ### Example: Generate a reference screenshot ```bash +# BMP: PPSSPPHeadless.exe --graphics=software --screenshot-save=reference.bmp frame.ppdmp +# PNG (lossless, recommended for storing references): +PPSSPPHeadless.exe --graphics=software --screenshot-save=reference.png frame.ppdmp ``` -This outputs a 512×272 BMP file with 32-bit BGRA pixel data. The file size is always 557,110 bytes (54-byte header + 512 × 272 × 4 bytes). +The BMP output is 512×272 with 32-bit BGRA pixel data. The file size is always 557,110 bytes (54-byte header + 512 × 272 × 4 bytes). PNG output is 512×272 RGBA. ### Example: Compare against a reference @@ -95,8 +101,8 @@ This outputs a 512×272 BMP file with 32-bit BGRA pixel data. The file size is a # frame.ppdmp → looks for frame.png (next to the .ppdmp) PPSSPPHeadless.exe --graphics=software --compare frame.ppdmp -# Explicit reference file: -PPSSPPHeadless.exe --graphics=software --screenshot=reference.bmp --max-mse=0.5 frame.ppdmp +# Explicit reference file (saves a visual comparison to diff.png): +PPSSPPHeadless.exe --graphics=software --screenshot=reference.bmp --screenshot-diff=diff.png --max-mse=0.5 frame.ppdmp ``` When the MSE exceeds `--max-mse`, the following files are saved in the working directory: diff --git a/libretro/LibretroGraphicsContext.cpp b/libretro/LibretroGraphicsContext.cpp index 310bb3ba39..26592b94d6 100644 --- a/libretro/LibretroGraphicsContext.cpp +++ b/libretro/LibretroGraphicsContext.cpp @@ -157,7 +157,7 @@ LibretroGraphicsContext *LibretroGraphicsContext::CreateGraphicsContext() { return ctx; } -std::vector TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 stride, u32 h) { +std::vector ConvertFramebufferForLibretro(const GPUDebugBuffer *buffer, u32 stride, u32 h) { // If the output was small, act like everything outside was 0. // This can happen depending on viewport parameters. u32 safeW = std::min(stride, buffer->GetStride()); diff --git a/libretro/LibretroGraphicsContext.h b/libretro/LibretroGraphicsContext.h index 5e0bc2ac4c..4629ee9717 100644 --- a/libretro/LibretroGraphicsContext.h +++ b/libretro/LibretroGraphicsContext.h @@ -5,6 +5,7 @@ #include "Common/GPU/GraphicsContext.h" #include "Common/GPU/thin3d_create.h" +#include "Common/CommonTypes.h" #include "Core/Config.h" #include "Core/System.h" #include "GPU/GPUState.h" @@ -67,6 +68,8 @@ protected: retro_hw_render_callback hw_render_ = {}; }; +std::vector ConvertFramebufferForLibretro(const GPUDebugBuffer *buffer, u32 stride, u32 h); + class LibretroSoftwareContext : public LibretroGraphicsContext { public: LibretroSoftwareContext() {} @@ -76,7 +79,7 @@ public: u16 h = NATIVEHEIGHT; if (gpu) { gpu->GetOutputFramebuffer(buf); - const std::vector pixels = TranslateDebugBufferToCompare(&buf, w, h); + const std::vector pixels = ConvertFramebufferForLibretro(&buf, w, h); memcpy(soft_bmp, pixels.data(), SOFT_BMP_SIZE); } u32 offset = g_Config.bDisplayCropTo16x9 ? w << 1 : 0;