From 893ac3e3db4f6c76abfc04e765b005b3bb67525c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 11:04:41 +0200 Subject: [PATCH 01/14] Frametests: add framedump rendering test runner and PNG screenshot support - Add frametests.py: walks a dump tree, renders each dump per config variant through PPSSPPHeadless, generates reference images when missing and compares MSE when present, and writes a self-contained HTML report. The JSON config (which lives with the test set, not in the repo) points at the data tree and defines variants as suffix -> CLI args, e.g. 'soft': '--graphics=software'. - Headless: --screenshot-save saves PNG when the path ends in .png; new --screenshot-diff always writes a visual comparison when comparing; screenshot comparison failures (mismatch or unloadable reference) now fail the test instead of passing silently. - Read back framebuffers top-down, flipping only for BMP output/input (fixes upside-down PNG references). Sync libretro copy accordingly. - Document the system in docs/frametest.md; add AGENTS.md reference. --- .gitignore | 7 +- AGENTS.md | 10 + Core/CmdLine.cpp | 3 +- Core/CmdLine.h | 1 + docs/frametest.md | 191 ++++++++++++ frametests.py | 415 +++++++++++++++++++++++++++ headless/Compare.cpp | 47 +-- headless/Compare.h | 1 + headless/Headless.cpp | 26 +- headless/README.md | 14 +- libretro/LibretroGraphicsContext.cpp | 2 +- libretro/LibretroGraphicsContext.h | 5 +- 12 files changed, 694 insertions(+), 28 deletions(-) create mode 100644 docs/frametest.md create mode 100644 frametests.py diff --git a/.gitignore b/.gitignore index 89b166c2e2..d58ac5312a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,9 @@ ppsspp.ini imgui.ini PPSSPPControls.dat +# Frametest output (see frametests.py) +frametest-out/ + # Gradle/Android Studio .gradle .idea @@ -147,5 +150,5 @@ RAPrefs_PPSSPP.cfg # For CLion cmake-build-*/ - -/.vscode/ + +/.vscode/ diff --git a/AGENTS.md b/AGENTS.md index 510478deaf..0675509672 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -152,6 +152,16 @@ pspautotests are a large set of tests of the PSP OS's API surface, and thus test See docs/pspautotests.md for a workflow for running pspautotests and improving PPSSPP with the results. +## Framedump rendering tests (frametests) + +There is a rendering test system that replays GE frame dumps (`.ppdmp`) through PPSSPPHeadless and compares +the output against reference images, driven by the `frametests.py` script and a JSON config per test set. +When changing rendering code, consider running these tests. See docs/frametest.md for full documentation. + +Note: `headless/Compare.cpp` reads back framebuffers top-down; the flip to bottom-up is only applied when +writing BMPs (and when reading BMP references). `TranslateDebugBufferToCompare` also exists as a copy in +`libretro/LibretroGraphicsContext.cpp` - keep the two in sync. + ## Adding HLE modules HLE module implementations live in `Core/HLE/sce.cpp` / `.h` (e.g. `sceOpenPSID.cpp`, `scePauth.cpp` are good diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index bbbb672cd8..86ebb822f2 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -181,7 +181,8 @@ static const CommandLineParam g_autoParams[] = { {POFF(log), CmdParamType::String, "log", '\0', "Output log to FILE", CmdLineMode::Application}, {POFF(enableLogging), CmdParamType::Bool, "log", '\0', "Full log output, not just emulated printfs", CmdLineMode::Headless}, {POFF(screenshotFilename), CmdParamType::String, "screenshot", '\0', "Compare rendered output against a reference screenshot FILE", CmdLineMode::Headless}, - {POFF(screenshotFilenameSave), CmdParamType::String, "screenshot-save", '\0', "Save rendered screenshot to specified path", CmdLineMode::Headless}, + {POFF(screenshotFilenameSave), CmdParamType::String, "screenshot-save", '\0', "Save rendered screenshot to specified path (PNG if the path ends in .png, BMP otherwise)", CmdLineMode::Headless}, + {POFF(screenshotFilenameDiff), CmdParamType::String, "screenshot-diff", '\0', "Save a visual comparison image to FILE when comparing screenshots", CmdLineMode::Headless}, {POFF(timeout), CmdParamType::Double, "timeout", '\0', "Set the timeout value", CmdLineMode::Headless}, {POFF(maxScreenshotError), CmdParamType::Double, "max-mse", '\0', "Maximum allowed MSE error for screenshot comparison", CmdLineMode::Headless}, {POFF(mountIso), CmdParamType::String, "mount", 'm', "Mount ISO/CSO on umd1:", CmdLineMode::Headless}, diff --git a/Core/CmdLine.h b/Core/CmdLine.h index 4d94bbc5cb..211b5853f4 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -95,6 +95,7 @@ struct CommandLineOptions { std::optional screenshotFilename; std::optional screenshotFilenameSave; + std::optional screenshotFilenameDiff; // Headless: mount an ISO/CSO on umd1:. std::optional mountIso; diff --git a/docs/frametest.md b/docs/frametest.md new file mode 100644 index 0000000000..d481ef4177 --- /dev/null +++ b/docs/frametest.md @@ -0,0 +1,191 @@ +# Framedump rendering tests (frametests) + +PPSSPP has a test system that replays GE frame dumps (`.ppdmp`) through +`PPSSPPHeadless` with various rendering settings, comparing the rendered +output against stored reference images. It is designed to catch rendering +regressions across GPU backends and configurations. + +The system has three parts: + +- **The test runner**: `frametests.py` (repo root). Test-set agnostic - it + works against any test data tree pointed to by its configuration. +- **The configuration**: a JSON file describing where the test data lives and + which rendering variants to run (e.g. `frametests/frametests.json`). +- **The test set**: the frame dumps and reference images (e.g. + `frametests/dumps/` and `frametests/ref/`). Different environments can use + different test sets - GitHub CI uses the one in the repo, custom machines + can use their own (possibly much larger) ones. + +## Quick start + +```bash +# Build PPSSPPHeadless (see headless/README.md), then: +python3 frametests.py frametests/frametests.json +``` + +The first run generates reference images for any dumps that don't have them +yet (status `NEW`) and compares the rest (status `PASS`/`FAIL`). A summary is +printed and a self-contained HTML report is written to +`frametests/frametest-out/report.html` (default output dir; gitignored). + +Exit code is `0` if all tests passed, `1` if anything failed, `2` on +configuration/usage errors. + +## Configuration + +The configuration is a single JSON file. All relative paths are resolved +against the config file's directory, so the same script works with any test +set by just pointing `--config` at a different file. + +```json +{ + "testRoot": "dumps", + "refRoot": "ref", + "outputRoot": "frametest-out", + "headlessPath": "", + "timeout": 60, + "maxMse": 0.0, + "variants": { + "soft": "--graphics=software" + } +} +``` + +| Key | Description | +|-----------------|-----------------------------------------------------------------------------| +| `testRoot` | Directory tree containing frame dumps (`.ppdmp` files, possibly wrapped in `.zip`). | +| `refRoot` | Where reference images live; mirrors the `testRoot` tree. | +| `outputRoot` | Where logs, diff images, the report, and generated references go. | +| `headlessPath` | Path to the `PPSSPPHeadless` binary. Empty = auto-detect (see below). | +| `timeout` | Per-test timeout in seconds. | +| `maxMse` | Maximum allowed MSE for screenshot comparison (0 = exact match). | +| `variants` | Map of variant name to command line arguments for the headless binary. | + +### Variants + +Each test runs once per variant. The variant name is appended to the +reference image filename, so each variant gets its own reference set: + +``` +dumps/Depth/11578 Virtua Tennis pause menu ULES00126_0002.zip + → ref/Depth/11578 Virtua Tennis pause menu ULES00126_0002-soft.png +``` + +The variant value is an arbitrary command line argument string passed to +`PPSSPPHeadless`, so new rendering configurations are just new entries (or a +different config file on a machine with the right GPU): + +```json +"variants": { + "soft": "--graphics=software", + "gl": "--graphics=opengl", + "gl-4x": "--graphics=opengl --resolution-scale=4" +} +``` + +## How a test runs + +For each dump (recursively under `testRoot`) and each variant: + +- If the reference image `-.png` is **missing**: the dump is + rendered and the output saved as the new reference. Status: `NEW`. This is + how references are created - run locally, then commit the generated images + (also copied to `/generated/` for convenience). +- If the reference **exists**: the dump is rendered, the output saved to + `/actuals/`, and compared against the reference using MSE + (mean squared error over R, G, B per pixel, alpha ignored). A visual + comparison image (actual / reference+diff) is saved to + `/diffs/` whenever a comparison runs. Status: `PASS` or `FAIL` + (mismatch, crash, or timeout). + +A `FAIL` with no MSE reported usually means the headless binary crashed +before producing a screenshot, and a reference image that can't be loaded +(corrupt) is reported as `ERROR`. The full emulator log of every non-passing +test is kept in `/logs/`. + +### Reference images + +- PNG, 512×272 (480×272 display in a 512-wide framebuffer), stored top-down + (row 0 = top of screen). The BMP output format is bottom-up per the BMP + spec; the flip is applied only when writing BMPs. +- Generated with `--graphics=software` they are fully deterministic: a + subsequent run produces byte-identical output, so `maxMse` can be 0. +- If rendering code changes the output, existing references may need + regenerating: delete the affected reference images and re-run to regenerate + them. + +### Headless flags used + +The runner uses `--screenshot=` (compare), `--screenshot-save=` +(save output; PNG if the path ends in `.png`, else BMP) and +`--screenshot-diff=` (always write a visual comparison when comparing). +See `headless/README.md` for details. + +## Command line options + +``` +usage: frametests.py [OPTIONS] [CONFIG.json] +``` + +| Option | Description | +|--------------------|-----------------------------------------------------------------------| +| `CONFIG.json` | Path to the configuration file (default: `frametests/frametests.json`). | +| `--filter=SUBSTR` | Only run dumps whose relative path contains SUBSTR (case-insensitive). | +| `--strict` | Treat missing reference images as failures (a configuration error). | +| `--out-mode=all\|failures` | `all` keeps everything in the output dir; `failures` keeps only artifacts of non-passing tests (smaller CI artifacts). Default: `all`. | + +The headless binary is located, in order of preference: + +1. `headlessPath` from the config file. +2. The `PPSSPP_HEADLESS` environment variable. +3. Well-known paths relative to the current directory (e.g. + `Windows/x64/Debug/PPSSPPHeadless.exe`, `build/PPSSPPHeadless`), newest + by modification time. + +## CI integration + +In CI (`GITHUB_ACTIONS` is set) the runner behaves as if `--strict` was +passed and prints `::error` annotations for each failing test, so failures +show up inline in the GitHub Actions log. A typical CI job: + +```yaml +- name: Build headless + run: ./b.sh --headless +- name: Run frametests + run: python3 frametests.py frametests/frametests.json --out-mode=failures +- name: Upload report + uses: actions/upload-artifact@v4 + with: + name: frametest-report + path: frametests/frametest-out/ +``` + +A missing reference image in CI means the test set is incomplete - the run +fails and the generated reference is available in +`frametests/frametest-out/generated/` (part of the uploaded artifact) so it +can be committed. + +## Custom machines + +The script is designed to run on machines with different GPUs and bigger test +sets than CI. Copy the repo, point at a custom config: + +```bash +python3 frametests.py /path/to/my-config.json +``` + +The config can live anywhere and point at any test data tree; only the +script itself is shared. Add hardware-specific variants (`--graphics=vulkan`, +`--msaa=...`, etc.) to the machine's own config - no code changes needed for +new flag combinations, as long as the headless binary supports them. + +## Adding a new framedump + +1. Drop the dump into `frametests/dumps/` (either as a `.ppdmp` or zipped). +2. Run `python3 frametests.py frametests/frametests.json --filter=` - + the reference image(s) are generated and copied to `/generated/`. +3. Commit the dump and the reference images under `frametests/ref/`. + +The CI test set (dumps and references) is maintained separately from the +runner; the `frametests/` directory is just one of several possible test sets +and may become a git submodule at some point. diff --git a/frametests.py b/frametests.py new file mode 100644 index 0000000000..d075bd39e7 --- /dev/null +++ b/frametests.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Framedump rendering test runner for PPSSPPHeadless. + +Runs GE frame dumps (".ppdmp", possibly zipped) through PPSSPPHeadless with a +set of rendering variants (command line option sets), generating reference +images when missing and comparing against existing ones when present. Produces +a self-contained HTML report and returns a nonzero exit code on failure. + +This script is test-set agnostic: it reads its configuration from a JSON file +(similar to frametests/frametests.json) that points at the test data tree, +so it can be used against different test sets (the CI one, or bigger private +ones on custom machines). + +Usage: + frametests.py [OPTIONS] [CONFIG.json] + +Example: + python3 frametests.py frametests/frametests.json +""" + +import argparse +import base64 +import glob +import html +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import time +from pathlib import Path + +# test.py-style candidate paths for the headless binary, relative to the +# current working directory, in preference order. +HEADLESS_CANDIDATES = [ + "Windows/x64/Debug/PPSSPPHeadless.exe", + "Windows/Debug/PPSSPPHeadless.exe", + "Windows/x64/Release/PPSSPPHeadless.exe", + "Windows/Release/PPSSPPHeadless.exe", + "build/PPSSPPHeadless", + "build-headless/PPSSPPHeadless", + "build*/PPSSPPHeadless", + "PPSSPPHeadless", + "ppsspp/PPSSPPHeadless", +] + +MSE_RE = re.compile(r"Screenshot MSE: ([0-9.eE+-]+)") +LOG_EMBED_LIMIT = 64 * 1024 + +STATUS_PASS = "PASS" +STATUS_FAIL = "FAIL" +STATUS_NEW = "NEW" +STATUS_ERROR = "ERROR" + + +def find_headless(config_dir, config_path): + """Locate the PPSSPPHeadless binary. The config headlessPath wins, then + the PPSSPP_HEADLESS env var, then candidate paths (newest by mtime).""" + candidates = [] + if config_path: + p = Path(config_path) + candidates.append(p if p.is_absolute() else config_dir / p) + env_path = os.environ.get("PPSSPP_HEADLESS") + if env_path: + candidates.append(Path(env_path)) + for pattern in HEADLESS_CANDIDATES: + candidates.extend(Path(m) for m in glob.glob(pattern)) + found = [c for c in candidates if c.is_file()] + if not found: + return None + return max(found, key=lambda p: p.stat().st_mtime) + + +def strip_dump_extensions(name): + """Derive the base name for a dump, stripping .zip and .ppdmp extensions.""" + base = name + lower = base.lower() + if lower.endswith(".zip"): + base = base[:-4] + lower = base.lower() + if lower.endswith(".ppdmp"): + base = base[:-6] + return base + + +def collect_dumps(test_root): + """Recursively collect frame dumps (.ppdmp files and .zip wrappers).""" + dumps = [] + for dirpath, dirnames, filenames in os.walk(test_root): + dirnames[:] = [d for d in sorted(dirnames) if not d.startswith(".")] + for filename in sorted(filenames): + lower = filename.lower() + if lower.endswith(".ppdmp") or lower.endswith(".zip"): + dumps.append(Path(dirpath) / filename) + return sorted(dumps) + + +def run_test(headless, dump_path, variant_args, ref_path, actual_path, diff_path, max_mse, timeout, output_dir): + """Run one dump with one variant's args. Returns (status, mse, output, timed_out).""" + generate = not ref_path.exists() + args = [str(headless)] + list(variant_args) + if generate: + args.append("--screenshot-save=" + str(ref_path)) + else: + args.extend([ + "--screenshot-save=" + str(actual_path), + "--screenshot=" + str(ref_path), + "--max-mse=" + str(max_mse), + "--screenshot-diff=" + str(diff_path), + ]) + args.append(str(dump_path)) + + timed_out = False + try: + proc = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd=str(output_dir), text=True, encoding="utf-8", errors="replace") + output, _ = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + output, _ = proc.communicate() + timed_out = True + returncode = proc.returncode + + # Headless may drop these in the cwd on failure (outside GITHUB_ACTIONS). + for stray in ("__testfailure.bmp", "__testcompare.png"): + try: + (output_dir / stray).unlink() + except OSError: + pass + + match = MSE_RE.search(output) + mse = float(match.group(1)) if match else None + + if timed_out: + return STATUS_FAIL, mse, output, True + if generate: + if returncode != 0 or not ref_path.exists(): + return STATUS_ERROR, mse, output, False + return STATUS_NEW, mse, output, False + if returncode != 0: + return STATUS_FAIL, mse, output, False + return STATUS_PASS, mse, output, False + + +def image_data_uri(path): + """Read an image file as a PNG data URI, or None if missing.""" + try: + data = path.read_bytes() + except OSError: + return None + return "data:image/png;base64," + base64.b64encode(data).decode("ascii") + + +def write_report(report_path, results, variants): + rows = [] + for res in results: + cls = "failed" if res["status"] in (STATUS_FAIL, STATUS_ERROR) else ("new" if res["status"] == STATUS_NEW else "") + mse = ('MSE: %.6f' % res["mse"]) if res["mse"] is not None else "" + detail = '%s' % html.escape(res["detail"]) if res["detail"] else "" + ref_img = 'reference' % res["ref_uri"] if res["ref_uri"] else "" + actual_img = 'actual' % res["actual_uri"] if res["actual_uri"] else "" + diff_img = 'diff' % res["diff_uri"] if res["diff_uri"] else "" + log = ('
log
%s
' % html.escape(res["log"])) if res["log"] else "" + rows.append("""
+
+ {status} + {variant} + {name} + {mse}{detail} +
+
+ {ref_img}{actual_img}{diff_img} +
+ {log} +
""".format( + cls=cls, + status=html.escape(res["status"]), + variant=html.escape(res["variant"]), + name=html.escape(res["dump_name"]), + mse=mse, + detail=detail, + ref_img=ref_img, + actual_img=actual_img, + diff_img=diff_img, + log=log, + )) + + counts = {} + for res in results: + counts[res["status"]] = counts.get(res["status"], 0) + 1 + summary = ", ".join("%s: %d" % (k, v) for k, v in sorted(counts.items())) + + variant_rows = "" + for variant in variants: + var_counts = {} + for res in results: + if res["variant"] == variant: + var_counts[res["status"]] = var_counts.get(res["status"], 0) + 1 + variant_rows += "%s%s" % ( + html.escape(variant), html.escape(", ".join("%s: %d" % (k, v) for k, v in sorted(var_counts.items())))) + + report = """ + + + +PPSSPP frametests report + + + +

PPSSPP frametests report

+

{summary}

+

Variants

+ + +{variant_rows} +
variantresults
+

Results

+{rows} + +""".format(summary=html.escape(summary), variant_rows=variant_rows, rows="\n".join(rows)) + + report_path.write_text(report, encoding="utf-8") + return report_path + + +def main(): + parser = argparse.ArgumentParser(description="Run PPSSPP framedump rendering tests.") + parser.add_argument("config", nargs="?", default="frametests/frametests.json", + help="Path to the JSON configuration file (default: frametests/frametests.json)") + parser.add_argument("--filter", default="", help="Only run dumps whose path contains this substring (case-insensitive)") + parser.add_argument("--strict", action="store_true", + help="Treat missing reference images as failures (configuration error)") + parser.add_argument("--out-mode", choices=["all", "failures"], default="all", + help="What to keep in the output directory: everything, or only failures (default: all)") + args = parser.parse_args() + + strict = args.strict or bool(os.environ.get("GITHUB_ACTIONS")) + + config_path = Path(args.config) + if not config_path.is_file(): + print("ERROR: config file not found: %s" % config_path, file=sys.stderr) + return 2 + config_dir = config_path.parent + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + test_root = (config_dir / config["testRoot"]).resolve() + ref_root = (config_dir / config["refRoot"]).resolve() + output_root = (config_dir / config.get("outputRoot", "frametest-out")).resolve() + timeout = float(config.get("timeout", 60)) + max_mse = float(config.get("maxMse", 0.0)) + variants = config.get("variants", {}) + if not variants: + print("ERROR: no variants defined in %s" % config_path, file=sys.stderr) + return 2 + + headless = find_headless(config_dir, config.get("headlessPath", "")) + if headless is None: + print("ERROR: PPSSPPHeadless binary not found. Set 'headlessPath' in %s or PPSSPP_HEADLESS, or run from the repo root." % config_path, file=sys.stderr) + return 2 + + if not test_root.is_dir(): + print("ERROR: test root not found: %s (check 'testRoot' in %s)" % (test_root, config_path), file=sys.stderr) + return 2 + + dumps = collect_dumps(test_root) + if not dumps: + print("ERROR: no frame dumps found under %s" % test_root, file=sys.stderr) + return 2 + + if args.filter: + filter_lower = args.filter.lower() + dumps = [d for d in dumps if filter_lower in str(d.relative_to(test_root)).lower()] + + log_dir = output_root / "logs" + diff_dir = output_root / "diffs" + actual_dir = output_root / "actuals" + generated_dir = output_root / "generated" + for d in (log_dir, diff_dir, actual_dir, generated_dir): + d.mkdir(parents=True, exist_ok=True) + + print("Headless: %s" % headless) + print("Test root: %s" % test_root) + print("Variants: %s" % ", ".join(variants.keys())) + print("Running %d dumps..." % len(dumps)) + + results = [] + failures = 0 + errors = 0 + new_refs = 0 + start_time = time.time() + + for dump in dumps: + rel = dump.relative_to(test_root) + base = strip_dump_extensions(dump.name) + rel_dir = rel.parent + for variant, variant_args_str in variants.items(): + variant_args = shlex.split(variant_args_str) + ref_path = ref_root / rel_dir / ("%s-%s.png" % (base, variant)) + actual_path = actual_dir / rel_dir / ("%s-%s.png" % (base, variant)) + diff_path = diff_dir / rel_dir / ("%s-%s.png" % (base, variant)) + log_path = log_dir / rel_dir / ("%s-%s.log" % (base, variant)) + for p in (ref_path, actual_path, diff_path, log_path): + p.parent.mkdir(parents=True, exist_ok=True) + + status, mse, output, timed_out = run_test( + headless, dump, variant_args, ref_path, actual_path, diff_path, + max_mse, timeout, output_root) + + detail = "" + if status == STATUS_FAIL: + if timed_out: + detail = "timed out" + elif mse is None: + if "Unable to read screenshot" in output: + status = STATUS_ERROR + detail = "reference image could not be loaded (corrupt or unreadable)" + else: + detail = "no screenshot MSE reported" + else: + detail = "MSE %.6f exceeds maximum %.6f" % (mse, max_mse) + if status == STATUS_ERROR and not detail: + detail = "headless failed or produced no reference image" + if status == STATUS_NEW: + detail = "reference image generated" + + # Track counters and emit GitHub Actions annotations. + if status == STATUS_FAIL: + failures += 1 + if os.environ.get("GITHUB_ACTIONS"): + print("::error file=%s::%s failed (%s)" % (rel, variant, detail)) + elif status == STATUS_ERROR: + errors += 1 + if os.environ.get("GITHUB_ACTIONS"): + print("::error file=%s::%s errored (%s)" % (rel, variant, detail)) + elif status == STATUS_NEW: + new_refs += 1 + gen_path = generated_dir / rel_dir / ("%s-%s.png" % (base, variant)) + gen_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ref_path, gen_path) + if strict: + failures += 1 + if os.environ.get("GITHUB_ACTIONS"): + print("::error file=%s::%s missing reference image (config error; generated one saved to %s)" % (rel, variant, gen_path)) + + # Keep artifacts (log, actual, diff) for everything in "all" mode, + # or only for failures/errors otherwise. + keep_artifacts = args.out_mode == "all" or status in (STATUS_FAIL, STATUS_ERROR) + if keep_artifacts: + log_path.write_text(output, encoding="utf-8", errors="replace") + else: + for p in (log_path, actual_path, diff_path): + try: + p.unlink() + except OSError: + pass + + if status != STATUS_PASS: + print("[%s] %s (%s)%s" % (status, rel, variant, " - " + detail if detail else "")) + + # In "failures" mode, only embed images for non-passing tests to + # keep the report (and thus the CI artifact) small. + ref_uri = None + if args.out_mode == "all" or status != STATUS_PASS: + if "could not be loaded" not in detail: + ref_uri = image_data_uri(ref_path) + results.append({ + "dump_name": str(rel), + "variant": variant, + "status": status, + "mse": mse, + "detail": detail, + "log": output if keep_artifacts and len(output) < LOG_EMBED_LIMIT else "", + "ref_uri": ref_uri, + "actual_uri": image_data_uri(actual_path) if actual_path.exists() else None, + "diff_uri": image_data_uri(diff_path) if diff_path.exists() else None, + }) + + elapsed = time.time() - start_time + print("Done in %.1fs: %d tests, %d failures, %d errors, %d new references." % ( + elapsed, len(results), failures, errors, new_refs)) + + report_path = output_root / "report.html" + write_report(report_path, results, list(variants.keys())) + print("Report written to: %s" % report_path) + + if failures or errors: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/headless/Compare.cpp b/headless/Compare.cpp index a91c5fbf8d..eee48b3051 100644 --- a/headless/Compare.cpp +++ b/headless/Compare.cpp @@ -305,19 +305,17 @@ std::vector TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 const u32 *pixels32 = (const u32 *)buffer->GetData(); const u16 *pixels16 = (const u16 *)buffer->GetData(); int outStride = buffer->GetStride(); - if (!buffer->GetFlipped()) { - // Bitmaps are flipped, so we have to compare backwards in this case. + if (buffer->GetFlipped()) { + // The buffer is stored bottom-up (e.g. to fit a bitmap), so read backwards + // here to get top-down output. int toLastRow = outStride * (h > buffer->GetHeight() ? buffer->GetHeight() - 1 : h - 1); pixels32 += toLastRow; pixels16 += toLastRow; outStride = -outStride; } - // Skip the bottom of the image if the buffer was smaller. Remember, we're flipped. + // The output is top-down; if the buffer was smaller, the image starts at the top. u32 *dst = &data[0]; - if (safeH < h) { - dst += (h - safeH) * stride; - } for (u32 y = 0; y < safeH; ++y) { switch (buffer->GetFormat()) { @@ -414,17 +412,19 @@ double ScreenshotComparer::Compare(const Path &screenshotFilename) { double errors = 0; if (asBitmap_) { - // The reference is flipped and BGRA by default for the common BMP compare case. + // The reference is a BMP, stored bottom-up and BGRA by default for the + // common BMP compare case, so read it backwards (pixels_ is top-down). for (u32 y = 0; y < h_; ++y) { - u32 yoff = y * referenceStride_; + u32 yoff = (h_ - y - 1) * referenceStride_; for (u32 x = 0; x < w_; ++x) errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]); } } else { + // The reference is a PNG, stored top-down, so no flip needed. // Just convert to BGRA for simplicity. ConvertRGBA8888ToBGRA8888(reference_, reference_, h_ * referenceStride_); for (u32 y = 0; y < h_; ++y) { - u32 yoff = (h_ - y - 1) * referenceStride_; + u32 yoff = y * referenceStride_; for (u32 x = 0; x < w_; ++x) errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]); } @@ -448,7 +448,9 @@ bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) { FILE *saved = File::OpenCFile(resultFilename, "wb"); if (saved) { fwrite(&header, sizeof(header), 1, saved); - fwrite(pixels_.data(), sizeof(u32), stride_ * h_, saved); + // Bitmaps are stored bottom-up, so write the rows backwards (pixels_ is top-down). + for (u32 y = 0; y < h_; ++y) + fwrite(pixels_.data() + (h_ - y - 1) * stride_, sizeof(u32), stride_, saved); fclose(saved); return true; @@ -457,23 +459,36 @@ bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) { return false; } +bool ScreenshotComparer::SaveActualPNG(const Path &resultFilename) { + std::vector rgba((size_t)stride_ * h_ * 4); + const u32 *pixels = pixels_.data(); + for (size_t i = 0; i < (size_t)stride_ * h_; ++i) { + u32 p = pixels[i]; + rgba[i * 4 + 0] = (p >> 16) & 0xFF; + rgba[i * 4 + 1] = (p >> 8) & 0xFF; + rgba[i * 4 + 2] = p & 0xFF; + rgba[i * 4 + 3] = (p >> 24) & 0xFF; + } + return pngSave(resultFilename, rgba.data(), stride_, h_, 4); +} + bool ScreenshotComparer::SaveVisualComparisonPNG(const Path &resultFilename) { std::unique_ptr comparison(new u32[w_ * 2 * h_ * 2]); if (asBitmap_) { - // The reference is flipped and BGRA by default for the common BMP compare case. + // The reference is a BMP, stored bottom-up, so read it backwards. for (u32 y = 0; y < h_; ++y) { - u32 yoff = y * referenceStride_; - u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2; + u32 yoff = (h_ - y - 1) * referenceStride_; + u32 comparisonRow = y * 2 * w_ * 2; for (u32 x = 0; x < w_; ++x) { PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]); } } } else { - // Reference is already in BGRA either way. + // Reference is a PNG, stored top-down, so no flip needed. for (u32 y = 0; y < h_; ++y) { - u32 yoff = (h_ - y - 1) * referenceStride_; - u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2; + u32 yoff = y * referenceStride_; + u32 comparisonRow = y * 2 * w_ * 2; for (u32 x = 0; x < w_; ++x) { PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]); } diff --git a/headless/Compare.h b/headless/Compare.h index 04ab899568..a9240440a9 100644 --- a/headless/Compare.h +++ b/headless/Compare.h @@ -47,6 +47,7 @@ public: } bool SaveActualBitmap(const Path &filename); + bool SaveActualPNG(const Path &filename); bool SaveVisualComparisonPNG(const Path &filename); protected: diff --git a/headless/Headless.cpp b/headless/Headless.cpp index af52010547..c2bd6c09da 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -70,7 +70,9 @@ static Path g_comparisonScreenshot; static Path g_screenshotSavePath; +static Path g_screenshotDiffPath; static double g_maxScreenshotError = 0.0; +static bool g_screenshotFailed = false; static std::string g_debugOutputBuffer; static bool g_writeFailureScreenshot = true; static bool g_writeDebugOutput = true; @@ -200,7 +202,8 @@ void System_SendDebugScreenshot(const uint8_t *data, int width, int height) { // If a screenshot save path is set, save unconditionally. if (!g_screenshotSavePath.empty()) { ScreenshotComparer saver(pixels, FRAME_STRIDE, FRAME_WIDTH, FRAME_HEIGHT); - if (saver.SaveActualBitmap(g_screenshotSavePath)) + bool saved = g_screenshotSavePath.GetFileExtension() == ".png" ? saver.SaveActualPNG(g_screenshotSavePath) : saver.SaveActualBitmap(g_screenshotSavePath); + if (saved) SendAndCollectOutput("Screenshot saved to: " + g_screenshotSavePath.ToVisualString() + "\n"); } @@ -213,10 +216,12 @@ void System_SendDebugScreenshot(const uint8_t *data, int width, int height) { double errors = comparer.Compare(g_comparisonScreenshot); if (errors < 0) { SendAndCollectOutput(comparer.GetError() + "\n"); + g_screenshotFailed = true; } if (errors > g_maxScreenshotError) { SendAndCollectOutput(StringFromFormat("Screenshot MSE: %f\n", errors)); + g_screenshotFailed = true; } if (errors > g_maxScreenshotError && g_writeFailureScreenshot) { @@ -224,6 +229,12 @@ void System_SendDebugScreenshot(const uint8_t *data, int width, int height) { SendAndCollectOutput("Actual output written to: __testfailure.bmp\n"); comparer.SaveVisualComparisonPNG(Path("__testcompare.png")); } + + // If a diff path is set, always save the visual comparison (regardless of pass/fail). + if (!g_screenshotDiffPath.empty() && errors >= 0) { + if (comparer.SaveVisualComparisonPNG(g_screenshotDiffPath)) + SendAndCollectOutput("Screenshot comparison saved to: " + g_screenshotDiffPath.ToVisualString() + "\n"); + } } static GraphicsContext *CreateGraphicsContext(GPUCore gpuCore, std::string **deviceSetting) { @@ -286,6 +297,7 @@ static bool RunAutoTest(GraphicsContext *graphicsContext, CoreParameter &corePar // Kinda ugly, trying to guesstimate the test name from filename... currentTestName = GetTestName(coreParameter.fileToStart); + g_screenshotFailed = false; std::string output; if (opt.compare || opt.bench) @@ -384,6 +396,11 @@ static bool RunAutoTest(GraphicsContext *graphicsContext, CoreParameter &corePar passed = CompareOutput(coreParameter.fileToStart, output, opt.verbose, opt.printEqualLines); } + // Screenshot comparison failures are recorded in System_SendDebugScreenshot. + if (!g_comparisonScreenshot.empty() && g_screenshotFailed) { + passed = false; + } + return passed; } @@ -472,7 +489,7 @@ int RunTests(GraphicsContext *graphicsContext, CoreParameter &coreParameter, con std::string testName = GetTestName(coreParameter.fileToStart); printf(" %s - %f seconds average\n", testName.c_str(), (et - st) / runs); } - if (testOptions.compare) { + if (testOptions.compare || !g_comparisonScreenshot.empty()) { std::string testName = GetTestName(coreParameter.fileToStart); if (passed) { passedTests.push_back(testName); @@ -483,7 +500,7 @@ int RunTests(GraphicsContext *graphicsContext, CoreParameter &coreParameter, con } } - if (testOptions.compare) { + if (testOptions.compare || !g_comparisonScreenshot.empty()) { printf("%d tests passed, %d tests failed, %d tests missing.\n", (int)passedTests.size(), (int)failedTests.size(), (int)missingTests.size()); if (!failedTests.empty()) { printf("Failed tests:\n"); @@ -762,6 +779,9 @@ int main(int argc, const char* argv[]) { if (cmdLineOptions.screenshotFilenameSave.has_value()) { SetScreenshotSavePath(Path(std::string(cmdLineOptions.screenshotFilenameSave.value()))); } + if (cmdLineOptions.screenshotFilenameDiff.has_value()) { + g_screenshotDiffPath = Path(std::string(cmdLineOptions.screenshotFilenameDiff.value())); + } SetWriteFailureScreenshot(!getenv("GITHUB_ACTIONS") && !testOptions.bench); SetWriteDebugOutput(!testOptions.compare && !testOptions.bench); diff --git a/headless/README.md b/headless/README.md index 4ce27eaffc..8e13caed3b 100644 --- a/headless/README.md +++ b/headless/README.md @@ -43,7 +43,8 @@ PPSSPPHeadless file.elf|file.prx|file.ppdmp [...] [options] | `-o`, `--odslog` | Write log to `OutputDebugString` (Windows only). | | `--graphics=` | GPU backend: `software`, `gles`, `directx11`, `vulkan`. | | `--screenshot=` | Compare the rendered output against a reference screenshot. | -| `--screenshot-save=` | Save the rendered output to a BMP file (no comparison). | +| `--screenshot-save=` | Save the rendered output to a file (PNG if the path ends in `.png`, BMP otherwise). | +| `--screenshot-diff=` | When comparing screenshots, save a visual comparison image to this file (always, regardless of pass/fail). | | `--max-mse=` | Maximum allowed Mean Squared Error for screenshot comparison (default: 0 = exact). | | `--compare` / `-c` | Compare test output with `.expected` text file and/or screenshot (see below). | | `--timeout=` | Abort test if it takes longer than this. | @@ -80,13 +81,18 @@ GE frame dumps are recordings of a single frame's GE graphics commands. When a ` 3. The framebuffer is captured automatically (512×272 stride, 480×272 visible). 4. A screenshot is sent for comparison/saving via `--compare`, `--screenshot`, or `--screenshot-save`. +For batch rendering tests over sets of frame dumps, see `docs/frametest.md` (the `frametests.py` runner). + ### Example: Generate a reference screenshot ```bash +# BMP: PPSSPPHeadless.exe --graphics=software --screenshot-save=reference.bmp frame.ppdmp +# PNG (lossless, recommended for storing references): +PPSSPPHeadless.exe --graphics=software --screenshot-save=reference.png frame.ppdmp ``` -This outputs a 512×272 BMP file with 32-bit BGRA pixel data. The file size is always 557,110 bytes (54-byte header + 512 × 272 × 4 bytes). +The BMP output is 512×272 with 32-bit BGRA pixel data. The file size is always 557,110 bytes (54-byte header + 512 × 272 × 4 bytes). PNG output is 512×272 RGBA. ### Example: Compare against a reference @@ -95,8 +101,8 @@ This outputs a 512×272 BMP file with 32-bit BGRA pixel data. The file size is a # frame.ppdmp → looks for frame.png (next to the .ppdmp) PPSSPPHeadless.exe --graphics=software --compare frame.ppdmp -# Explicit reference file: -PPSSPPHeadless.exe --graphics=software --screenshot=reference.bmp --max-mse=0.5 frame.ppdmp +# Explicit reference file (saves a visual comparison to diff.png): +PPSSPPHeadless.exe --graphics=software --screenshot=reference.bmp --screenshot-diff=diff.png --max-mse=0.5 frame.ppdmp ``` When the MSE exceeds `--max-mse`, the following files are saved in the working directory: diff --git a/libretro/LibretroGraphicsContext.cpp b/libretro/LibretroGraphicsContext.cpp index 310bb3ba39..26592b94d6 100644 --- a/libretro/LibretroGraphicsContext.cpp +++ b/libretro/LibretroGraphicsContext.cpp @@ -157,7 +157,7 @@ LibretroGraphicsContext *LibretroGraphicsContext::CreateGraphicsContext() { return ctx; } -std::vector TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 stride, u32 h) { +std::vector ConvertFramebufferForLibretro(const GPUDebugBuffer *buffer, u32 stride, u32 h) { // If the output was small, act like everything outside was 0. // This can happen depending on viewport parameters. u32 safeW = std::min(stride, buffer->GetStride()); diff --git a/libretro/LibretroGraphicsContext.h b/libretro/LibretroGraphicsContext.h index 5e0bc2ac4c..4629ee9717 100644 --- a/libretro/LibretroGraphicsContext.h +++ b/libretro/LibretroGraphicsContext.h @@ -5,6 +5,7 @@ #include "Common/GPU/GraphicsContext.h" #include "Common/GPU/thin3d_create.h" +#include "Common/CommonTypes.h" #include "Core/Config.h" #include "Core/System.h" #include "GPU/GPUState.h" @@ -67,6 +68,8 @@ protected: retro_hw_render_callback hw_render_ = {}; }; +std::vector ConvertFramebufferForLibretro(const GPUDebugBuffer *buffer, u32 stride, u32 h); + class LibretroSoftwareContext : public LibretroGraphicsContext { public: LibretroSoftwareContext() {} @@ -76,7 +79,7 @@ public: u16 h = NATIVEHEIGHT; if (gpu) { gpu->GetOutputFramebuffer(buf); - const std::vector pixels = TranslateDebugBufferToCompare(&buf, w, h); + const std::vector pixels = ConvertFramebufferForLibretro(&buf, w, h); memcpy(soft_bmp, pixels.data(), SOFT_BMP_SIZE); } u32 offset = g_Config.bDisplayCropTo16x9 ? w << 1 : 0; From 032781e301fefab55cbff628dff9da11aa7e3e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 11:08:01 +0200 Subject: [PATCH 02/14] Add headless.py wrapper: locate newest PPSSPPHeadless and forward all arguments --- headless.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 headless.py diff --git a/headless.py b/headless.py new file mode 100644 index 0000000000..5bc64d9bb3 --- /dev/null +++ b/headless.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Small wrapper that locates the newest PPSSPPHeadless binary (same way +test.py does) and forwards all command line arguments to it. + +Usage: + python3 headless.py [OPTIONS] [FILES...] +""" + +import glob +import os +import subprocess +import sys + +PPSSPP_EXECUTABLES = [ + # Windows + "Windows\\Debug\\PPSSPPHeadless.exe", + "Windows\\Release\\PPSSPPHeadless.exe", + "Windows\\x64\\Debug\\PPSSPPHeadless.exe", + "Windows\\x64\\Release\\PPSSPPHeadless.exe", + "build*/PPSSPPHeadless.exe", + "./PPSSPPHeadless.exe", + # Mac + "build*/Debug/PPSSPPHeadless", + "build*/Release/PPSSPPHeadless", + "build*/RelWithDebInfo/PPSSPPHeadless", + "build*/MinSizeRel/PPSSPPHeadless", + # Linux + "build*/PPSSPPHeadless", + "./PPSSPPHeadless", +] + + +def find_headless(): + env_path = os.environ.get("PPSSPP_HEADLESS") + if env_path and os.path.exists(env_path): + return env_path + possible_exes = [glob.glob(f) for f in PPSSPP_EXECUTABLES] + possible_exes = [x for sublist in possible_exes for x in sublist] + existing = [f for f in possible_exes if os.path.exists(f)] + if not existing: + return None + return max(existing, key=os.path.getmtime) + + +def main(): + headless = find_headless() + if headless is None: + print("ERROR: PPSSPPHeadless binary not found. Set PPSSPP_HEADLESS or run from the repo root.", file=sys.stderr) + return 2 + return subprocess.call([headless] + sys.argv[1:]) + + +if __name__ == "__main__": + sys.exit(main()) From 8c9bf04addf1c355ef59466ce436760f7490a6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 11:20:24 +0200 Subject: [PATCH 03/14] Headless: force opaque PNG screenshots by default, add --screenshot-keep-alpha Games often use the framebuffer alpha channel for non-visual purposes, so saved PNGs could look fully transparent in image viewers. Force alpha to 255 when writing PNG screenshots unless --screenshot-keep-alpha is passed. The MSE comparison ignores alpha either way. --- Core/CmdLine.cpp | 1 + Core/CmdLine.h | 2 ++ docs/frametest.md | 4 ++++ headless/Compare.cpp | 7 +++++-- headless/Compare.h | 2 +- headless/Headless.cpp | 6 +++++- headless/README.md | 1 + 7 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 86ebb822f2..1ed72500e6 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -183,6 +183,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(screenshotFilename), CmdParamType::String, "screenshot", '\0', "Compare rendered output against a reference screenshot FILE", CmdLineMode::Headless}, {POFF(screenshotFilenameSave), CmdParamType::String, "screenshot-save", '\0', "Save rendered screenshot to specified path (PNG if the path ends in .png, BMP otherwise)", CmdLineMode::Headless}, {POFF(screenshotFilenameDiff), CmdParamType::String, "screenshot-diff", '\0', "Save a visual comparison image to FILE when comparing screenshots", CmdLineMode::Headless}, + {POFF(screenshotSaveKeepAlpha), CmdParamType::Bool, "screenshot-keep-alpha", '\0', "Preserve the alpha channel when saving PNG screenshots (default: alpha is forced to 255)", CmdLineMode::Headless}, {POFF(timeout), CmdParamType::Double, "timeout", '\0', "Set the timeout value", CmdLineMode::Headless}, {POFF(maxScreenshotError), CmdParamType::Double, "max-mse", '\0', "Maximum allowed MSE error for screenshot comparison", CmdLineMode::Headless}, {POFF(mountIso), CmdParamType::String, "mount", 'm', "Mount ISO/CSO on umd1:", CmdLineMode::Headless}, diff --git a/Core/CmdLine.h b/Core/CmdLine.h index 211b5853f4..89fe4cc995 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -96,6 +96,8 @@ struct CommandLineOptions { std::optional screenshotFilename; std::optional screenshotFilenameSave; std::optional screenshotFilenameDiff; + // Headless: preserve the alpha channel when saving PNG screenshots. + std::optional screenshotSaveKeepAlpha; // Headless: mount an ISO/CSO on umd1:. std::optional mountIso; diff --git a/docs/frametest.md b/docs/frametest.md index d481ef4177..9f18e356c6 100644 --- a/docs/frametest.md +++ b/docs/frametest.md @@ -108,6 +108,10 @@ test is kept in `/logs/`. - PNG, 512×272 (480×272 display in a 512-wide framebuffer), stored top-down (row 0 = top of screen). The BMP output format is bottom-up per the BMP spec; the flip is applied only when writing BMPs. +- The alpha channel is forced to 255 when writing PNGs (games often use alpha + for non-visual purposes, which would otherwise produce transparent-looking + images); pass `--screenshot-keep-alpha` to the headless binary to preserve + it (e.g. via a variant). The MSE comparison ignores alpha either way. - Generated with `--graphics=software` they are fully deterministic: a subsequent run produces byte-identical output, so `maxMse` can be 0. - If rendering code changes the output, existing references may need diff --git a/headless/Compare.cpp b/headless/Compare.cpp index eee48b3051..359c766b95 100644 --- a/headless/Compare.cpp +++ b/headless/Compare.cpp @@ -459,7 +459,7 @@ bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) { return false; } -bool ScreenshotComparer::SaveActualPNG(const Path &resultFilename) { +bool ScreenshotComparer::SaveActualPNG(const Path &resultFilename, bool keepAlpha) { std::vector rgba((size_t)stride_ * h_ * 4); const u32 *pixels = pixels_.data(); for (size_t i = 0; i < (size_t)stride_ * h_; ++i) { @@ -467,7 +467,10 @@ bool ScreenshotComparer::SaveActualPNG(const Path &resultFilename) { rgba[i * 4 + 0] = (p >> 16) & 0xFF; rgba[i * 4 + 1] = (p >> 8) & 0xFF; rgba[i * 4 + 2] = p & 0xFF; - rgba[i * 4 + 3] = (p >> 24) & 0xFF; + // Games often use the alpha channel for non-visual purposes, which + // results in fully transparent images in PNG viewers. By default, + // force alpha to 255 when writing the image. + rgba[i * 4 + 3] = keepAlpha ? ((p >> 24) & 0xFF) : 0xFF; } return pngSave(resultFilename, rgba.data(), stride_, h_, 4); } diff --git a/headless/Compare.h b/headless/Compare.h index a9240440a9..e8b4700a21 100644 --- a/headless/Compare.h +++ b/headless/Compare.h @@ -47,7 +47,7 @@ public: } bool SaveActualBitmap(const Path &filename); - bool SaveActualPNG(const Path &filename); + bool SaveActualPNG(const Path &filename, bool keepAlpha = false); bool SaveVisualComparisonPNG(const Path &filename); protected: diff --git a/headless/Headless.cpp b/headless/Headless.cpp index c2bd6c09da..34b4587103 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -71,6 +71,7 @@ static Path g_comparisonScreenshot; static Path g_screenshotSavePath; static Path g_screenshotDiffPath; +static bool g_screenshotSaveKeepAlpha = false; static double g_maxScreenshotError = 0.0; static bool g_screenshotFailed = false; static std::string g_debugOutputBuffer; @@ -202,7 +203,7 @@ void System_SendDebugScreenshot(const uint8_t *data, int width, int height) { // If a screenshot save path is set, save unconditionally. if (!g_screenshotSavePath.empty()) { ScreenshotComparer saver(pixels, FRAME_STRIDE, FRAME_WIDTH, FRAME_HEIGHT); - bool saved = g_screenshotSavePath.GetFileExtension() == ".png" ? saver.SaveActualPNG(g_screenshotSavePath) : saver.SaveActualBitmap(g_screenshotSavePath); + bool saved = g_screenshotSavePath.GetFileExtension() == ".png" ? saver.SaveActualPNG(g_screenshotSavePath, g_screenshotSaveKeepAlpha) : saver.SaveActualBitmap(g_screenshotSavePath); if (saved) SendAndCollectOutput("Screenshot saved to: " + g_screenshotSavePath.ToVisualString() + "\n"); } @@ -782,6 +783,9 @@ int main(int argc, const char* argv[]) { if (cmdLineOptions.screenshotFilenameDiff.has_value()) { g_screenshotDiffPath = Path(std::string(cmdLineOptions.screenshotFilenameDiff.value())); } + if (cmdLineOptions.screenshotSaveKeepAlpha.has_value()) { + g_screenshotSaveKeepAlpha = cmdLineOptions.screenshotSaveKeepAlpha.value(); + } SetWriteFailureScreenshot(!getenv("GITHUB_ACTIONS") && !testOptions.bench); SetWriteDebugOutput(!testOptions.compare && !testOptions.bench); diff --git a/headless/README.md b/headless/README.md index 8e13caed3b..c6a0143e22 100644 --- a/headless/README.md +++ b/headless/README.md @@ -45,6 +45,7 @@ PPSSPPHeadless file.elf|file.prx|file.ppdmp [...] [options] | `--screenshot=` | Compare the rendered output against a reference screenshot. | | `--screenshot-save=` | Save the rendered output to a file (PNG if the path ends in `.png`, BMP otherwise). | | `--screenshot-diff=` | When comparing screenshots, save a visual comparison image to this file (always, regardless of pass/fail). | +| `--screenshot-keep-alpha` | Preserve the alpha channel when saving PNG screenshots (default: alpha is forced to 255, since games often use it for non-visual purposes). | | `--max-mse=` | Maximum allowed Mean Squared Error for screenshot comparison (default: 0 = exact). | | `--compare` / `-c` | Compare test output with `.expected` text file and/or screenshot (see below). | | `--timeout=` | Abort test if it takes longer than this. | From b0f070040b003156e8a0fdc93df5a3aed20300ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 11:36:52 +0200 Subject: [PATCH 04/14] Don't auto-enable logging in debug mode in headless, it's annoying. --- Core/CmdLine.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 1ed72500e6..6b22d561eb 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -295,10 +295,6 @@ CommandLineParseResult CommandLineOptions::Parse(int argc, const char *argv[], C constexpr std::string_view controlsOption = "--controlconfig="; constexpr std::string_view logLevelOption = "--loglevel="; -#ifdef _DEBUG - enableLogging = true; -#endif - // The rest is handled in NativeInit(). // NOTE: We don't increment i here, as we'll sometimes handle options that read the next argument. for (size_t i = 1; i < argc; ) { From b0db6f86ac96b7a9c27a963cbd9fdb4fbcb76da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 11:54:47 +0200 Subject: [PATCH 05/14] Scripts: use plain python in shebangs --- frametests.py | 2 +- headless.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frametests.py b/frametests.py index d075bd39e7..73228b931a 100644 --- a/frametests.py +++ b/frametests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Framedump rendering test runner for PPSSPPHeadless. Runs GE frame dumps (".ppdmp", possibly zipped) through PPSSPPHeadless with a diff --git a/headless.py b/headless.py index 5bc64d9bb3..ddb880155f 100644 --- a/headless.py +++ b/headless.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Small wrapper that locates the newest PPSSPPHeadless binary (same way test.py does) and forwards all command line arguments to it. From 7836d8c202369d888a141a402ddd328163cd7093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 11:58:46 +0200 Subject: [PATCH 06/14] Frametests: variant suffix/compare-suffix for shared references, output dir 'out' Variants can now declare a compare-suffix (defaulting to the variant's own suffix) to compare against another variant's reference images, enabling shared references across e.g. opengl/vulkan when their output matches. Reference images are only generated for variants whose suffix equals their compare-suffix; other variants fail clearly if the shared reference is missing. Rename the default output directory from frametest-out to out. --- .gitignore | 2 +- docs/frametest.md | 62 ++++++++++++++++++++++++++++------------------- frametests.py | 56 +++++++++++++++++++++++++++++++++--------- 3 files changed, 82 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index d58ac5312a..42043f3a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,7 @@ imgui.ini PPSSPPControls.dat # Frametest output (see frametests.py) -frametest-out/ +out/ # Gradle/Android Studio .gradle diff --git a/docs/frametest.md b/docs/frametest.md index 9f18e356c6..e429f197f3 100644 --- a/docs/frametest.md +++ b/docs/frametest.md @@ -26,7 +26,7 @@ python3 frametests.py frametests/frametests.json The first run generates reference images for any dumps that don't have them yet (status `NEW`) and compares the rest (status `PASS`/`FAIL`). A summary is printed and a self-contained HTML report is written to -`frametests/frametest-out/report.html` (default output dir; gitignored). +`frametests/out/report.html` (default output dir; gitignored). Exit code is `0` if all tests passed, `1` if anything failed, `2` on configuration/usage errors. @@ -35,18 +35,22 @@ configuration/usage errors. The configuration is a single JSON file. All relative paths are resolved against the config file's directory, so the same script works with any test -set by just pointing `--config` at a different file. +set by just pointing `--config` at a different file. The config lives with +the test set it describes (e.g. `frametests/frametests.json`) and is not +committed to the repository. ```json { "testRoot": "dumps", "refRoot": "ref", - "outputRoot": "frametest-out", + "outputRoot": "out", "headlessPath": "", "timeout": 60, "maxMse": 0.0, "variants": { - "soft": "--graphics=software" + "soft": { + "args": "--graphics=software" + } } } ``` @@ -55,42 +59,50 @@ set by just pointing `--config` at a different file. |-----------------|-----------------------------------------------------------------------------| | `testRoot` | Directory tree containing frame dumps (`.ppdmp` files, possibly wrapped in `.zip`). | | `refRoot` | Where reference images live; mirrors the `testRoot` tree. | -| `outputRoot` | Where logs, diff images, the report, and generated references go. | +| `outputRoot` | Where logs, diff images, the report, and generated references go (default `out`). | | `headlessPath` | Path to the `PPSSPPHeadless` binary. Empty = auto-detect (see below). | | `timeout` | Per-test timeout in seconds. | | `maxMse` | Maximum allowed MSE for screenshot comparison (0 = exact match). | -| `variants` | Map of variant name to command line arguments for the headless binary. | +| `variants` | Map of variant name to variant configuration (see below). | ### Variants -Each test runs once per variant. The variant name is appended to the -reference image filename, so each variant gets its own reference set: +Each test runs once per variant. A variant has a **suffix** and a +**compare-suffix**: -``` -dumps/Depth/11578 Virtua Tennis pause menu ULES00126_0002.zip - → ref/Depth/11578 Virtua Tennis pause menu ULES00126_0002-soft.png -``` - -The variant value is an arbitrary command line argument string passed to -`PPSSPPHeadless`, so new rendering configurations are just new entries (or a -different config file on a machine with the right GPU): +- The **suffix** (default: the variant name) is appended to the variant's own + output image filenames (`actuals`, `diffs`, `logs`). +- The **compare-suffix** (default: the suffix) names the reference image the + variant compares against: `-.png`. +- **Reference images are only generated for variants whose suffix matches + their compare-suffix.** This lets several variants share a single reference + set - e.g. OpenGL and Vulkan outputs should be nearly bitwise identical, so + a `gl` variant can compare against the `soft` references: ```json "variants": { - "soft": "--graphics=software", - "gl": "--graphics=opengl", - "gl-4x": "--graphics=opengl --resolution-scale=4" + "soft": { "args": "--graphics=software" }, + "gl": { "args": "--graphics=opengl", "compare-suffix": "soft" }, + "vul": { "args": "--graphics=vulkan", "compare-suffix": "soft" } } ``` +A plain string value (`"soft": "--graphics=software"`) is shorthand for +`{ "args": "...", "suffix": "soft", "compare-suffix": "soft" }`. + ## How a test runs For each dump (recursively under `testRoot`) and each variant: -- If the reference image `-.png` is **missing**: the dump is - rendered and the output saved as the new reference. Status: `NEW`. This is - how references are created - run locally, then commit the generated images - (also copied to `/generated/` for convenience). +- If the variant's reference image `-.png` is + **missing** and the variant generates references (suffix == + compare-suffix): the dump is rendered and the output saved as the new + reference. Status: `NEW`. This is how references are created - run locally, + then commit the generated images (also copied to `/generated/` + for convenience). +- If the reference is **missing** for a variant that doesn't generate + references (suffix != compare-suffix): Status: `ERROR` - the shared + reference needs to be generated by the matching variant first. - If the reference **exists**: the dump is rendered, the output saved to `/actuals/`, and compared against the reference using MSE (mean squared error over R, G, B per pixel, alpha ignored). A visual @@ -161,12 +173,12 @@ show up inline in the GitHub Actions log. A typical CI job: uses: actions/upload-artifact@v4 with: name: frametest-report - path: frametests/frametest-out/ + path: frametests/out/ ``` A missing reference image in CI means the test set is incomplete - the run fails and the generated reference is available in -`frametests/frametest-out/generated/` (part of the uploaded artifact) so it +`frametests/out/generated/` (part of the uploaded artifact) so it can be committed. ## Custom machines diff --git a/frametests.py b/frametests.py index 73228b931a..1818bf00bc 100644 --- a/frametests.py +++ b/frametests.py @@ -268,14 +268,31 @@ def main(): test_root = (config_dir / config["testRoot"]).resolve() ref_root = (config_dir / config["refRoot"]).resolve() - output_root = (config_dir / config.get("outputRoot", "frametest-out")).resolve() + output_root = (config_dir / config.get("outputRoot", "out")).resolve() timeout = float(config.get("timeout", 60)) max_mse = float(config.get("maxMse", 0.0)) - variants = config.get("variants", {}) - if not variants: + raw_variants = config.get("variants", {}) + if not raw_variants: print("ERROR: no variants defined in %s" % config_path, file=sys.stderr) return 2 + # Each variant has a suffix (used for its output images) and a compare-suffix + # (the reference image it compares against). Reference images are only + # generated for variants whose suffix matches their compare-suffix, so that + # other variants can share a reference (e.g. gl comparing against the soft + # reference). A plain string entry means args only, with both suffixes + # defaulting to the variant name. + variants = {} + for key, entry in raw_variants.items(): + if isinstance(entry, str): + args_str = entry + suffix = key + else: + args_str = entry.get("args", "") + suffix = entry.get("suffix", key) + compare_suffix = entry.get("compare-suffix", suffix) if not isinstance(entry, str) else suffix + variants[key] = {"args": shlex.split(args_str), "suffix": suffix, "compare_suffix": compare_suffix} + headless = find_headless(config_dir, config.get("headlessPath", "")) if headless is None: print("ERROR: PPSSPPHeadless binary not found. Set 'headlessPath' in %s or PPSSPP_HEADLESS, or run from the repo root." % config_path, file=sys.stderr) @@ -303,7 +320,13 @@ def main(): print("Headless: %s" % headless) print("Test root: %s" % test_root) - print("Variants: %s" % ", ".join(variants.keys())) + variant_desc = [] + for key, v in variants.items(): + if v["compare_suffix"] == v["suffix"]: + variant_desc.append(key) + else: + variant_desc.append("%s (ref: %s)" % (key, v["compare_suffix"])) + print("Variants: %s" % ", ".join(variant_desc)) print("Running %d dumps..." % len(dumps)) results = [] @@ -316,21 +339,30 @@ def main(): rel = dump.relative_to(test_root) base = strip_dump_extensions(dump.name) rel_dir = rel.parent - for variant, variant_args_str in variants.items(): - variant_args = shlex.split(variant_args_str) - ref_path = ref_root / rel_dir / ("%s-%s.png" % (base, variant)) + for variant, vinfo in variants.items(): + variant_args = vinfo["args"] + generate_refs = vinfo["suffix"] == vinfo["compare_suffix"] + ref_path = ref_root / rel_dir / ("%s-%s.png" % (base, vinfo["compare_suffix"])) actual_path = actual_dir / rel_dir / ("%s-%s.png" % (base, variant)) diff_path = diff_dir / rel_dir / ("%s-%s.png" % (base, variant)) log_path = log_dir / rel_dir / ("%s-%s.log" % (base, variant)) for p in (ref_path, actual_path, diff_path, log_path): p.parent.mkdir(parents=True, exist_ok=True) - status, mse, output, timed_out = run_test( - headless, dump, variant_args, ref_path, actual_path, diff_path, - max_mse, timeout, output_root) + if not ref_path.exists() and not generate_refs: + status = STATUS_ERROR + mse = None + output = "" + timed_out = False + else: + status, mse, output, timed_out = run_test( + headless, dump, variant_args, ref_path, actual_path, diff_path, + max_mse, timeout, output_root) detail = "" - if status == STATUS_FAIL: + if status == STATUS_ERROR and not ref_path.exists() and not generate_refs: + detail = "no reference image '%s' available for this variant (references are only generated for variants whose suffix matches their compare-suffix)" % vinfo["compare_suffix"] + elif status == STATUS_FAIL: if timed_out: detail = "timed out" elif mse is None: @@ -357,7 +389,7 @@ def main(): print("::error file=%s::%s errored (%s)" % (rel, variant, detail)) elif status == STATUS_NEW: new_refs += 1 - gen_path = generated_dir / rel_dir / ("%s-%s.png" % (base, variant)) + gen_path = generated_dir / rel_dir / ("%s-%s.png" % (base, vinfo["compare_suffix"])) gen_path.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(ref_path, gen_path) if strict: From 0d47851e92894667b27f5271873d555a3e27b3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 12:18:55 +0200 Subject: [PATCH 07/14] Add the frametests repo as a submodule --- .gitmodules | 3 +++ frametests | 1 + 2 files changed, 4 insertions(+) create mode 160000 frametests diff --git a/.gitmodules b/.gitmodules index fb5d0def50..113e40d276 100644 --- a/.gitmodules +++ b/.gitmodules @@ -69,3 +69,6 @@ path = ext/aemu_postoffice url = https://github.com/Kethen/aemu_postoffice.git branch = ppsspp +[submodule "frametests"] + path = frametests + url = git@github.com:hrydgard/ppsspp-frametest-ci.git diff --git a/frametests b/frametests new file mode 160000 index 0000000000..2e6ef2e98c --- /dev/null +++ b/frametests @@ -0,0 +1 @@ +Subproject commit 2e6ef2e98c5397e6c19e140da50c7ed9cf9f4df4 From 1fd9d3b24bb1cdc746fe0e9077f1b955280c01c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 14:00:37 +0200 Subject: [PATCH 08/14] CI: run frametests on the Linux test job and upload the report Extend the existing 'test' job (Linux leg) with the frametests run after the pspautotests, plus an artifact upload of the output so failures can be examined. Both steps are gated on the test set being present in the checkout, since dumps, references and config are maintained separately from the runner. --- .github/workflows/build.yml | 15 +++++++++++++++ docs/frametest.md | 20 ++++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e72f362c0..08b85321fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -458,6 +458,21 @@ jobs: working-directory: ${{ env.GITHUB_WORKSPACE }} run: python test.py -g --graphics=software + # Runs only when the frametests test set is present in the checkout + # (dumps + references + frametests.json are maintained separately from + # the runner and may not be committed yet). + - name: Execute frametests + if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' && hashFiles('frametests/frametests.json') != '' + working-directory: ${{ env.GITHUB_WORKSPACE }} + run: python3 frametests.py frametests/frametests.json --out-mode=failures + + - name: Upload frametest report + uses: actions/upload-artifact@v8 + if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' && hashFiles('frametests/frametests.json') != '' && always() + with: + name: frametest-report + path: frametests/out/ + test-headless-alpine: runs-on: ubuntu-latest container: diff --git a/docs/frametest.md b/docs/frametest.md index e429f197f3..c9deb69ab0 100644 --- a/docs/frametest.md +++ b/docs/frametest.md @@ -162,15 +162,23 @@ The headless binary is located, in order of preference: In CI (`GITHUB_ACTIONS` is set) the runner behaves as if `--strict` was passed and prints `::error` annotations for each failing test, so failures -show up inline in the GitHub Actions log. A typical CI job: +show up inline in the GitHub Actions log. + +The CI workflow extends the existing `test` job (Linux leg) in +`.github/workflows/build.yml`: after the pspautotests run it executes the +frametests and uploads the report as an artifact. The steps only run when +the test set (`frametests/dumps/`) is present in the checkout - the test +data and config are maintained separately from the runner and may not be +committed. Once committed, CI looks like: ```yaml -- name: Build headless - run: ./b.sh --headless -- name: Run frametests +- name: Execute frametests + if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' run: python3 frametests.py frametests/frametests.json --out-mode=failures -- name: Upload report - uses: actions/upload-artifact@v4 + +- name: Upload frametest report + uses: actions/upload-artifact@v8 + if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' && always() with: name: frametest-report path: frametests/out/ From 5ac33863c70bfcc8de46fd7ccb2f07a6d732ec07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 19:26:41 +0200 Subject: [PATCH 09/14] CI: fix upload-artifact version in frametest report step --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08b85321fd..1d5093e7bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -467,7 +467,7 @@ jobs: run: python3 frametests.py frametests/frametests.json --out-mode=failures - name: Upload frametest report - uses: actions/upload-artifact@v8 + uses: actions/upload-artifact@v7 if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' && hashFiles('frametests/frametests.json') != '' && always() with: name: frametest-report From 0e30353583bf53d21d61d86ca33339bc1be2e2b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 21:15:51 +0200 Subject: [PATCH 10/14] CI: fetch the frametests submodule in the test job and always run frametests on Linux The test set lives in the 'frametests' git submodule (hrydgard/ppsspp-frametest-ci). The test job previously only inited pspautotests, so the submodule was never synced on CI and the (now removed) hashFiles gates skipped the frametests steps. Use an HTTPS submodule URL so CI can fetch it without SSH keys. --- .github/workflows/build.yml | 9 +++------ .gitmodules | 2 +- docs/frametest.md | 34 +++++++++++++++++++--------------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1d5093e7bc..b9f8b7a201 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -423,7 +423,7 @@ jobs: - name: Fetch tests working-directory: ${{ env.GITHUB_WORKSPACE }} # Doing this to avoid ffmpeg and other large submodules. - run: git submodule update --init pspautotests + run: git submodule update --init pspautotests frametests - name: Install Linux dependencies if: runner.os == 'Linux' @@ -458,17 +458,14 @@ jobs: working-directory: ${{ env.GITHUB_WORKSPACE }} run: python test.py -g --graphics=software - # Runs only when the frametests test set is present in the checkout - # (dumps + references + frametests.json are maintained separately from - # the runner and may not be committed yet). - name: Execute frametests - if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' && hashFiles('frametests/frametests.json') != '' + if: runner.os == 'Linux' working-directory: ${{ env.GITHUB_WORKSPACE }} run: python3 frametests.py frametests/frametests.json --out-mode=failures - name: Upload frametest report uses: actions/upload-artifact@v7 - if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' && hashFiles('frametests/frametests.json') != '' && always() + if: runner.os == 'Linux' && always() with: name: frametest-report path: frametests/out/ diff --git a/.gitmodules b/.gitmodules index 113e40d276..8304380447 100644 --- a/.gitmodules +++ b/.gitmodules @@ -71,4 +71,4 @@ branch = ppsspp [submodule "frametests"] path = frametests - url = git@github.com:hrydgard/ppsspp-frametest-ci.git + url = https://github.com/hrydgard/ppsspp-frametest-ci.git diff --git a/docs/frametest.md b/docs/frametest.md index c9deb69ab0..55aed49435 100644 --- a/docs/frametest.md +++ b/docs/frametest.md @@ -11,10 +11,11 @@ The system has three parts: works against any test data tree pointed to by its configuration. - **The configuration**: a JSON file describing where the test data lives and which rendering variants to run (e.g. `frametests/frametests.json`). -- **The test set**: the frame dumps and reference images (e.g. - `frametests/dumps/` and `frametests/ref/`). Different environments can use - different test sets - GitHub CI uses the one in the repo, custom machines - can use their own (possibly much larger) ones. +- **The test set**: the frame dumps and reference images, maintained as the + `frametests` git submodule (repo `hrydgard/ppsspp-frametest-ci`, pinned by + commit - its config `frametests.json` lives inside it). Different + environments can use different test sets - GitHub CI uses the submodule, + custom machines can use their own (possibly much larger) ones. ## Quick start @@ -165,20 +166,21 @@ passed and prints `::error` annotations for each failing test, so failures show up inline in the GitHub Actions log. The CI workflow extends the existing `test` job (Linux leg) in -`.github/workflows/build.yml`: after the pspautotests run it executes the -frametests and uploads the report as an artifact. The steps only run when -the test set (`frametests/dumps/`) is present in the checkout - the test -data and config are maintained separately from the runner and may not be -committed. Once committed, CI looks like: +`.github/workflows/build.yml`: the `Fetch tests` step inits the `frametests` +submodule (alongside `pspautotests`), then after the pspautotests run it +executes the frametests and uploads the report as an artifact: ```yaml +- name: Fetch tests + run: git submodule update --init pspautotests frametests + - name: Execute frametests - if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' + if: runner.os == 'Linux' run: python3 frametests.py frametests/frametests.json --out-mode=failures - name: Upload frametest report - uses: actions/upload-artifact@v8 - if: runner.os == 'Linux' && hashFiles('frametests/dumps/**') != '' && always() + uses: actions/upload-artifact@v7 + if: runner.os == 'Linux' && always() with: name: frametest-report path: frametests/out/ @@ -208,8 +210,10 @@ new flag combinations, as long as the headless binary supports them. 1. Drop the dump into `frametests/dumps/` (either as a `.ppdmp` or zipped). 2. Run `python3 frametests.py frametests/frametests.json --filter=` - the reference image(s) are generated and copied to `/generated/`. -3. Commit the dump and the reference images under `frametests/ref/`. +3. Commit the dump and the reference images under `frametests/ref/`, and push + them to the `frametests` submodule repo (`hrydgard/ppsspp-frametest-ci`). + Then bump the submodule pointer in PPSSPP so CI picks up the new test. The CI test set (dumps and references) is maintained separately from the -runner; the `frametests/` directory is just one of several possible test sets -and may become a git submodule at some point. +runner; the `frametests/` submodule is just one of several possible test sets +- custom machines can use their own (possibly much larger) ones. From 246ebd4c36e831157f420ec8c4617ffb3c3936d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 21:37:36 +0200 Subject: [PATCH 11/14] Frametests: resolve headless path to absolute, handle launch failures The found binary path could be relative (e.g. 'PPSSPPHeadless' from the repo root in CI), but tests run with the output dir as cwd, so launching failed with FileNotFoundError. Resolve the path against the script's cwd, and turn launch failures into per-test ERROR results instead of crashing the whole run. AGENTS.md: never push without asking first. --- AGENTS.md | 1 + frametests.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 0675509672..182e247b5f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,7 @@ Ignore the folder ai_instructions in the root directory, it's old stuff from con 1. Keep style changes minimal unless requested. Follow existing code patterns and conventions. 2. Keep cross-platform parity in mind when changing shared code. See below for more multiplatform tips +3. Never `git push` (to any remote) without asking the user first. Committing locally is fine when asked; pushing requires explicit approval. ## Core Safety Checks diff --git a/frametests.py b/frametests.py index 1818bf00bc..ffcd0b31e8 100644 --- a/frametests.py +++ b/frametests.py @@ -67,7 +67,7 @@ def find_headless(config_dir, config_path): candidates.append(Path(env_path)) for pattern in HEADLESS_CANDIDATES: candidates.extend(Path(m) for m in glob.glob(pattern)) - found = [c for c in candidates if c.is_file()] + found = [c.resolve() for c in candidates if c.is_file()] if not found: return None return max(found, key=lambda p: p.stat().st_mtime) @@ -122,6 +122,9 @@ def run_test(headless, dump_path, variant_args, ref_path, actual_path, diff_path proc.kill() output, _ = proc.communicate() timed_out = True + except OSError as e: + # e.g. the binary couldn't be launched at all. + return STATUS_ERROR, None, "Failed to launch headless binary: %s\n" % e, False returncode = proc.returncode # Headless may drop these in the cwd on failure (outside GITHUB_ACTIONS). From 757dd75080ca467b088ce7ff2db3cf827d5d9fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 22:38:46 +0200 Subject: [PATCH 12/14] Frametests: bump submodule to Linux-regenerated references --- frametests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frametests b/frametests index 2e6ef2e98c..49f92977d1 160000 --- a/frametests +++ b/frametests @@ -1 +1 @@ -Subproject commit 2e6ef2e98c5397e6c19e140da50c7ed9cf9f4df4 +Subproject commit 49f92977d199e2de7f9f1fc6810e4ff15d189787 From c3705c7cef28192c7d87dd44d20f6b310247490b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 9 Aug 2026 23:01:23 +0200 Subject: [PATCH 13/14] Frametests: bump submodule (tolerate 1-pixel run-to-run noise) --- frametests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frametests b/frametests index 49f92977d1..fed5275c77 160000 --- a/frametests +++ b/frametests @@ -1 +1 @@ -Subproject commit 49f92977d199e2de7f9f1fc6810e4ff15d189787 +Subproject commit fed5275c77eaa1fe22a145e6f226bb71d9c6a691 From 66ddc3f5a26097f22566228145d6ac2a2515a48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Mon, 10 Aug 2026 00:46:56 +0200 Subject: [PATCH 14/14] Frametests: bump submodule (maxMse 0.1 for CPU float noise) --- frametests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frametests b/frametests index fed5275c77..4a0a4e4b27 160000 --- a/frametests +++ b/frametests @@ -1 +1 @@ -Subproject commit fed5275c77eaa1fe22a145e6f226bb71d9c6a691 +Subproject commit 4a0a4e4b27b0a7fbf0e2cf5cd4ed28b87e41ba67