#!/usr/bin/env python
"""Framedump rendering test runner for PPSSPPHeadless.
Runs GE frame dumps (".ppdmp", possibly zipped) through PPSSPPHeadless with a
set of rendering variants (command line option sets), generating reference
images when missing and comparing against existing ones when present. Produces
a self-contained HTML report and returns a nonzero exit code on failure.
This script is test-set agnostic: it reads its configuration from a JSON file
(similar to frametests/frametests.json) that points at the test data tree,
so it can be used against different test sets (the CI one, or bigger private
ones on custom machines).
Usage:
frametests.py [OPTIONS] [CONFIG.json]
Example:
python3 frametests.py frametests/frametests.json
"""
import argparse
import base64
import glob
import html
import json
import os
import re
import shlex
import shutil
import subprocess
import platform
import sys
import time
from pathlib import Path
# test.py-style candidate paths for the headless binary, relative to the
# current working directory, in preference order.
# The machine's own architecture comes first, the same way test.py picks: an x64 build runs on
# Windows-on-ARM too, under emulation, so looking for it first quietly tests the emulated build.
HEADLESS_CANDIDATES = ([
"Windows/ARM64/Debug/PPSSPPHeadless.exe",
"Windows/ARM64/Release/PPSSPPHeadless.exe",
] if platform.machine().lower() in ("arm64", "aarch64") else []) + [
"Windows/x64/Debug/PPSSPPHeadless.exe",
"Windows/Debug/PPSSPPHeadless.exe",
"Windows/x64/Release/PPSSPPHeadless.exe",
"Windows/Release/PPSSPPHeadless.exe",
"build/PPSSPPHeadless",
"build-headless/PPSSPPHeadless",
"build*/PPSSPPHeadless",
"PPSSPPHeadless",
"ppsspp/PPSSPPHeadless",
]
MSE_RE = re.compile(r"Screenshot MSE: ([0-9.eE+-]+)")
LOG_EMBED_LIMIT = 64 * 1024
STATUS_PASS = "PASS"
STATUS_FAIL = "FAIL"
STATUS_NEW = "NEW"
STATUS_ERROR = "ERROR"
def find_headless(config_dir, config_path):
"""Locate the PPSSPPHeadless binary. The config headlessPath wins, then
the PPSSPP_HEADLESS env var, then candidate paths (newest by mtime)."""
candidates = []
if config_path:
p = Path(config_path)
candidates.append(p if p.is_absolute() else config_dir / p)
env_path = os.environ.get("PPSSPP_HEADLESS")
if env_path:
candidates.append(Path(env_path))
for pattern in HEADLESS_CANDIDATES:
candidates.extend(Path(m) for m in glob.glob(pattern))
found = [c.resolve() for c in candidates if c.is_file()]
if not found:
return None
return max(found, key=lambda p: p.stat().st_mtime)
def strip_dump_extensions(name):
"""Derive the base name for a dump, stripping .zip and .ppdmp extensions."""
base = name
lower = base.lower()
if lower.endswith(".zip"):
base = base[:-4]
lower = base.lower()
if lower.endswith(".ppdmp"):
base = base[:-6]
return base
def collect_dumps(test_root):
"""Recursively collect frame dumps (.ppdmp files and .zip wrappers)."""
dumps = []
for dirpath, dirnames, filenames in os.walk(test_root):
dirnames[:] = [d for d in sorted(dirnames) if not d.startswith(".")]
for filename in sorted(filenames):
lower = filename.lower()
if lower.endswith(".ppdmp") or lower.endswith(".zip"):
dumps.append(Path(dirpath) / filename)
return sorted(dumps)
def run_test(headless, dump_path, variant_args, ref_path, actual_path, diff_path, max_mse, timeout, output_dir):
"""Run one dump with one variant's args. Returns (status, mse, output, timed_out)."""
generate = not ref_path.exists()
args = [str(headless)] + list(variant_args)
if generate:
args.append("--screenshot-save=" + str(ref_path))
else:
args.extend([
"--screenshot-save=" + str(actual_path),
"--screenshot=" + str(ref_path),
"--max-mse=" + str(max_mse),
"--screenshot-diff=" + str(diff_path),
])
args.append(str(dump_path))
timed_out = False
try:
proc = subprocess.Popen(
args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=str(output_dir), text=True, encoding="utf-8", errors="replace")
output, _ = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
output, _ = proc.communicate()
timed_out = True
except OSError as e:
# e.g. the binary couldn't be launched at all.
return STATUS_ERROR, None, "Failed to launch headless binary: %s\n" % e, False
returncode = proc.returncode
# Headless may drop these in the cwd on failure (outside GITHUB_ACTIONS).
for stray in ("__testfailure.bmp", "__testcompare.png"):
try:
(output_dir / stray).unlink()
except OSError:
pass
match = MSE_RE.search(output)
mse = float(match.group(1)) if match else None
if timed_out:
return STATUS_FAIL, mse, output, True
if generate:
if returncode != 0 or not ref_path.exists():
return STATUS_ERROR, mse, output, False
return STATUS_NEW, mse, output, False
if returncode != 0:
return STATUS_FAIL, mse, output, False
return STATUS_PASS, mse, output, False
class AdbDevice:
"""An Android device running ppsspp_headless over adb, from a directory under /data/local/tmp."""
def __init__(self, serial, device_dir):
self.serial = serial
self.dir = device_dir.rstrip("/")
def adb(self, *args, timeout=None, check=True):
cmd = ["adb"] + (["-s", self.serial] if self.serial else []) + list(args)
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout,
text=True, encoding="utf-8", errors="replace")
if check and proc.returncode != 0:
raise RuntimeError("%s failed:\n%s" % (" ".join(cmd), proc.stdout))
return proc
def shell(self, command, timeout=None, check=True):
return self.adb("shell", command, timeout=timeout, check=check)
def setup(self, binary, assets_dir, test_root):
"""Push the binary, the assets and the dumps. --sync skips files that are already there."""
self.shell("mkdir -p %s" % shlex.quote(self.dir))
self.adb("push", str(binary), self.dir + "/ppsspp_headless")
self.shell("chmod 755 %s/ppsspp_headless" % shlex.quote(self.dir))
self.adb("push", "--sync", str(assets_dir) + "/.", self.dir + "/assets")
self.adb("push", "--sync", str(test_root) + "/.", self.dir + "/dumps")
self.shell("rm -rf %s/shots && mkdir -p %s/shots" % (shlex.quote(self.dir), shlex.quote(self.dir)))
model = self.shell("getprop ro.product.model").stdout.strip()
soc = self.shell("getprop ro.soc.model", check=False).stdout.strip()
# SurfaceFlinger names the GPU, except where it reports its own render engine instead (Samsung's Skia).
gles = self.shell("dumpsys SurfaceFlinger | grep GLES", check=False).stdout.strip()
gpu = next((l.strip() for l in gles.splitlines() if re.search(r"Adreno|Mali|Immortalis|PowerVR|Xclipse", l)), "")
return ", ".join(s for s in (model, soc, gpu) if s)
def run_side_by_side(headless, host_args, device, dump_path, rel, variant_args, host_path, actual_path, diff_path,
timeout, output_dir, compare_args):
"""Run one dump on the device, and compare its screenshot approximately (Tools/image_compare.py) with
the host's, rendered with host_args. The host render is shared by all variants of a dump.
Returns (status, mse, output, timed_out, detail)."""
import image_compare
output = ""
if not host_path.exists():
args = [str(headless)] + list(host_args) + ["--screenshot-save=" + str(host_path), str(dump_path)]
try:
proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=str(output_dir),
timeout=timeout, text=True, encoding="utf-8", errors="replace")
output += "=== host ===\n" + proc.stdout
except subprocess.TimeoutExpired:
return STATUS_FAIL, None, output, True, "timed out on the host"
if not host_path.exists():
return STATUS_ERROR, None, output, False, "the host run produced no screenshot"
shot = "%s/shots/shot.png" % device.dir
device.shell("rm -f %s" % shlex.quote(shot), check=False)
command = "cd %s && ./ppsspp_headless %s --screenshot-save=%s %s" % (
shlex.quote(device.dir), " ".join(shlex.quote(a) for a in variant_args), shlex.quote(shot),
shlex.quote("dumps/" + rel.as_posix()))
try:
proc = device.shell(command, timeout=timeout, check=False)
output += "\n=== device ===\n" + proc.stdout
except subprocess.TimeoutExpired:
device.shell("pkill -f ppsspp_headless", check=False)
return STATUS_FAIL, None, output, True, "timed out on the device"
if device.adb("pull", shot, str(actual_path), check=False).returncode != 0 or not actual_path.exists():
return STATUS_ERROR, None, output, False, "the device run produced no screenshot"
a, b, area, blocks, bias = image_compare.compare(image_compare.load(str(host_path)), image_compare.load(str(actual_path)), compare_args)
if not area.any() and not blocks.any():
return STATUS_PASS, None, output, False, ""
image_compare.heatmap(str(diff_path), a, b, area, blocks, compare_args.block)
return STATUS_FAIL, None, output, False, "%d px, %d/%d blocks differ from the host (bias %+.1f %+.1f %+.1f)" % (
area.sum(), blocks.sum(), blocks.size, bias[0], bias[1], bias[2])
def image_data_uri(path):
"""Read an image file as a PNG data URI, or None if missing."""
try:
data = path.read_bytes()
except OSError:
return None
return "data:image/png;base64," + base64.b64encode(data).decode("ascii")
def write_report(report_path, results, variants):
rows = []
for res in results:
cls = "failed" if res["status"] in (STATUS_FAIL, STATUS_ERROR) else ("new" if res["status"] == STATUS_NEW else "")
mse = ('MSE: %.6f' % res["mse"]) if res["mse"] is not None else ""
detail = '%s' % html.escape(res["detail"]) if res["detail"] else ""
ref_img = '' % res["ref_uri"] if res["ref_uri"] else ""
actual_img = '
' % res["actual_uri"] if res["actual_uri"] else ""
diff_img = '
' % res["diff_uri"] if res["diff_uri"] else ""
log = ('
log
%s
{summary}
| variant | results |
|---|