Headless: Vulkan and OpenGL ES on Android, and frametests.py --adb

The Android headless build only did software rendering. Vulkan takes the
offscreen path desktop headless already uses, and OpenGL ES renders into an
EGL pbuffer, so both run from adb shell with no window. Assets are found next
to the executable, like on desktop, instead of in an APK path that no longer
exists.

frametests.py --adb pushes the binary, assets and dumps to a device, runs each
dump there with Vulkan and GLES, and compares the screenshots with the host's
Vulkan render using Tools/image_compare.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Henrik RydgårdandClaude Opus 5.5 committed 2026-10-13 14:44:34 +02:00
1 parent 6a15aff789
commit c7d60a5cfb
6 files changed
+348 -19

No files matched your search

+2 -1
View File
@@ -1087,7 +1087,8 @@ ifeq ($(HEADLESS),1)
LOCAL_SRC_FILES := \
$(SRC)/headless/Headless.cpp \
$(SRC)/headless/ReverseEngineer.cpp \
$(SRC)/headless/Compare.cpp
$(SRC)/headless/Compare.cpp \
$(SRC)/headless/AndroidHeadlessGLContext.cpp
include $(BUILD_EXECUTABLE)
endif
+19
View File
@@ -158,6 +158,8 @@ usage: frametests.py [OPTIONS] [CONFIG.json]
| `--filter=SUBSTR` | Only run dumps whose relative path contains SUBSTR (case-insensitive). |
| `--strict` | Treat missing reference images as failures (a configuration error). |
| `--out-mode=all\|failures` | `all` keeps everything in the output dir; `failures` keeps only artifacts of non-passing tests (smaller CI artifacts). Default: `all`. |
| `--variants=NAME=ARGS,...` | Run these variants instead of the config's. |
| `--adb[=SERIAL]` | Run on an Android device instead; see "Running on Android" below. |
The headless binary is located, in order of preference:
@@ -167,6 +169,23 @@ The headless binary is located, in order of preference:
`Windows/x64/Debug/PPSSPPHeadless.exe`, `build/PPSSPPHeadless`), newest
by modification time.
## Running on Android
`--adb` runs every dump on an Android device over adb, with the `ppsspp_headless` executable from the
legacy NDK build, and compares each screenshot with the host's Vulkan render of the same dump (through
`Tools/image_compare.py`, so rasterizer noise doesn't count). The references in the config aren't used:
the question is whether the device's drivers draw what a desktop GPU does.
```bash
cd android && NDK=/path/to/ndk ./ab.sh APP_ABI=arm64-v8a HEADLESS=1 SKIPAPP=1 -j8 && cd ..
python3 frametests.py frametests/frametests.json --adb
```
The first run pushes the binary, `assets/` and the dumps to `/data/local/tmp/ppsspp-frametests`; later runs
only push what changed. The variants default to `vulkan=--graphics=vulkan,gl=--graphics=gles`, and
`--host-args` sets how the host renders its side. A failing test's diff image shows host, device and the
differences side by side. Runs go to the device one at a time.
## CI integration
In CI (`GITHUB_ACTIONS` is set) the runner behaves as if `--strict` was
+120 -2
View File
@@ -154,6 +154,78 @@ def run_test(headless, dump_path, variant_args, ref_path, actual_path, diff_path
return STATUS_PASS, mse, output, False
class AdbDevice:
"""An Android device running ppsspp_headless over adb, from a directory under /data/local/tmp."""
def __init__(self, serial, device_dir):
self.serial = serial
self.dir = device_dir.rstrip("/")
def adb(self, *args, timeout=None, check=True):
cmd = ["adb"] + (["-s", self.serial] if self.serial else []) + list(args)
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout,
text=True, encoding="utf-8", errors="replace")
if check and proc.returncode != 0:
raise RuntimeError("%s failed:\n%s" % (" ".join(cmd), proc.stdout))
return proc
def shell(self, command, timeout=None, check=True):
return self.adb("shell", command, timeout=timeout, check=check)
def setup(self, binary, assets_dir, test_root):
"""Push the binary, the assets and the dumps. --sync skips files that are already there."""
self.shell("mkdir -p %s" % shlex.quote(self.dir))
self.adb("push", str(binary), self.dir + "/ppsspp_headless")
self.shell("chmod 755 %s/ppsspp_headless" % shlex.quote(self.dir))
self.adb("push", "--sync", str(assets_dir) + "/.", self.dir + "/assets")
self.adb("push", "--sync", str(test_root) + "/.", self.dir + "/dumps")
self.shell("rm -rf %s/shots && mkdir -p %s/shots" % (shlex.quote(self.dir), shlex.quote(self.dir)))
model = self.shell("getprop ro.product.model").stdout.strip()
gpu = self.shell("dumpsys SurfaceFlinger | grep GLES", check=False).stdout.strip()
return "%s (%s)" % (model, gpu) if gpu else model
def run_side_by_side(headless, host_args, device, dump_path, rel, variant_args, host_path, actual_path, diff_path,
timeout, output_dir, compare_args):
"""Run one dump on the device, and compare its screenshot approximately (Tools/image_compare.py) with
the host's, rendered with host_args. The host render is shared by all variants of a dump.
Returns (status, mse, output, timed_out, detail)."""
import image_compare
output = ""
if not host_path.exists():
args = [str(headless)] + list(host_args) + ["--screenshot-save=" + str(host_path), str(dump_path)]
try:
proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=str(output_dir),
timeout=timeout, text=True, encoding="utf-8", errors="replace")
output += "=== host ===\n" + proc.stdout
except subprocess.TimeoutExpired:
return STATUS_FAIL, None, output, True, "timed out on the host"
if not host_path.exists():
return STATUS_ERROR, None, output, False, "the host run produced no screenshot"
shot = "%s/shots/shot.png" % device.dir
device.shell("rm -f %s" % shlex.quote(shot), check=False)
command = "cd %s && ./ppsspp_headless %s --screenshot-save=%s %s" % (
shlex.quote(device.dir), " ".join(shlex.quote(a) for a in variant_args), shlex.quote(shot),
shlex.quote("dumps/" + rel.as_posix()))
try:
proc = device.shell(command, timeout=timeout, check=False)
output += "\n=== device ===\n" + proc.stdout
except subprocess.TimeoutExpired:
device.shell("pkill -f ppsspp_headless", check=False)
return STATUS_FAIL, None, output, True, "timed out on the device"
if device.adb("pull", shot, str(actual_path), check=False).returncode != 0 or not actual_path.exists():
return STATUS_ERROR, None, output, False, "the device run produced no screenshot"
a, b, area, blocks, bias = image_compare.compare(image_compare.load(str(host_path)), image_compare.load(str(actual_path)), compare_args)
if not area.any() and not blocks.any():
return STATUS_PASS, None, output, False, ""
image_compare.heatmap(str(diff_path), a, b, area, blocks, compare_args.block)
return STATUS_FAIL, None, output, False, "%d px, %d/%d blocks differ from the host (bias %+.1f %+.1f %+.1f)" % (
area.sum(), blocks.sum(), blocks.size, bias[0], bias[1], bias[2])
def image_data_uri(path):
"""Read an image file as a PNG data URI, or None if missing."""
try:
@@ -263,6 +335,19 @@ def main():
help="Treat missing reference images as failures (configuration error)")
parser.add_argument("--out-mode", choices=["all", "failures"], default="all",
help="What to keep in the output directory: everything, or only failures (default: all)")
parser.add_argument("--adb", nargs="?", const="", default=None, metavar="SERIAL",
help="Run each dump on an Android device over adb, and compare its screenshot approximately "
"(Tools/image_compare.py) with the host's Vulkan render instead of with the references. "
"SERIAL picks a device when several are connected")
parser.add_argument("--device-headless", default="android/libs/arm64-v8a/ppsspp_headless",
help="With --adb: the Android headless binary to push (default: android/libs/arm64-v8a/ppsspp_headless)")
parser.add_argument("--device-dir", default="/data/local/tmp/ppsspp-frametests",
help="With --adb: where on the device the binary, assets and dumps go")
parser.add_argument("--host-args", default="--graphics=vulkan",
help="With --adb: how the host renders the reference each device run is compared with (default: --graphics=vulkan)")
parser.add_argument("--variants", default="",
help="Variants to run, as name=args pairs separated by commas, replacing the config's "
"(default with --adb: vulkan=--graphics=vulkan,gl=--graphics=gles)")
args = parser.parse_args()
strict = args.strict or bool(os.environ.get("GITHUB_ACTIONS"))
@@ -281,6 +366,10 @@ def main():
timeout = float(config.get("timeout", 60))
max_mse = float(config.get("maxMse", 0.0))
raw_variants = config.get("variants", {})
if not args.variants and args.adb is not None:
args.variants = "vulkan=--graphics=vulkan,gl=--graphics=gles"
if args.variants:
raw_variants = dict(v.split("=", 1) for v in args.variants.split(","))
if not raw_variants:
print("ERROR: no variants defined in %s" % config_path, file=sys.stderr)
return 2
@@ -320,6 +409,25 @@ def main():
filter_lower = args.filter.lower()
dumps = [d for d in dumps if filter_lower in str(d.relative_to(test_root)).lower()]
device = None
compare_args = None
if args.adb is not None:
sys.path.insert(0, str(Path(__file__).resolve().parent / "Tools"))
import image_compare
compare_args = argparse.Namespace(tolerance=12, radius=1, min_neighbors=5, block=8, block_tolerance=8.0, crop=(480, 272))
device_headless = Path(args.device_headless)
if not device_headless.is_file():
print("ERROR: Android headless binary not found: %s" % device_headless, file=sys.stderr)
return 2
device = AdbDevice(args.adb, args.device_dir)
shutil.rmtree(output_root / "host", ignore_errors=True)
print("Pushing %s, assets and dumps to %s..." % (device_headless, device.dir))
try:
print("Device: %s" % device.setup(device_headless, Path(__file__).resolve().parent / "assets", test_root))
except (RuntimeError, OSError) as e:
print("ERROR: %s" % e, file=sys.stderr)
return 2
log_dir = output_root / "logs"
diff_dir = output_root / "diffs"
actual_dir = output_root / "actuals"
@@ -358,7 +466,15 @@ def main():
for p in (ref_path, actual_path, diff_path, log_path):
p.parent.mkdir(parents=True, exist_ok=True)
if not ref_path.exists() and not generate_refs:
remote_detail = None
if device:
# The host's screenshot is the reference, so the exact references aren't involved.
ref_path = output_root / "host" / rel_dir / ("%s.png" % base)
ref_path.parent.mkdir(parents=True, exist_ok=True)
status, mse, output, timed_out, remote_detail = run_side_by_side(
headless, shlex.split(args.host_args), device, dump, rel, variant_args, ref_path, actual_path, diff_path,
timeout, output_root, compare_args)
elif not ref_path.exists() and not generate_refs:
status = STATUS_ERROR
mse = None
output = ""
@@ -369,7 +485,9 @@ def main():
max_mse, timeout, output_root)
detail = ""
if status == STATUS_ERROR and not ref_path.exists() and not generate_refs:
if remote_detail is not None:
detail = remote_detail
elif status == STATUS_ERROR and not ref_path.exists() and not generate_refs:
detail = "no reference image '%s' available for this variant (references are only generated for variants whose suffix matches their compare-suffix)" % vinfo["compare_suffix"]
elif status == STATUS_FAIL:
if timed_out:
+115
View File
@@ -0,0 +1,115 @@
// Copyright (c) 2012- PPSSPP Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0 or later versions.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
#include "ppsspp_config.h"
#if PPSSPP_PLATFORM(ANDROID)
#include "headless/AndroidHeadlessGLContext.h"
#include "Common/GPU/OpenGL/GLCommon.h"
#include "Common/GPU/OpenGL/GLFeatures.h"
#include "Common/GPU/thin3d_create.h"
#include "Common/StringUtils.h"
#include "Core/Config.h"
#include "Core/System.h"
// EGL_OPENGL_ES3_BIT_KHR, from EGL_KHR_create_context. Not in every platform header.
#ifndef EGL_OPENGL_ES3_BIT_KHR
#define EGL_OPENGL_ES3_BIT_KHR 0x0040
#endif
bool AndroidHeadlessGLContext::InitAPI(void *wnd, std::string *deviceName, std::string *errorMessage) {
display_ = eglGetDisplay(EGL_DEFAULT_DISPLAY);
if (display_ == EGL_NO_DISPLAY || !eglInitialize(display_, nullptr, nullptr)) {
*errorMessage = StringFromFormat("eglInitialize failed: %04x", eglGetError());
return false;
}
// ES 3 first, like the app. ES 2 only if the driver has nothing else.
for (int version = 3; version >= 2; version--) {
const EGLint configAttribs[] = {
EGL_SURFACE_TYPE, EGL_PBUFFER_BIT,
EGL_RENDERABLE_TYPE, version == 3 ? EGL_OPENGL_ES3_BIT_KHR : EGL_OPENGL_ES2_BIT,
EGL_RED_SIZE, 8,
EGL_GREEN_SIZE, 8,
EGL_BLUE_SIZE, 8,
EGL_ALPHA_SIZE, 8,
EGL_DEPTH_SIZE, 24,
EGL_STENCIL_SIZE, 8,
EGL_NONE,
};
EGLConfig config;
EGLint numConfigs = 0;
if (!eglChooseConfig(display_, configAttribs, &config, 1, &numConfigs) || numConfigs == 0) {
continue;
}
const EGLint contextAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, version, EGL_NONE };
context_ = eglCreateContext(display_, config, EGL_NO_CONTEXT, contextAttribs);
if (context_ == EGL_NO_CONTEXT) {
continue;
}
const EGLint surfaceAttribs[] = { EGL_WIDTH, width_, EGL_HEIGHT, height_, EGL_NONE };
surface_ = eglCreatePbufferSurface(display_, config, surfaceAttribs);
if (surface_ == EGL_NO_SURFACE) {
eglDestroyContext(display_, context_);
context_ = EGL_NO_CONTEXT;
continue;
}
break;
}
if (context_ == EGL_NO_CONTEXT) {
*errorMessage = StringFromFormat("No usable EGL pbuffer config: %04x", eglGetError());
return false;
}
if (!eglMakeCurrent(display_, surface_, surface_, context_)) {
*errorMessage = StringFromFormat("eglMakeCurrent failed: %04x", eglGetError());
return false;
}
CheckGLExtensions();
SetGPUBackend(GPUBackend::OPENGL);
draw_ = Draw::T3DCreateGLContext(false);
renderManager_ = (GLRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
renderManager_->SetInflightFrames(g_Config.iInflightFrames);
bool success = draw_->CreatePresets();
_assert_(success);
renderManager_->SetSwapFunction([this]() {
eglSwapBuffers(display_, surface_);
});
return success;
}
void AndroidHeadlessGLContext::ShutdownSurface() {
delete draw_;
draw_ = nullptr;
if (display_ != EGL_NO_DISPLAY) {
eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
if (surface_ != EGL_NO_SURFACE) {
eglDestroySurface(display_, surface_);
}
if (context_ != EGL_NO_CONTEXT) {
eglDestroyContext(display_, context_);
}
eglTerminate(display_);
}
surface_ = EGL_NO_SURFACE;
context_ = EGL_NO_CONTEXT;
display_ = EGL_NO_DISPLAY;
}
#endif
+77
View File
@@ -0,0 +1,77 @@
// Copyright (c) 2012- PPSSPP Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0 or later versions.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
#pragma once
#include "ppsspp_config.h"
#if PPSSPP_PLATFORM(ANDROID)
#include <EGL/egl.h>
#include "Common/GPU/GraphicsContext.h"
#include "Common/GPU/OpenGL/GLRenderManager.h"
// An OpenGL ES context with no window, for running headless from adb shell. It renders into an
// EGL pbuffer, which stands in for the backbuffer.
class AndroidHeadlessGLContext : public GraphicsContext {
public:
AndroidHeadlessGLContext(int width, int height) : width_(width), height_(height) {}
~AndroidHeadlessGLContext() { delete draw_; }
bool InitAPI(void *wnd, std::string *deviceNameSetting, std::string *errorMessage) override;
bool InitSurface(WindowSystem winsys, void *data1, void *data2, std::string *errorMessage) override {
return true;
}
void ShutdownSurface() override;
bool NeedsSeparateEmuThread() const override { return true; }
Draw::DrawContext *GetDrawContext() override {
return draw_;
}
void ThreadStart() override {
renderManager_->ThreadStart(draw_);
}
bool ThreadFrame() override {
return renderManager_->ThreadFrame();
}
void ThreadEnd() override {
renderManager_->ThreadEnd();
}
void Resize() override {}
// Call from emu thread
void NotifyEmuThreadExit() override {
renderManager_->NotifyEmuThreadExit();
}
private:
Draw::DrawContext *draw_ = nullptr;
GLRenderManager *renderManager_ = nullptr;
EGLDisplay display_ = EGL_NO_DISPLAY;
EGLSurface surface_ = EGL_NO_SURFACE;
EGLContext context_ = EGL_NO_CONTEXT;
int width_;
int height_;
};
#endif
+15 -16
View File
@@ -40,7 +40,6 @@
#include "Common/CPUDetect.h"
#include "Common/ExceptionHandlerSetup.h"
#include "Common/File/VFS/VFS.h"
#include "Common/File/VFS/ZipFileReader.h"
#include "Common/File/VFS/DirectoryReader.h"
#include "Common/File/FileUtil.h"
#include "GPU/Debugger/Playback.h"
@@ -77,6 +76,8 @@
#include "WindowsHeadlessHost.h"
#elif defined(SDL)
#include "SDLHeadlessGLGraphicsContext.h"
#elif PPSSPP_PLATFORM(ANDROID)
#include "AndroidHeadlessGLContext.h"
#endif
static Path g_comparisonScreenshot;
@@ -366,8 +367,7 @@ static GraphicsContext *CreateGraphicsContext(GPUCore gpuCore, std::string **dev
*deviceSetting = nullptr;
return nullptr;
#elif PPSSPP_PLATFORM(ANDROID)
// The Android headless build is compile-tested only and never actually needs to create a graphics context at runtime.
// However, it would still be nice if hardware rendering worked - it can be executed over adb.
// Both hardware backends are created up front, with no window (see the caller).
*deviceSetting = nullptr;
return nullptr;
#else
@@ -1062,7 +1062,7 @@ int main(int argc, const char* argv[]) {
// We don't bother with a window.
graphicsContext = new NullGraphicsContext();
} else {
#if PPSSPP_PLATFORM(ANDROID) || defined(HEADLESS_NO_SDL)
#if defined(HEADLESS_NO_SDL)
fprintf(stderr, "Headless graphics context creation is not supported on this platform.\n");
return 1;
#else
@@ -1076,8 +1076,16 @@ int main(int argc, const char* argv[]) {
} else if (gpuCore == GPUCORE_GLES) {
// So does OpenGL, into a framebuffer object of its own.
graphicsContext = new CGLHeadlessGraphicsContext(480, 272);
#elif PPSSPP_PLATFORM(ANDROID)
} else if (gpuCore == GPUCORE_GLES) {
// Into an EGL pbuffer.
graphicsContext = new AndroidHeadlessGLContext(480, 272);
#endif
} else {
#if PPSSPP_PLATFORM(ANDROID)
fprintf(stderr, "Only --graphics=vulkan and --graphics=gles are supported on Android\n");
return 1;
#else
// TODO: Will we need a larger window for higher resolutions? Well, not if we use buffered rendering.
window = CreateHiddenWindow(480, 272, cmdLineOptions.gpuBackend.value_or(GPUBackend::OPENGL), &windowDesc);
if (!windowDesc.Valid()) {
@@ -1090,6 +1098,7 @@ int main(int argc, const char* argv[]) {
fprintf(stderr, "Failed to create a graphics context for GPU core\n");
return 1;
}
#endif
}
#endif
}
@@ -1199,13 +1208,11 @@ int main(int argc, const char* argv[]) {
// Try to find the assets flash0 directory. Often this is from a subdirectory.
// This is needed for our fallback fonts.
// This is needed for our fallback fonts. On Android, push assets/ next to the executable.
Path nextPath = exePath;
for (int i = 0; i < 5; ++i) {
if (File::Exists(nextPath / "assets/flash0")) {
#if !PPSSPP_PLATFORM(ANDROID)
g_VFS.Register("", new DirectoryReader(nextPath / "assets"));
#endif
break;
}
@@ -1248,15 +1255,7 @@ int main(int argc, const char* argv[]) {
g_writeDebugOutput = !testOptions.compare && !testOptions.bench;
g_forwardHostOutput = !testOptions.compare && !testOptions.bench;
#if PPSSPP_PLATFORM(ANDROID)
// For some reason the debugger installs it with this name?
if (File::Exists(Path("/data/app/org.ppsspp.ppsspp-2.apk"))) {
g_VFS.Register("", ZipFileReader::Create(Path("/data/app/org.ppsspp.ppsspp-2.apk"), "assets/"));
}
if (File::Exists(Path("/data/app/org.ppsspp.ppsspp.apk"))) {
g_VFS.Register("", ZipFileReader::Create(Path("/data/app/org.ppsspp.ppsspp.apk"), "assets/"));
}
#elif PPSSPP_PLATFORM(LINUX)
#if PPSSPP_PLATFORM(LINUX) && !PPSSPP_PLATFORM(ANDROID)
g_VFS.Register("", new DirectoryReader(Path("/usr/local/share/ppsspp/assets")));
g_VFS.Register("", new DirectoryReader(Path("/usr/local/share/games/ppsspp/assets")));
g_VFS.Register("", new DirectoryReader(Path("/usr/share/ppsspp/assets")));