diff --git a/android/jni/Android.mk b/android/jni/Android.mk index e5931fba84..9b9f750966 100644 --- a/android/jni/Android.mk +++ b/android/jni/Android.mk @@ -1087,7 +1087,8 @@ ifeq ($(HEADLESS),1) LOCAL_SRC_FILES := \ $(SRC)/headless/Headless.cpp \ $(SRC)/headless/ReverseEngineer.cpp \ - $(SRC)/headless/Compare.cpp + $(SRC)/headless/Compare.cpp \ + $(SRC)/headless/AndroidHeadlessGLContext.cpp include $(BUILD_EXECUTABLE) endif diff --git a/docs/frametest.md b/docs/frametest.md index ea5b0e8d10..9ec88292f9 100644 --- a/docs/frametest.md +++ b/docs/frametest.md @@ -158,6 +158,8 @@ usage: frametests.py [OPTIONS] [CONFIG.json] | `--filter=SUBSTR` | Only run dumps whose relative path contains SUBSTR (case-insensitive). | | `--strict` | Treat missing reference images as failures (a configuration error). | | `--out-mode=all\|failures` | `all` keeps everything in the output dir; `failures` keeps only artifacts of non-passing tests (smaller CI artifacts). Default: `all`. | +| `--variants=NAME=ARGS,...` | Run these variants instead of the config's. | +| `--adb[=SERIAL]` | Run on an Android device instead; see "Running on Android" below. | The headless binary is located, in order of preference: @@ -167,6 +169,23 @@ The headless binary is located, in order of preference: `Windows/x64/Debug/PPSSPPHeadless.exe`, `build/PPSSPPHeadless`), newest by modification time. +## Running on Android + +`--adb` runs every dump on an Android device over adb, with the `ppsspp_headless` executable from the +legacy NDK build, and compares each screenshot with the host's Vulkan render of the same dump (through +`Tools/image_compare.py`, so rasterizer noise doesn't count). The references in the config aren't used: +the question is whether the device's drivers draw what a desktop GPU does. + +```bash +cd android && NDK=/path/to/ndk ./ab.sh APP_ABI=arm64-v8a HEADLESS=1 SKIPAPP=1 -j8 && cd .. +python3 frametests.py frametests/frametests.json --adb +``` + +The first run pushes the binary, `assets/` and the dumps to `/data/local/tmp/ppsspp-frametests`; later runs +only push what changed. The variants default to `vulkan=--graphics=vulkan,gl=--graphics=gles`, and +`--host-args` sets how the host renders its side. A failing test's diff image shows host, device and the +differences side by side. Runs go to the device one at a time. + ## CI integration In CI (`GITHUB_ACTIONS` is set) the runner behaves as if `--strict` was diff --git a/frametests.py b/frametests.py index 5065edc2cd..d18970d54d 100644 --- a/frametests.py +++ b/frametests.py @@ -154,6 +154,78 @@ def run_test(headless, dump_path, variant_args, ref_path, actual_path, diff_path return STATUS_PASS, mse, output, False +class AdbDevice: + """An Android device running ppsspp_headless over adb, from a directory under /data/local/tmp.""" + + def __init__(self, serial, device_dir): + self.serial = serial + self.dir = device_dir.rstrip("/") + + def adb(self, *args, timeout=None, check=True): + cmd = ["adb"] + (["-s", self.serial] if self.serial else []) + list(args) + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout, + text=True, encoding="utf-8", errors="replace") + if check and proc.returncode != 0: + raise RuntimeError("%s failed:\n%s" % (" ".join(cmd), proc.stdout)) + return proc + + def shell(self, command, timeout=None, check=True): + return self.adb("shell", command, timeout=timeout, check=check) + + def setup(self, binary, assets_dir, test_root): + """Push the binary, the assets and the dumps. --sync skips files that are already there.""" + self.shell("mkdir -p %s" % shlex.quote(self.dir)) + self.adb("push", str(binary), self.dir + "/ppsspp_headless") + self.shell("chmod 755 %s/ppsspp_headless" % shlex.quote(self.dir)) + self.adb("push", "--sync", str(assets_dir) + "/.", self.dir + "/assets") + self.adb("push", "--sync", str(test_root) + "/.", self.dir + "/dumps") + self.shell("rm -rf %s/shots && mkdir -p %s/shots" % (shlex.quote(self.dir), shlex.quote(self.dir))) + model = self.shell("getprop ro.product.model").stdout.strip() + gpu = self.shell("dumpsys SurfaceFlinger | grep GLES", check=False).stdout.strip() + return "%s (%s)" % (model, gpu) if gpu else model + + +def run_side_by_side(headless, host_args, device, dump_path, rel, variant_args, host_path, actual_path, diff_path, + timeout, output_dir, compare_args): + """Run one dump on the device, and compare its screenshot approximately (Tools/image_compare.py) with + the host's, rendered with host_args. The host render is shared by all variants of a dump. + Returns (status, mse, output, timed_out, detail).""" + import image_compare + + output = "" + if not host_path.exists(): + args = [str(headless)] + list(host_args) + ["--screenshot-save=" + str(host_path), str(dump_path)] + try: + proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=str(output_dir), + timeout=timeout, text=True, encoding="utf-8", errors="replace") + output += "=== host ===\n" + proc.stdout + except subprocess.TimeoutExpired: + return STATUS_FAIL, None, output, True, "timed out on the host" + if not host_path.exists(): + return STATUS_ERROR, None, output, False, "the host run produced no screenshot" + + shot = "%s/shots/shot.png" % device.dir + device.shell("rm -f %s" % shlex.quote(shot), check=False) + command = "cd %s && ./ppsspp_headless %s --screenshot-save=%s %s" % ( + shlex.quote(device.dir), " ".join(shlex.quote(a) for a in variant_args), shlex.quote(shot), + shlex.quote("dumps/" + rel.as_posix())) + try: + proc = device.shell(command, timeout=timeout, check=False) + output += "\n=== device ===\n" + proc.stdout + except subprocess.TimeoutExpired: + device.shell("pkill -f ppsspp_headless", check=False) + return STATUS_FAIL, None, output, True, "timed out on the device" + if device.adb("pull", shot, str(actual_path), check=False).returncode != 0 or not actual_path.exists(): + return STATUS_ERROR, None, output, False, "the device run produced no screenshot" + + a, b, area, blocks, bias = image_compare.compare(image_compare.load(str(host_path)), image_compare.load(str(actual_path)), compare_args) + if not area.any() and not blocks.any(): + return STATUS_PASS, None, output, False, "" + image_compare.heatmap(str(diff_path), a, b, area, blocks, compare_args.block) + return STATUS_FAIL, None, output, False, "%d px, %d/%d blocks differ from the host (bias %+.1f %+.1f %+.1f)" % ( + area.sum(), blocks.sum(), blocks.size, bias[0], bias[1], bias[2]) + + def image_data_uri(path): """Read an image file as a PNG data URI, or None if missing.""" try: @@ -263,6 +335,19 @@ def main(): help="Treat missing reference images as failures (configuration error)") parser.add_argument("--out-mode", choices=["all", "failures"], default="all", help="What to keep in the output directory: everything, or only failures (default: all)") + parser.add_argument("--adb", nargs="?", const="", default=None, metavar="SERIAL", + help="Run each dump on an Android device over adb, and compare its screenshot approximately " + "(Tools/image_compare.py) with the host's Vulkan render instead of with the references. " + "SERIAL picks a device when several are connected") + parser.add_argument("--device-headless", default="android/libs/arm64-v8a/ppsspp_headless", + help="With --adb: the Android headless binary to push (default: android/libs/arm64-v8a/ppsspp_headless)") + parser.add_argument("--device-dir", default="/data/local/tmp/ppsspp-frametests", + help="With --adb: where on the device the binary, assets and dumps go") + parser.add_argument("--host-args", default="--graphics=vulkan", + help="With --adb: how the host renders the reference each device run is compared with (default: --graphics=vulkan)") + parser.add_argument("--variants", default="", + help="Variants to run, as name=args pairs separated by commas, replacing the config's " + "(default with --adb: vulkan=--graphics=vulkan,gl=--graphics=gles)") args = parser.parse_args() strict = args.strict or bool(os.environ.get("GITHUB_ACTIONS")) @@ -281,6 +366,10 @@ def main(): timeout = float(config.get("timeout", 60)) max_mse = float(config.get("maxMse", 0.0)) raw_variants = config.get("variants", {}) + if not args.variants and args.adb is not None: + args.variants = "vulkan=--graphics=vulkan,gl=--graphics=gles" + if args.variants: + raw_variants = dict(v.split("=", 1) for v in args.variants.split(",")) if not raw_variants: print("ERROR: no variants defined in %s" % config_path, file=sys.stderr) return 2 @@ -320,6 +409,25 @@ def main(): filter_lower = args.filter.lower() dumps = [d for d in dumps if filter_lower in str(d.relative_to(test_root)).lower()] + device = None + compare_args = None + if args.adb is not None: + sys.path.insert(0, str(Path(__file__).resolve().parent / "Tools")) + import image_compare + compare_args = argparse.Namespace(tolerance=12, radius=1, min_neighbors=5, block=8, block_tolerance=8.0, crop=(480, 272)) + device_headless = Path(args.device_headless) + if not device_headless.is_file(): + print("ERROR: Android headless binary not found: %s" % device_headless, file=sys.stderr) + return 2 + device = AdbDevice(args.adb, args.device_dir) + shutil.rmtree(output_root / "host", ignore_errors=True) + print("Pushing %s, assets and dumps to %s..." % (device_headless, device.dir)) + try: + print("Device: %s" % device.setup(device_headless, Path(__file__).resolve().parent / "assets", test_root)) + except (RuntimeError, OSError) as e: + print("ERROR: %s" % e, file=sys.stderr) + return 2 + log_dir = output_root / "logs" diff_dir = output_root / "diffs" actual_dir = output_root / "actuals" @@ -358,7 +466,15 @@ def main(): for p in (ref_path, actual_path, diff_path, log_path): p.parent.mkdir(parents=True, exist_ok=True) - if not ref_path.exists() and not generate_refs: + remote_detail = None + if device: + # The host's screenshot is the reference, so the exact references aren't involved. + ref_path = output_root / "host" / rel_dir / ("%s.png" % base) + ref_path.parent.mkdir(parents=True, exist_ok=True) + status, mse, output, timed_out, remote_detail = run_side_by_side( + headless, shlex.split(args.host_args), device, dump, rel, variant_args, ref_path, actual_path, diff_path, + timeout, output_root, compare_args) + elif not ref_path.exists() and not generate_refs: status = STATUS_ERROR mse = None output = "" @@ -369,7 +485,9 @@ def main(): max_mse, timeout, output_root) detail = "" - if status == STATUS_ERROR and not ref_path.exists() and not generate_refs: + if remote_detail is not None: + detail = remote_detail + elif status == STATUS_ERROR and not ref_path.exists() and not generate_refs: detail = "no reference image '%s' available for this variant (references are only generated for variants whose suffix matches their compare-suffix)" % vinfo["compare_suffix"] elif status == STATUS_FAIL: if timed_out: diff --git a/headless/AndroidHeadlessGLContext.cpp b/headless/AndroidHeadlessGLContext.cpp new file mode 100644 index 0000000000..381fef69c0 --- /dev/null +++ b/headless/AndroidHeadlessGLContext.cpp @@ -0,0 +1,115 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#include "ppsspp_config.h" + +#if PPSSPP_PLATFORM(ANDROID) + +#include "headless/AndroidHeadlessGLContext.h" +#include "Common/GPU/OpenGL/GLCommon.h" +#include "Common/GPU/OpenGL/GLFeatures.h" +#include "Common/GPU/thin3d_create.h" +#include "Common/StringUtils.h" +#include "Core/Config.h" +#include "Core/System.h" + +// EGL_OPENGL_ES3_BIT_KHR, from EGL_KHR_create_context. Not in every platform header. +#ifndef EGL_OPENGL_ES3_BIT_KHR +#define EGL_OPENGL_ES3_BIT_KHR 0x0040 +#endif + +bool AndroidHeadlessGLContext::InitAPI(void *wnd, std::string *deviceName, std::string *errorMessage) { + display_ = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (display_ == EGL_NO_DISPLAY || !eglInitialize(display_, nullptr, nullptr)) { + *errorMessage = StringFromFormat("eglInitialize failed: %04x", eglGetError()); + return false; + } + + // ES 3 first, like the app. ES 2 only if the driver has nothing else. + for (int version = 3; version >= 2; version--) { + const EGLint configAttribs[] = { + EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, + EGL_RENDERABLE_TYPE, version == 3 ? EGL_OPENGL_ES3_BIT_KHR : EGL_OPENGL_ES2_BIT, + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_DEPTH_SIZE, 24, + EGL_STENCIL_SIZE, 8, + EGL_NONE, + }; + EGLConfig config; + EGLint numConfigs = 0; + if (!eglChooseConfig(display_, configAttribs, &config, 1, &numConfigs) || numConfigs == 0) { + continue; + } + const EGLint contextAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, version, EGL_NONE }; + context_ = eglCreateContext(display_, config, EGL_NO_CONTEXT, contextAttribs); + if (context_ == EGL_NO_CONTEXT) { + continue; + } + const EGLint surfaceAttribs[] = { EGL_WIDTH, width_, EGL_HEIGHT, height_, EGL_NONE }; + surface_ = eglCreatePbufferSurface(display_, config, surfaceAttribs); + if (surface_ == EGL_NO_SURFACE) { + eglDestroyContext(display_, context_); + context_ = EGL_NO_CONTEXT; + continue; + } + break; + } + if (context_ == EGL_NO_CONTEXT) { + *errorMessage = StringFromFormat("No usable EGL pbuffer config: %04x", eglGetError()); + return false; + } + if (!eglMakeCurrent(display_, surface_, surface_, context_)) { + *errorMessage = StringFromFormat("eglMakeCurrent failed: %04x", eglGetError()); + return false; + } + + CheckGLExtensions(); + SetGPUBackend(GPUBackend::OPENGL); + draw_ = Draw::T3DCreateGLContext(false); + renderManager_ = (GLRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER); + renderManager_->SetInflightFrames(g_Config.iInflightFrames); + bool success = draw_->CreatePresets(); + _assert_(success); + renderManager_->SetSwapFunction([this]() { + eglSwapBuffers(display_, surface_); + }); + return success; +} + +void AndroidHeadlessGLContext::ShutdownSurface() { + delete draw_; + draw_ = nullptr; + + if (display_ != EGL_NO_DISPLAY) { + eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); + if (surface_ != EGL_NO_SURFACE) { + eglDestroySurface(display_, surface_); + } + if (context_ != EGL_NO_CONTEXT) { + eglDestroyContext(display_, context_); + } + eglTerminate(display_); + } + surface_ = EGL_NO_SURFACE; + context_ = EGL_NO_CONTEXT; + display_ = EGL_NO_DISPLAY; +} + +#endif diff --git a/headless/AndroidHeadlessGLContext.h b/headless/AndroidHeadlessGLContext.h new file mode 100644 index 0000000000..d9de2d0823 --- /dev/null +++ b/headless/AndroidHeadlessGLContext.h @@ -0,0 +1,77 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include "ppsspp_config.h" + +#if PPSSPP_PLATFORM(ANDROID) + +#include + +#include "Common/GPU/GraphicsContext.h" +#include "Common/GPU/OpenGL/GLRenderManager.h" + +// An OpenGL ES context with no window, for running headless from adb shell. It renders into an +// EGL pbuffer, which stands in for the backbuffer. +class AndroidHeadlessGLContext : public GraphicsContext { +public: + AndroidHeadlessGLContext(int width, int height) : width_(width), height_(height) {} + ~AndroidHeadlessGLContext() { delete draw_; } + + bool InitAPI(void *wnd, std::string *deviceNameSetting, std::string *errorMessage) override; + bool InitSurface(WindowSystem winsys, void *data1, void *data2, std::string *errorMessage) override { + return true; + } + void ShutdownSurface() override; + + bool NeedsSeparateEmuThread() const override { return true; } + + Draw::DrawContext *GetDrawContext() override { + return draw_; + } + + void ThreadStart() override { + renderManager_->ThreadStart(draw_); + } + + bool ThreadFrame() override { + return renderManager_->ThreadFrame(); + } + + void ThreadEnd() override { + renderManager_->ThreadEnd(); + } + + void Resize() override {} + + // Call from emu thread + void NotifyEmuThreadExit() override { + renderManager_->NotifyEmuThreadExit(); + } + +private: + Draw::DrawContext *draw_ = nullptr; + GLRenderManager *renderManager_ = nullptr; + EGLDisplay display_ = EGL_NO_DISPLAY; + EGLSurface surface_ = EGL_NO_SURFACE; + EGLContext context_ = EGL_NO_CONTEXT; + int width_; + int height_; +}; + +#endif diff --git a/headless/Headless.cpp b/headless/Headless.cpp index 3e7cb8201c..7093e286fb 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -40,7 +40,6 @@ #include "Common/CPUDetect.h" #include "Common/ExceptionHandlerSetup.h" #include "Common/File/VFS/VFS.h" -#include "Common/File/VFS/ZipFileReader.h" #include "Common/File/VFS/DirectoryReader.h" #include "Common/File/FileUtil.h" #include "GPU/Debugger/Playback.h" @@ -77,6 +76,8 @@ #include "WindowsHeadlessHost.h" #elif defined(SDL) #include "SDLHeadlessGLGraphicsContext.h" +#elif PPSSPP_PLATFORM(ANDROID) +#include "AndroidHeadlessGLContext.h" #endif static Path g_comparisonScreenshot; @@ -366,8 +367,7 @@ static GraphicsContext *CreateGraphicsContext(GPUCore gpuCore, std::string **dev *deviceSetting = nullptr; return nullptr; #elif PPSSPP_PLATFORM(ANDROID) - // The Android headless build is compile-tested only and never actually needs to create a graphics context at runtime. - // However, it would still be nice if hardware rendering worked - it can be executed over adb. + // Both hardware backends are created up front, with no window (see the caller). *deviceSetting = nullptr; return nullptr; #else @@ -1062,7 +1062,7 @@ int main(int argc, const char* argv[]) { // We don't bother with a window. graphicsContext = new NullGraphicsContext(); } else { -#if PPSSPP_PLATFORM(ANDROID) || defined(HEADLESS_NO_SDL) +#if defined(HEADLESS_NO_SDL) fprintf(stderr, "Headless graphics context creation is not supported on this platform.\n"); return 1; #else @@ -1076,8 +1076,16 @@ int main(int argc, const char* argv[]) { } else if (gpuCore == GPUCORE_GLES) { // So does OpenGL, into a framebuffer object of its own. graphicsContext = new CGLHeadlessGraphicsContext(480, 272); +#elif PPSSPP_PLATFORM(ANDROID) + } else if (gpuCore == GPUCORE_GLES) { + // Into an EGL pbuffer. + graphicsContext = new AndroidHeadlessGLContext(480, 272); #endif } else { +#if PPSSPP_PLATFORM(ANDROID) + fprintf(stderr, "Only --graphics=vulkan and --graphics=gles are supported on Android\n"); + return 1; +#else // TODO: Will we need a larger window for higher resolutions? Well, not if we use buffered rendering. window = CreateHiddenWindow(480, 272, cmdLineOptions.gpuBackend.value_or(GPUBackend::OPENGL), &windowDesc); if (!windowDesc.Valid()) { @@ -1090,6 +1098,7 @@ int main(int argc, const char* argv[]) { fprintf(stderr, "Failed to create a graphics context for GPU core\n"); return 1; } +#endif } #endif } @@ -1199,13 +1208,11 @@ int main(int argc, const char* argv[]) { // Try to find the assets flash0 directory. Often this is from a subdirectory. - // This is needed for our fallback fonts. + // This is needed for our fallback fonts. On Android, push assets/ next to the executable. Path nextPath = exePath; for (int i = 0; i < 5; ++i) { if (File::Exists(nextPath / "assets/flash0")) { -#if !PPSSPP_PLATFORM(ANDROID) g_VFS.Register("", new DirectoryReader(nextPath / "assets")); -#endif break; } @@ -1248,15 +1255,7 @@ int main(int argc, const char* argv[]) { g_writeDebugOutput = !testOptions.compare && !testOptions.bench; g_forwardHostOutput = !testOptions.compare && !testOptions.bench; -#if PPSSPP_PLATFORM(ANDROID) - // For some reason the debugger installs it with this name? - if (File::Exists(Path("/data/app/org.ppsspp.ppsspp-2.apk"))) { - g_VFS.Register("", ZipFileReader::Create(Path("/data/app/org.ppsspp.ppsspp-2.apk"), "assets/")); - } - if (File::Exists(Path("/data/app/org.ppsspp.ppsspp.apk"))) { - g_VFS.Register("", ZipFileReader::Create(Path("/data/app/org.ppsspp.ppsspp.apk"), "assets/")); - } -#elif PPSSPP_PLATFORM(LINUX) +#if PPSSPP_PLATFORM(LINUX) && !PPSSPP_PLATFORM(ANDROID) g_VFS.Register("", new DirectoryReader(Path("/usr/local/share/ppsspp/assets"))); g_VFS.Register("", new DirectoryReader(Path("/usr/local/share/games/ppsspp/assets"))); g_VFS.Register("", new DirectoryReader(Path("/usr/share/ppsspp/assets")));