diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e9b13e102..f3139a1729 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1338,7 +1338,15 @@ if(HEADLESS) SDL/SDLUtil.h ) endif() - if(APPLE) + if(IOS) + list(APPEND HeadlessSource + Common/Render/Text/draw_text_cocoa.mm + Common/Render/Text/draw_text_cocoa.h + ios/HeadlessMain.mm + ) + set_source_files_properties(Common/Render/Text/draw_text_cocoa.mm ios/HeadlessMain.mm + PROPERTIES COMPILE_FLAGS -fobjc-arc) + elseif(APPLE) list(APPEND HeadlessSource Common/Render/Text/draw_text_cocoa.mm Common/Render/Text/draw_text_cocoa.h @@ -1368,8 +1376,14 @@ if(HEADLESS) Windows/W32Util/Misc.h ) endif() - add_executable(PPSSPPHeadless ${HeadlessSource}) - target_link_libraries(PPSSPPHeadless ${COCOA_LIBRARY} ${QUARTZ_CORE_LIBRARY} ${IOKIT_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} Core) + if(IOS) + # An app bundle, set up further down with the iOS app (see ios/HeadlessMain.mm). + add_executable(PPSSPPHeadless MACOSX_BUNDLE ${HeadlessSource}) + target_link_libraries(PPSSPPHeadless ${CMAKE_THREAD_LIBS_INIT} Core) + else() + add_executable(PPSSPPHeadless ${HeadlessSource}) + target_link_libraries(PPSSPPHeadless ${COCOA_LIBRARY} ${QUARTZ_CORE_LIBRARY} ${IOKIT_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} Core) + endif() if(SDL_LIB_TARGET) target_link_libraries(PPSSPPHeadless ${SDL_LIB_TARGET}) endif() @@ -1710,6 +1724,36 @@ if(IOS AND NOT LIBRETRO) COMMAND plutil -replace CFBundleDisplayName -string "${DISPLAY_NAME}" "${APP_DIR_NAME}/Info.plist" COMMAND plutil -replace CFBundleIdentifier -string "${BUNDLE_IDENTIFIER}" "${APP_DIR_NAME}/Info.plist" ) + + if(HEADLESS AND NOT TVOS) + # PPSSPPHeadless as an app of its own (org.ppsspp.headless), for running frame dumps on a + # device. Signed for development, so devicectl can copy files in and out of its container. + target_link_libraries(PPSSPPHeadless + "${CMAKE_CURRENT_SOURCE_DIR}/ios/MoltenVK/MoltenVK.xcframework/${MOLTENVK_SLICE}/libMoltenVK.a" + "-framework Foundation -framework UIKit -framework CoreGraphics -framework CoreText -framework QuartzCore -framework Metal -framework IOSurface -framework AudioToolbox -framework AVFoundation -framework CoreMedia -framework CoreVideo -framework UniformTypeIdentifiers" + ) + if(NOT ICONV_LIBRARY) + target_link_libraries(PPSSPPHeadless iconv) + endif() + set(HEADLESS_APP_DIR "$") + add_custom_command(TARGET PPSSPPHeadless POST_BUILD + COMMAND mkdir -p \"${HEADLESS_APP_DIR}\" + COMMAND tar -c -C ${CMAKE_CURRENT_BINARY_DIR} --exclude .DS_Store --exclude .git assets | tar -x -C \"${HEADLESS_APP_DIR}\" + ) + set_target_properties(PPSSPPHeadless PROPERTIES + XCODE_GENERATE_SCHEME YES + MACOSX_BUNDLE_INFO_PLIST "${CMAKE_CURRENT_SOURCE_DIR}/ios/Headless-Info.plist" + XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER org.ppsspp.headless + XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET ${DEPLOYMENT_TARGET} + XCODE_ATTRIBUTE_TARGETED_DEVICE_FAMILY "1,2" + XCODE_ATTRIBUTE_ENABLE_BITCODE NO + XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "Apple Development" + XCODE_ATTRIBUTE_CODE_SIGN_STYLE "Automatic" + ) + if(DEVELOPMENT_TEAM_ID) + set_target_properties(PPSSPPHeadless PROPERTIES XCODE_ATTRIBUTE_DEVELOPMENT_TEAM ${DEVELOPMENT_TEAM_ID}) + endif() + endif() endif() if(MACOSX AND NOT IOS) diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index ed01b84d49..7c2bad9e61 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -206,6 +206,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(enableLogging), CmdParamType::Bool, "log", '\0', "Full log output, not just emulated printfs", CmdLineMode::Headless}, {POFF(screenshotFilename), CmdParamType::String, "screenshot", '\0', "Compare rendered output against a reference screenshot FILE", CmdLineMode::Headless}, {POFF(screenshotFilenameSave), CmdParamType::String, "screenshot-save", '\0', "Save rendered screenshot to specified path (PNG if the path ends in .png, BMP otherwise)", CmdLineMode::Headless}, + {POFF(screenshotDir), CmdParamType::String, "screenshot-dir", '\0', "Save each test's screenshot as DIR/.png (relative test paths keep their folders)", CmdLineMode::Headless}, {POFF(replayEnd), CmdParamType::Int, "replay-end", '\0', "When replaying a GE frame dump, only draw primitives up to this one (1-based)", CmdLineMode::Headless}, {POFF(dumpCompat), CmdParamType::Bool, "dump-compat", '\0', "Apply the game's compat.ini settings when replaying a GE frame dump (off by default)", CmdLineMode::Headless}, {POFF(hardwareTransform), CmdParamType::Bool, "hardware-transform", '\0', "Transform vertices on the GPU (default true); false transforms them on the CPU", CmdLineMode::Headless}, diff --git a/Core/CmdLine.h b/Core/CmdLine.h index b65522ec0a..c5b0bd0326 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -195,6 +195,8 @@ struct CommandLineOptions { std::optional screenshotFilename; std::optional screenshotFilenameSave; + // Headless: save each test's screenshot as DIR/.png, for many dumps in one run. + std::optional screenshotDir; std::optional depthFilenameSave; std::optional dumpAudioFilename; std::optional replayEnd; diff --git a/docs/frametest.md b/docs/frametest.md index c3e4ed28b7..047fd36e84 100644 --- a/docs/frametest.md +++ b/docs/frametest.md @@ -160,6 +160,7 @@ usage: frametests.py [OPTIONS] [CONFIG.json] | `--out-mode=all\|failures` | `all` keeps everything in the output dir; `failures` keeps only artifacts of non-passing tests (smaller CI artifacts). Default: `all`. | | `--variants=NAME=ARGS,...` | Run these variants instead of the config's. | | `--adb[=SERIAL]` | Run on an Android device instead; see "Running on Android" below. | +| `--ios[=DEVICE]` | Run on an iPhone or iPad instead; see "Running on iOS" below. | The headless binary is located, in order of preference: @@ -188,6 +189,46 @@ side. `--dump-compat` applies the game's `compat.ini` settings, which headless o without them some games look wrong on every backend (Coded Arms needs `AccurateSplines`). A failing test's diff image shows host, device and the differences side by side. Runs go to the device one at a time. +## Running on iOS + +`--ios` does the same on an iPhone or iPad, comparing against the host's Vulkan render the same way. +An iOS device only runs apps, so PPSSPPHeadless is built as one there (`org.ppsspp.headless`, +`ios/HeadlessMain.mm`): it starts UIKit, runs the usual headless `main` on a thread with the arguments +`devicectl` launched it with, and exits. The screen stays black meanwhile, since it renders offscreen. +It needs a development-signed build, which is what lets `devicectl` copy files in and out of it. + +```bash +cmake -S . -B build-ios-headless -GXcode -DCMAKE_TOOLCHAIN_FILE=cmake/Toolchains/ios.cmake \ + -DIOS_APP_STORE=ON -DDEVELOPMENT_TEAM_ID= -DHEADLESS=ON +xcodebuild -project build-ios-headless/PPSSPP.xcodeproj -scheme PPSSPPHeadless -configuration Release \ + -destination 'generic/platform=iOS' -allowProvisioningUpdates build +python3 frametests.py frametests/frametests.json --ios +``` + +Keep the device unlocked while it runs. frametests.py installs the app, copies the dumps the device doesn't +have yet into its `Documents/dumps`, and renders each variant's dumps in launches of up to 100 (each +`devicectl` call costs a second or two, so one launch per dump would be slow). A dump that crashes the app +ends its launch; the rest go into the next one. The variants default to `vulkan=--graphics=vulkan +--dump-compat`, as iOS has no OpenGL. There's no JIT without a debugger attached, so the device runs the +IR interpreter and decodes vertices without the vertex JIT. + +To run the app by hand: + +```bash +xcrun devicectl device copy to --device --domain-type appDataContainer \ + --domain-identifier org.ppsspp.headless --source dump.ppdmp --destination Documents/dump.ppdmp +xcrun devicectl device process launch --device --terminate-existing --console org.ppsspp.headless \ + --graphics=vulkan --screenshot-dir=shots dump.ppdmp +``` + +Relative paths are relative to `Documents`, and `--console` shows its output. `--ios-clear=DIR` +(handled by the app, not headless) deletes a folder under `Documents` first. `devicectl` has no way to +delete files, and **never pass it `--remove-existing-content`**: that empties all of `Documents`, not +just the destination folder. + +The `build-ios` folder works too if it's configured with `-DHEADLESS=ON`, as long as Xcode doesn't +have that project open: `xcodebuild` then hangs before building anything, even `xcodebuild -list`. + ## CI integration In CI (`GITHUB_ACTIONS` is set) the runner behaves as if `--strict` was diff --git a/frametests.py b/frametests.py index e4f10a6955..3eb4b11020 100644 --- a/frametests.py +++ b/frametests.py @@ -188,24 +188,171 @@ class AdbDevice: return ", ".join(s for s in (model, soc, gpu) if s) +class IosDevice: + """An iPhone or iPad running PPSSPPHeadless as an app (ios/HeadlessMain.mm), driven with devicectl. + Dumps go in the app's Documents folder. One launch renders a batch of dumps (--screenshot-dir), + since each devicectl call costs a few seconds.""" + + BUNDLE_ID = "org.ppsspp.headless" + BATCH = 100 + + def __init__(self, device): + self.device = device + + def devicectl(self, *args, timeout=None, check=True): + cmd = ["xcrun", "devicectl"] + list(args) + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout, + text=True, encoding="utf-8", errors="replace") + if check and proc.returncode != 0: + raise RuntimeError("%s failed:\n%s" % (" ".join(cmd), proc.stdout)) + return proc + + def container_args(self): + return ["--device", self.device, "--domain-type", "appDataContainer", "--domain-identifier", self.BUNDLE_ID] + + def find_device(self): + """The device to use: the one asked for, or the only connected iPhone/iPad. Returns a description.""" + import tempfile + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "devices.json") + self.devicectl("list", "devices", "--json-output", path, "--quiet") + with open(path, "r", encoding="utf-8") as f: + devices = json.load(f)["result"]["devices"] + candidates = [] + for d in devices: + hw = d.get("hardwareProperties", {}) + props = d.get("deviceProperties", {}) + conn = d.get("connectionProperties", {}) + if hw.get("reality") != "physical" or hw.get("platform") != "iOS": + continue + names = (hw.get("udid"), d.get("identifier"), props.get("name")) + if self.device and self.device not in names: + continue + if conn.get("tunnelState") == "unavailable": + continue + desc = "%s, iOS %s" % (hw.get("marketingName") or hw.get("productType"), props.get("osVersionNumber", "?")) + candidates.append((hw.get("udid"), desc)) + if len(candidates) != 1: + raise RuntimeError("expected one connected iOS device%s, found %d: %s" % ( + " matching '%s'" % self.device if self.device else "", len(candidates), candidates)) + self.device = candidates[0][0] + return candidates[0][1] + + def setup(self, app, test_root, sync_dumps): + desc = self.find_device() + self.devicectl("device", "install", "app", "--device", self.device, str(app), timeout=600) + if sync_dumps: + self.sync_dumps(test_root) + return desc + + def remote_files(self, subdirectory): + """{relative path: size} of the files under subdirectory of the app's container, or {} if it's missing.""" + import tempfile + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "files.json") + proc = self.devicectl("device", "info", "files", *self.container_args(), "--subdirectory", subdirectory, + "--json-output", path, "--quiet", check=False, timeout=600) + if proc.returncode != 0 or not os.path.exists(path): + return {} + with open(path, "r", encoding="utf-8") as f: + files = json.load(f)["result"]["files"] + return {f["relativePath"]: f["metadata"]["size"] for f in files if not f["resources"]["isDirectory"]} + + def sync_dumps(self, test_root): + """Copy the dumps that aren't on the device already (by name and size). Nothing is ever deleted: + devicectl can't, and its --remove-existing-content empties all of Documents. Dumps left over from + another test set do no harm, since only the ones asked for run.""" + remote = self.remote_files("Documents/dumps") + local = {p.relative_to(test_root).as_posix(): p.stat().st_size for p in test_root.rglob("*") if p.is_file()} + changed = sorted(rel for rel, size in local.items() if remote.get(rel) != size) + if not changed: + print("The dumps are already on the device.") + elif len(changed) > 20: + # Each copy costs a second or so, so many files go faster as one folder. + print("Copying all %d files under %s (%d new or changed)..." % (len(local), test_root, len(changed))) + self.devicectl("device", "copy", "to", *self.container_args(), "--source", str(test_root), + "--destination", "Documents/dumps", timeout=7200) + else: + print("Copying %d new or changed dumps..." % len(changed)) + for rel in changed: + self.devicectl("device", "copy", "to", *self.container_args(), "--source", str(test_root / rel), + "--destination", "Documents/dumps/" + rel, timeout=600) + + def run_batch(self, variant, variant_args, rels, timeout, local_dir, log): + """Render rels (dump paths relative to the test root) with variant_args. Screenshots end up in + local_dir/dumps/.png. A dump that crashes the app ends its batch, so the rest are retried + in a new launch. log receives the console output.""" + import tempfile + remote_shots = "shots/" + variant + # The first launch empties it, so screenshots from an earlier run can't stand in for missing ones. + clear = ["--ios-clear=" + remote_shots] + pending = list(rels) + while pending: + batch = pending[:self.BATCH] + print(" %s: %d dumps left, launching a batch of %d..." % (variant, len(pending), len(batch))) + command = ["device", "process", "launch", "--device", self.device, "--terminate-existing", "--console", + self.BUNDLE_ID] + clear + list(variant_args) + [ + "--timeout-wall=%g" % timeout, "--screenshot-dir=" + remote_shots] + ["dumps/" + r.as_posix() for r in batch] + clear = [] + try: + proc = self.devicectl(*command, timeout=timeout * len(batch) + 60, check=False) + log.append(proc.stdout) + except subprocess.TimeoutExpired: + log.append("devicectl launch timed out\n") + with tempfile.TemporaryDirectory() as tmp: + # To a folder that doesn't exist yet, which it becomes. + dest = os.path.join(tmp, "shots") + got = self.devicectl("device", "copy", "from", *self.container_args(), "--source", "Documents/" + remote_shots, + "--destination", dest, check=False, timeout=600) + if got.returncode == 0 and os.path.isdir(dest): + shutil.copytree(dest, str(local_dir), dirs_exist_ok=True) + done = [r for r in batch if (local_dir / "dumps" / (r.as_posix() + ".png")).exists()] + if not done: + # The first one took the app down (or every one of them failed); skip it. + pending = pending[1:] + continue + last = batch.index(done[-1]) + # Everything up to the last one that rendered had its turn. Of the rest, the first one is + # what ended the batch. + pending = batch[last + 2:] + pending[len(batch):] if last + 1 < len(batch) else pending[len(batch):] + + +def render_host(headless, host_args, dump_path, host_path, timeout, output_dir): + """Render dump_path on the host with host_args, unless an earlier variant already did. + Returns (output, error) where error is None or (status, timed_out, detail).""" + if host_path.exists(): + return "", None + args = [str(headless)] + list(host_args) + ["--screenshot-save=" + str(host_path), str(dump_path)] + try: + proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=str(output_dir), + timeout=timeout, text=True, encoding="utf-8", errors="replace") + except subprocess.TimeoutExpired: + return "", (STATUS_FAIL, True, "timed out on the host") + output = "=== host ===\n" + proc.stdout + if not host_path.exists(): + return output, (STATUS_ERROR, False, "the host run produced no screenshot") + return output, None + + +def compare_with_host(host_path, actual_path, diff_path, compare_args): + """Compare a device screenshot approximately (Tools/image_compare.py) with the host's. Returns (status, detail).""" + import image_compare + a, b, area, blocks, bias = image_compare.compare(image_compare.load(str(host_path)), image_compare.load(str(actual_path)), compare_args) + if not area.any() and not blocks.any(): + return STATUS_PASS, "" + image_compare.heatmap(str(diff_path), a, b, area, blocks, compare_args.block) + return STATUS_FAIL, "%d px, %d/%d blocks differ from the host (bias %+.1f %+.1f %+.1f)" % ( + area.sum(), blocks.sum(), blocks.size, bias[0], bias[1], bias[2]) + + def run_side_by_side(headless, host_args, device, dump_path, rel, variant_args, host_path, actual_path, diff_path, timeout, output_dir, compare_args): """Run one dump on the device, and compare its screenshot approximately (Tools/image_compare.py) with the host's, rendered with host_args. The host render is shared by all variants of a dump. Returns (status, mse, output, timed_out, detail).""" - import image_compare - - output = "" - if not host_path.exists(): - args = [str(headless)] + list(host_args) + ["--screenshot-save=" + str(host_path), str(dump_path)] - try: - proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=str(output_dir), - timeout=timeout, text=True, encoding="utf-8", errors="replace") - output += "=== host ===\n" + proc.stdout - except subprocess.TimeoutExpired: - return STATUS_FAIL, None, output, True, "timed out on the host" - if not host_path.exists(): - return STATUS_ERROR, None, output, False, "the host run produced no screenshot" + output, error = render_host(headless, host_args, dump_path, host_path, timeout, output_dir) + if error: + return error[0], None, output, error[1], error[2] shot = "%s/shots/shot.png" % device.dir device.shell("rm -f %s" % shlex.quote(shot), check=False) @@ -221,12 +368,22 @@ def run_side_by_side(headless, host_args, device, dump_path, rel, variant_args, if device.adb("pull", shot, str(actual_path), check=False).returncode != 0 or not actual_path.exists(): return STATUS_ERROR, None, output, False, "the device run produced no screenshot" - a, b, area, blocks, bias = image_compare.compare(image_compare.load(str(host_path)), image_compare.load(str(actual_path)), compare_args) - if not area.any() and not blocks.any(): - return STATUS_PASS, None, output, False, "" - image_compare.heatmap(str(diff_path), a, b, area, blocks, compare_args.block) - return STATUS_FAIL, None, output, False, "%d px, %d/%d blocks differ from the host (bias %+.1f %+.1f %+.1f)" % ( - area.sum(), blocks.sum(), blocks.size, bias[0], bias[1], bias[2]) + status, detail = compare_with_host(host_path, actual_path, diff_path, compare_args) + return status, None, output, False, detail + + +def ios_side_by_side(headless, host_args, dump_path, shot_path, device_log, host_path, actual_path, diff_path, + timeout, output_dir, compare_args): + """run_side_by_side for a dump the iOS device already rendered, in a batch, to shot_path.""" + output, error = render_host(headless, host_args, dump_path, host_path, timeout, output_dir) + if error: + return error[0], None, output, error[1], error[2] + output += "\n=== device (whole batch) ===\n" + device_log + if not shot_path.exists(): + return STATUS_ERROR, None, output, False, "the device run produced no screenshot (crashed or timed out?)" + shutil.copyfile(shot_path, actual_path) + status, detail = compare_with_host(host_path, actual_path, diff_path, compare_args) + return status, None, output, False, detail def image_data_uri(path): @@ -346,8 +503,15 @@ def main(): help="With --adb: the Android headless binary to push (default: android/libs/arm64-v8a/ppsspp_headless)") parser.add_argument("--device-dir", default="/data/local/tmp/ppsspp-frametests", help="With --adb: where on the device the binary, assets and dumps go") + parser.add_argument("--ios", nargs="?", const="", default=None, metavar="DEVICE", + help="Like --adb, on an iPhone or iPad over devicectl, with PPSSPPHeadless built as an app " + "(see docs/frametest.md). DEVICE (UDID or name) picks one when several are connected") + parser.add_argument("--ios-app", default="build-ios-headless/Release-iphoneos/PPSSPPHeadless.app", + help="With --ios: the app to install (default: build-ios-headless/Release-iphoneos/PPSSPPHeadless.app)") + parser.add_argument("--no-sync", action="store_true", + help="With --ios: don't copy the dumps over, they're already on the device") parser.add_argument("--host-args", default="--graphics=vulkan --dump-compat", - help="With --adb: how the host renders the reference each device run is compared with " + help="With --adb or --ios: how the host renders the reference each device run is compared with " "(default: --graphics=vulkan --dump-compat)") parser.add_argument("--variants", default="", help="Variants to run, as name=args pairs separated by commas, replacing the config's " @@ -373,6 +537,9 @@ def main(): if not args.variants and args.adb is not None: # With the game's compat settings on both sides, which is what the game would look like. args.variants = "vulkan=--graphics=vulkan --dump-compat,gl=--graphics=gles --dump-compat" + if not args.variants and args.ios is not None: + # iOS is Vulkan (MoltenVK) only. + args.variants = "vulkan=--graphics=vulkan --dump-compat" if args.variants: raw_variants = dict(v.split("=", 1) for v in args.variants.split(",")) if not raw_variants: @@ -415,17 +582,31 @@ def main(): dumps = [d for d in dumps if filter_lower in str(d.relative_to(test_root)).lower()] device = None + ios = None compare_args = None - if args.adb is not None: + if args.adb is not None or args.ios is not None: sys.path.insert(0, str(Path(__file__).resolve().parent / "Tools")) import image_compare compare_args = argparse.Namespace(tolerance=12, radius=1, min_neighbors=5, block=8, block_tolerance=8.0, crop=(480, 272)) + shutil.rmtree(output_root / "host", ignore_errors=True) + if args.ios is not None: + ios_app = Path(args.ios_app) + if not ios_app.is_dir(): + print("ERROR: iOS headless app not found: %s" % ios_app, file=sys.stderr) + return 2 + ios = IosDevice(args.ios) + print("Installing %s%s..." % (ios_app, "" if args.no_sync else " and copying the dumps")) + try: + print("Device: %s" % ios.setup(ios_app, test_root, not args.no_sync)) + except (RuntimeError, OSError, subprocess.TimeoutExpired) as e: + print("ERROR: %s" % e, file=sys.stderr) + return 2 + if args.adb is not None: device_headless = Path(args.device_headless) if not device_headless.is_file(): print("ERROR: Android headless binary not found: %s" % device_headless, file=sys.stderr) return 2 device = AdbDevice(args.adb, args.device_dir) - shutil.rmtree(output_root / "host", ignore_errors=True) print("Pushing %s, assets and dumps to %s..." % (device_headless, device.dir)) try: print("Device: %s" % device.setup(device_headless, Path(__file__).resolve().parent / "assets", test_root)) @@ -457,6 +638,17 @@ def main(): new_refs = 0 start_time = time.time() + # The device renders every dump of a variant up front, in batches. + ios_shots = output_root / "ios-shots" + ios_logs = {} + if ios: + shutil.rmtree(ios_shots, ignore_errors=True) + rels = [d.relative_to(test_root) for d in dumps] + for variant, vinfo in variants.items(): + log = [] + ios.run_batch(variant, vinfo["args"], rels, timeout, ios_shots / variant, log) + ios_logs[variant] = "".join(log) + for dump in dumps: rel = dump.relative_to(test_root) base = strip_dump_extensions(dump.name) @@ -472,7 +664,14 @@ def main(): p.parent.mkdir(parents=True, exist_ok=True) remote_detail = None - if device: + if ios: + ref_path = output_root / "host" / rel_dir / ("%s.png" % base) + ref_path.parent.mkdir(parents=True, exist_ok=True) + shot_path = ios_shots / variant / "dumps" / (rel.as_posix() + ".png") + status, mse, output, timed_out, remote_detail = ios_side_by_side( + headless, shlex.split(args.host_args), dump, shot_path, ios_logs[variant], ref_path, actual_path, + diff_path, timeout, output_root, compare_args) + elif device: # The host's screenshot is the reference, so the exact references aren't involved. ref_path = output_root / "host" / rel_dir / ("%s.png" % base) ref_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/headless/Headless.cpp b/headless/Headless.cpp index 31a88c4a15..508e9488fc 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -82,6 +82,7 @@ static Path g_comparisonScreenshot; static Path g_screenshotSavePath; +static Path g_screenshotDir; static Path g_depthSavePath; static bool g_screenshotRenderTarget = false; static Path g_screenshotDiffPath; @@ -125,6 +126,11 @@ jobject Android_GetActivity(JNIEnv *env) { bool System_AudioRecordingIsAvailable() { return false; } #endif +#if PPSSPP_PLATFORM(IOS) +// Defined by the iOS app (ios/main.mm), which this doesn't link. +bool System_AudioRecordingIsAvailable() { return false; } +#endif + // Temporary hacks around annoying linking errors. void NativeFrame(GraphicsContext *graphicsContext) { } void NativeResized() { } @@ -141,7 +147,12 @@ float System_GetPropertyFloat(SystemProperty prop) { return -1.0f; } bool System_GetPropertyBool(SystemProperty prop) { switch (prop) { case SYSPROP_IS_HEADLESS: return true; +#if PPSSPP_PLATFORM(IOS) + // Not without a debugger attached, which a devicectl launch doesn't have. + case SYSPROP_CAN_JIT: return false; +#else case SYSPROP_CAN_JIT: return true; +#endif default: return false; } @@ -366,8 +377,8 @@ static GraphicsContext *CreateGraphicsContext(GPUCore gpuCore, std::string **dev // one either. These still run fine with --graphics=software, which needs no graphics context. *deviceSetting = nullptr; return nullptr; -#elif PPSSPP_PLATFORM(ANDROID) - // Both hardware backends are created up front, with no window (see the caller). +#elif PPSSPP_PLATFORM(ANDROID) || PPSSPP_PLATFORM(IOS) + // The hardware backends are created up front, with no window (see the caller). *deviceSetting = nullptr; return nullptr; #else @@ -716,6 +727,11 @@ int RunTests(GraphicsContext *graphicsContext, CoreParameter &coreParameter, con missingTests.push_back(testFilenames[i]); continue; } + if (!g_screenshotDir.empty()) { + const Path rel(coreParameter.fileToStart.IsAbsolute() ? coreParameter.fileToStart.GetFilename() : testFilenames[i]); + g_screenshotSavePath = g_screenshotDir / (rel.ToString() + ".png"); + File::CreateFullPath(g_screenshotSavePath.NavigateUp()); + } if (testOptions.compare) { printf("%s:\n", coreParameter.fileToStart.c_str()); } @@ -778,7 +794,12 @@ public: } }; +#if PPSSPP_PLATFORM(IOS) +// An iOS app can't be a bare executable: ios/HeadlessMain.mm starts UIKit and calls this. +int HeadlessMain(int argc, const char* argv[]) { +#else int main(int argc, const char* argv[]) { +#endif PROFILE_INIT(); TimeInit(); #if PPSSPP_PLATFORM(WINDOWS) @@ -791,7 +812,7 @@ int main(int argc, const char* argv[]) { perror("Unable to ignore SIGPIPE"); } #endif -#if PPSSPP_PLATFORM(MAC) +#if PPSSPP_PLATFORM(MAC) || PPSSPP_PLATFORM(IOS) // MoltenVK logs its setup and every unsupported feature to the console, which mixes into the // test output. Only its errors, unless asked for more. setenv("MVK_CONFIG_LOG_LEVEL", "1", 0); @@ -1040,7 +1061,8 @@ int main(int argc, const char* argv[]) { g_Config.iForceEnableHLE = 0xFFFFFFFF & ~g_Config.iDisableHLE; } - CPUCore cpuCore = CPUCore::JIT; + // Where there's no JIT (iOS), allocating its executable memory gets the process killed. + CPUCore cpuCore = System_GetPropertyBool(SYSPROP_CAN_JIT) ? CPUCore::JIT : CPUCore::IR_INTERPRETER; if (cmdLineOptions.cpuCore.has_value()) { cpuCore = cmdLineOptions.cpuCore.value(); } @@ -1097,6 +1119,9 @@ int main(int argc, const char* argv[]) { #if PPSSPP_PLATFORM(ANDROID) fprintf(stderr, "Only --graphics=vulkan and --graphics=gles are supported on Android\n"); return 1; +#elif PPSSPP_PLATFORM(IOS) + fprintf(stderr, "Only --graphics=vulkan is supported on iOS\n"); + return 1; #else // TODO: Will we need a larger window for higher resolutions? Well, not if we use buffered rendering. window = CreateHiddenWindow(480, 272, cmdLineOptions.gpuBackend.value_or(GPUBackend::OPENGL), &windowDesc); @@ -1157,7 +1182,13 @@ int main(int argc, const char* argv[]) { coreParameter.pixelHeight = 272 * coreParameter.renderScaleFactor; coreParameter.fastForward = true; +#if PPSSPP_PLATFORM(IOS) + // GetExeDirectory() knows nothing on iOS, but argv[0] is the executable inside the .app, + // next to the bundled assets. + Path exePath = Path(std::string(argv[0])).NavigateUp(); +#else Path exePath = File::GetExeDirectory(); +#endif // --memstick, applied by ApplyToConfig() further up, wins. This runs after it, so without the // check the default below would silently overwrite whatever was asked for. @@ -1168,6 +1199,9 @@ int main(int argc, const char* argv[]) { g_Config.memStickDirectory = exePath / "memstick"; File::CreateDir(g_Config.memStickDirectory, true); CreateSysDirectories(); +#elif PPSSPP_PLATFORM(IOS) + // The app container's root isn't writable. + g_Config.memStickDirectory = Path(std::string(getenv("HOME"))) / "Library" / "memstick"; #elif !PPSSPP_PLATFORM(ANDROID) g_Config.memStickDirectory = Path(std::string(getenv("HOME"))) / ".ppsspp"; #endif @@ -1256,6 +1290,9 @@ int main(int argc, const char* argv[]) { if (cmdLineOptions.screenshotFilename.has_value()) { SetComparisonScreenshot(Path(std::string(cmdLineOptions.screenshotFilename.value())), testOptions.maxScreenshotError); } + if (cmdLineOptions.screenshotDir.has_value()) { + g_screenshotDir = Path(cmdLineOptions.screenshotDir.value()); + } if (cmdLineOptions.screenshotFilenameSave.has_value()) { SetScreenshotSavePath(Path(std::string(cmdLineOptions.screenshotFilenameSave.value()))); } @@ -1378,7 +1415,7 @@ int main(int argc, const char* argv[]) { ShutdownWebServer(); } -#if PPSSPP_PLATFORM(ANDROID) || defined(HEADLESS_NO_SDL) +#if PPSSPP_PLATFORM(ANDROID) || PPSSPP_PLATFORM(IOS) || defined(HEADLESS_NO_SDL) // ... see above #else if (window) { diff --git a/headless/README.md b/headless/README.md index 63866f869e..7c6a17bebd 100644 --- a/headless/README.md +++ b/headless/README.md @@ -44,6 +44,7 @@ PPSSPPHeadless file.elf|file.prx|file.ppdmp [...] [options] | `--graphics=` | GPU backend: `software`, `gles`, `directx11`, `vulkan`. | | `--screenshot=` | Compare the rendered output against a reference screenshot. | | `--screenshot-save=` | Save the rendered output to a file (PNG if the path ends in `.png`, BMP otherwise). | +| `--screenshot-dir=` | Save each test's output as `/.png`, so one run can render many dumps. | | `--screenshot-diff=` | When comparing screenshots, save a visual comparison image to this file (always, regardless of pass/fail). | | `--screenshot-keep-alpha` | Preserve the alpha channel when saving PNG screenshots (default: alpha is forced to 255, since games often use it for non-visual purposes). | | `--max-mse=` | Maximum allowed Mean Squared Error for screenshot comparison (default: 0 = exact). | diff --git a/ios/Headless-Info.plist b/ios/Headless-Info.plist new file mode 100644 index 0000000000..57237fab42 --- /dev/null +++ b/ios/Headless-Info.plist @@ -0,0 +1,49 @@ + + + + + CFBundleDisplayName + PPSSPPHeadless + CFBundleExecutable + PPSSPPHeadless + CFBundleIdentifier + org.ppsspp.headless + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + PPSSPPHeadless + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1.0 + LSRequiresIPhoneOS + + UIRequiredDeviceCapabilities + + arm64 + + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + UIWindowSceneSessionRoleApplication + + + UISceneConfigurationName + Default + UISceneDelegateClassName + HeadlessSceneDelegate + + + + + UILaunchScreen + + UIFileSharingEnabled + + + diff --git a/ios/HeadlessMain.mm b/ios/HeadlessMain.mm new file mode 100644 index 0000000000..776f1ed72e --- /dev/null +++ b/ios/HeadlessMain.mm @@ -0,0 +1,99 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +// PPSSPPHeadless as an iOS app. iOS only runs app bundles, and kills one that doesn't bring up +// UIKit, so this starts UIKit and runs the headless main on a thread of its own, with the +// arguments devicectl launched it with: +// +// xcrun devicectl device process launch --console --device org.ppsspp.headless +// +// Relative paths are resolved against the app's Documents directory, which +// `devicectl device copy to/from --domain-type appDataContainer` reaches. See frametests.py --ios. + +#import + +#include +#include +#include +#include +#include +#include + +int HeadlessMain(int argc, const char *argv[]); + +// DarwinFileSystemServices presents its pickers on the app's view controller. Headless has none. +UIViewController *sharedViewController = nil; + +// iOS 27 stops an app that doesn't adopt the scene lifecycle, even one that draws nothing. +@interface HeadlessSceneDelegate : UIResponder +@property (strong, nonatomic) UIWindow *window; +@end + +@implementation HeadlessSceneDelegate +@end + +@interface HeadlessAppDelegate : UIResponder +@end + +@implementation HeadlessAppDelegate + +- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + NSString *documents = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES).firstObject; + chdir(documents.UTF8String); + + NSArray *arguments = [NSProcessInfo processInfo].arguments; + std::vector args; + for (NSString *arg in arguments) { + // --ios-clear=DIR deletes a folder under Documents first, since devicectl can't delete + // anything (its --remove-existing-content empties all of Documents). + if ([arg hasPrefix:@"--ios-clear="]) { + NSString *dir = [arg substringFromIndex:12]; + if (dir.length == 0 || [dir hasPrefix:@"/"] || [dir containsString:@".."]) { + printf("--ios-clear needs a folder inside Documents: %s\n", dir.UTF8String); + exit(1); + } + [[NSFileManager defaultManager] removeItemAtPath:[documents stringByAppendingPathComponent:dir] error:nil]; + continue; + } + args.push_back(arg.UTF8String); + } + + std::thread([args]() { + std::vector argv; + for (const std::string &arg : args) { + argv.push_back(arg.c_str()); + } + argv.push_back(nullptr); + int ret = HeadlessMain((int)args.size(), argv.data()); + fflush(stdout); + fflush(stderr); + exit(ret); + }).detach(); + return YES; +} + +@end + +int main(int argc, char *argv[]) { + // devicectl --console only forwards stdout, from a pipe, which would otherwise be block buffered. + dup2(STDOUT_FILENO, STDERR_FILENO); + setvbuf(stdout, nullptr, _IOLBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + @autoreleasepool { + return UIApplicationMain(argc, argv, nil, NSStringFromClass([HeadlessAppDelegate class])); + } +}