Headless: Run several instances that talk to each other over ad hoc

--instance=N sets PPSSPP_ID, so each instance gets its own 127.0.0.N address
and MAC; headless left it at 0, which made its address 127.0.0.0.
--realtime keeps emulated time from running ahead of the wall clock. Only
display flips paced headless, and a test that doesn't draw never flips, so
two instances waiting for each other over the network timed out at once.
--adhoc-server runs the built-in ad hoc server.

Tools/adhoc_multi.py runs a multi-console pspautotest (X_host, X_client,
X_client2, ...) on one instance per role, with the built-in server or, with
--relay, a local aemu_postoffice in relay mode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Core/CmdLine.cpp

@@ -254,6 +254,9 @@ static const CommandLineParam g_autoParams[] = {
 	{POFF(timeoutEmulated), CmdParamType::Double, "timeout-emulated", '\0', "Stop the run after this many emulated seconds", CmdLineMode::Headless},
 	// The old name for --timeout-wall, kept working because it's in a lot of scripts.
 	{POFF(timeoutWall), CmdParamType::Double, "timeout", '\0', "Alias for --timeout-wall", CmdLineMode::Headless},
+	{POFF(instance), CmdParamType::Int, "instance", '\0', "Run as instance N of several talking to each other over ad hoc (own 127.0.0.N address and MAC)", CmdLineMode::Headless},
+	{POFF(realtime), CmdParamType::Bool, "realtime", '\0', "Run at PSP speed instead of as fast as possible", CmdLineMode::Headless},
+	{POFF(adhocServer), CmdParamType::Bool, "adhoc-server", '\0', "Run the built-in ad hoc server"},
 	{POFF(frameStats), CmdParamType::Bool, "frame-stats", '\0', "Print flips per emulated second at the end of the run, counted from --state's load", CmdLineMode::Headless},
 	{POFF(maxScreenshotError), CmdParamType::Double, "max-mse", '\0', "Maximum allowed MSE error for screenshot comparison", CmdLineMode::Headless},
 	{POFF(mountIso), CmdParamType::String, "mount", 'm', "Mount ISO/CSO on umd1: when booting an ELF/PBP or --vsh"},
@@ -566,6 +569,10 @@ CommandLineParseResult CommandLineOptions::Parse(int argc, const char *argv[], C
 }
 
 void CommandLineOptions::ApplyToConfig() const {
+	if (adhocServer.has_value()) {
+		g_Config.bEnableAdhocServer = adhocServer.value();
+		g_Config.DoNotSaveSetting(&g_Config.bEnableAdhocServer);
+	}
 	if (fuseId.has_value()) {
 		g_Config.sFuseID = fuseId.value();
 		g_Config.DoNotSaveSetting(&g_Config.sFuseID);

Core/CmdLine.h

@@ -194,6 +194,12 @@ struct CommandLineOptions {
 	std::optional<bool> printEqualLines;
 	// Headless: print flips per emulated second at the end, counted from the loaded state (if any).
 	std::optional<bool> frameStats;
+	// Headless: which of several instances running at once this is (sets PPSSPP_ID, and with it the
+	// instance's 127.0.0.N address and MAC), and whether to run at PSP speed rather than flat out.
+	// Both are for ad hoc tests where instances talk to each other.
+	std::optional<int> instance;
+	std::optional<bool> realtime;
+	std::optional<bool> adhocServer;
 
 	std::optional<std::string> screenshotFilename;
 	std::optional<std::string> screenshotFilenameSave;

Tools/adhoc_multi.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# Runs a multi-console ad hoc pspautotest on several PPSSPPHeadless instances at once, the way
+# tests/net/adhoc/pair/record.py runs it on several PSPs.
+#
+# A test NAME is a set of programs NAME_host.prx, NAME_client.prx, NAME_client2.prx, ... with an
+# .expected each. The host runs as instance 1 with the built-in ad hoc server, each client as the next
+# instance, all at PSP speed (--realtime) since they wait for each other over the network. Instance N
+# binds to 127.0.0.N, so on macOS the loopback aliases have to exist first:
+#   sudo ifconfig lo0 alias 127.0.0.2 up   (and .3, .4, ... for more clients)
+#
+# With --relay, the instances instead use an aemu_postoffice server already running on this machine
+# (adhocctl on 27312, relay on 27313; see ext/aemu_postoffice/hosting.md), in relay mode. The relay
+# tells players apart by MAC, so they can all share 127.0.0.1 and no aliases are needed.
+#
+# Usage: python3 Tools/adhoc_multi.py [--headless PATH] [--relay] net/adhoc/pair/basic [more tests...]
+
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+import time
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+TESTS_DIR = os.path.join(ROOT, "pspautotests", "tests")
+TIMEOUT = 120
+
+
+def find_headless():
+	candidates = []
+	for d in os.listdir(ROOT):
+		for name in ("PPSSPPHeadless", "PPSSPPHeadless.exe"):
+			path = os.path.join(ROOT, d, name)
+			if d.startswith("build") and os.path.isfile(path):
+				candidates.append(path)
+	if not candidates:
+		sys.exit("No PPSSPPHeadless found in build*/; pass --headless")
+	return max(candidates, key=os.path.getmtime)
+
+
+def roles_of(test):
+	base = os.path.join(TESTS_DIR, test)
+	roles = ["host", "client"]
+	n = 2
+	while os.path.exists("%s_client%d.prx" % (base, n)):
+		roles.append("client%d" % n)
+		n += 1
+	for role in roles:
+		if not os.path.exists("%s_%s.prx" % (base, role)):
+			sys.exit("Missing %s_%s.prx" % (test, role))
+	return roles
+
+
+RELAY_INI = """[Network]
+proAdhocServer = 127.0.0.1
+AdhocServerRelayMode = 1
+"""
+
+
+def run_test(headless, test, extra, relay_ini):
+	roles = roles_of(test)
+	memstick = os.path.join(os.path.dirname(headless), "memstick")
+	procs = []
+	for i, role in enumerate(roles):
+		cmd = [headless, "--root", os.path.join(ROOT, "pspautotests") + "/", "--compare", "--timeout-wall=%d" % TIMEOUT,
+			"--graphics=software", "--memstick=" + memstick, "--realtime", "--instance=%d" % (i + 1)]
+		if relay_ini:
+			cmd.append("--appendconfig=" + relay_ini)
+		elif i == 0:
+			cmd.append("--adhoc-server")
+		cmd += extra
+		cmd.append(os.path.join(TESTS_DIR, "%s_%s.prx" % (test, role)))
+		procs.append((role, subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)))
+		if i == 0:
+			# Let the host's built-in server come up before the clients log in to it.
+			time.sleep(1.0)
+
+	passed = True
+	for role, proc in procs:
+		out, _ = proc.communicate(timeout=TIMEOUT + 30)
+		ok = proc.returncode == 0
+		passed = passed and ok
+		print("%s %s_%s" % ("PASS" if ok else "FAIL", test, role))
+		if not ok:
+			sys.stdout.write(out.decode("utf-8", "replace"))
+	return passed
+
+
+def main():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("--headless", default=None)
+	parser.add_argument("--relay", action="store_true", help="Use a local aemu_postoffice server in relay mode")
+	parser.add_argument("tests", nargs="+")
+	args, extra = parser.parse_known_args()
+	headless = args.headless or find_headless()
+	relay_ini = None
+	if args.relay:
+		f = tempfile.NamedTemporaryFile("w", suffix=".ini", delete=False)
+		f.write(RELAY_INI)
+		f.close()
+		relay_ini = f.name
+	failed = [t for t in args.tests if not run_test(headless, t, extra, relay_ini)]
+	print("%d of %d passed" % (len(args.tests) - len(failed), len(args.tests)))
+	return 1 if failed else 0
+
+
+if __name__ == "__main__":
+	sys.exit(main())

docs/pspautotests-hardware.md

@@ -297,6 +297,46 @@ hardware preserves the original case in `d_name` (`readme.txt`) where PPSSPP upp
 uppercase 8.3 (`readme.txt` -> `README~1.TXT`, `MiXeD.txt` -> `MIXED~1.TXT`) where PPSSPP only does
 so on a collision.
 
+## Tests that need two PSPs
+
+The ad hoc tests in `tests/net/adhoc/pair` and `tests/net/inet/pair` need two consoles talking to each
+other. Each test is two programs, `X_host` and `X_client`, built from one `X.inc`, each with its own
+`.expected`. They print only relative facts ("peer", "local"), so the files don't depend on which
+console ran which side.
+
+Both PSPs are connected over USB, each with its own bridge, and the second bridge needs its own base
+port (a bridge takes 8 ports from its base, so 3001 is taken) and its own root, so the two consoles'
+`__testoutput.txt` don't collide:
+
+```bash
+cd pspautotests && usbhostfs_pc -b 3000 &
+cd pspautotests/tests && usbhostfs_pc -b 4000 . &
+```
+
+Which console each bridge claims isn't up to you, and can change after a reconnect; check with
+`pspsh -p 3000 -e pspver`. Then record both sides at once:
+
+```bash
+python3 tests/net/adhoc/pair/record.py --host-port 3000 basic
+python3 tests/net/adhoc/pair/record.py --host-port 3000 net/inet/pair/lan
+```
+
+This is the one case where two test programs run at once - one per console, never two on one PSP.
+Things learned the hard way:
+
+- **Sync the two sides, don't sleep.** `pair.h` has `pairSync(step)`, which exchanges step numbers over a
+  PDP port until both sides have reached the same point. Anything that depends on how fast the other
+  side gets somewhere is a flaky test.
+- **Print buckets, not counts or times.** Radio loss and timing vary: print "45 or more of 50", "1-2s".
+- **Run each pair test twice** and diff the results before believing them.
+- **make doesn't track `.inc` and header dependencies by itself**; the Makefiles here add them. Without
+  that, a change to `pair.h` silently runs the old binary.
+- **Real access points need slack.** The first unicast to a new IP can be lost while ARP resolves, and a
+  weak signal stalls TCP for seconds.
+
+To run the same tests on several PPSSPP instances, see `Tools/adhoc_multi.py` and
+[pspautotests.md](pspautotests.md#multi-console-ad-hoc-tests).
+
 ## Committing
 
 `pspautotests` is a git submodule, so a new test is two commits:

docs/pspautotests.md

@@ -83,6 +83,24 @@ python test.py video/mp4/avchold video/mpeg/decodeend --graphics=software --mems
 CI has no firmware dump, so these tests stay in `tests_next`, and a change to the video path
 needs this run done by hand.
 
+### Multi-console ad hoc tests
+
+The tests in `tests/net/adhoc/pair` (one host program and one client program each, see
+[pspautotests-hardware.md](pspautotests-hardware.md#tests-that-need-two-psps)) need several emulators
+running at once, so `test.py` doesn't run them. `Tools/adhoc_multi.py` does: it starts one
+PPSSPPHeadless per role with `--realtime` (PSP speed, since the sides wait for each other over the real
+network) and `--instance=N` (its own 127.0.0.N address and MAC), the host with `--adhoc-server`, and
+compares each against its own `.expected`:
+
+```bash
+python3 Tools/adhoc_multi.py net/adhoc/pair/basic
+```
+
+Instance N binds to 127.0.0.N, which macOS doesn't have by default:
+`sudo ifconfig lo0 alias 127.0.0.2 up` (and .3, ... for more clients). With `--relay`, the instances
+use an aemu_postoffice server running locally (build it from `ext/aemu_postoffice/server_cpp`) in relay
+mode instead, which needs no aliases.
+
 ### What you'll see
 
 **Passing test:**

headless/Headless.cpp

@@ -55,6 +55,7 @@
 #include "Core/CmdLine.h"
 #include "Common/Net/HTTPRequest.h"
 #include "Core/Config.h"
+#include "Core/Instance.h"
 #include "Core/ConfigValues.h"
 #include "Core/Core.h"
 #include "Core/CoreTiming.h"
@@ -431,6 +432,8 @@ struct AutoTestOptions {
 	bool bench;
 	bool printEqualLines;
 	bool frameStats;
+	// Keep emulated time from running ahead of the wall clock (--realtime).
+	bool realtime;
 	// What --disable-hle asked for, or 0 if it was not passed. Only an explicit request binds:
 	// sceMpeg and sceMp4 run the firmware module by default now and fall back to the HLE wherever
 	// none is installed, which must not fail every run on such a machine.
@@ -656,6 +659,15 @@ static bool RunAutoTest(GraphicsContext *graphicsContext, CoreParameter &corePar
 
 		// Finer steps when input is scripted, so short presses land.
 		int blockTicks = (int)usToCycles(1000000 / (opt.presses.empty() ? 10 : 60));
+		if (opt.realtime) {
+			// Display flips are the only thing that paces headless otherwise, and a test that doesn't
+			// draw never flips, so the emulated clock would race ahead of instances it talks to.
+			blockTicks = (int)usToCycles(1000);
+			const double ahead = emulatedElapsed - (time_now_d() - startTime);
+			if (ahead > 0.001) {
+				sleep_ms((int)(std::min(ahead, 0.05) * 1000.0), "realtime");
+			}
+		}
 		PSP_RunLoopFor(blockTicks);
 
 		// If we were rendering, this might be a nice time to do something about it.
@@ -952,6 +964,7 @@ int main(int argc, const char* argv[]) {
 	testOptions.verbose = cmdLineOptions.verbose.value_or(false);
 	testOptions.printEqualLines = cmdLineOptions.printEqualLines.value_or(false);
 	testOptions.frameStats = cmdLineOptions.frameStats.value_or(false);
+	testOptions.realtime = cmdLineOptions.realtime.value_or(false);
 	// BUTTONS is a sceCtrl mask (hex with 0x), DURATION defaults to 0.15s.
 	if (cmdLineOptions.pressScript.has_value()) {
 		std::vector<std::string> items;
@@ -1178,6 +1191,9 @@ int main(int argc, const char* argv[]) {
 	// ApplyToConfig() has the final say, applied after RestoreDefaults() and the headless
 	// overrides above, so a matching command line flag always wins.
 	cmdLineOptions.ApplyToConfig();
+	// Instances talking to each other over ad hoc need their own 127.0.0.N address and MAC, which come
+	// from PPSSPP_ID. Headless doesn't run the instance counter the app uses to pick one.
+	PPSSPP_ID = (uint8_t)cmdLineOptions.instance.value_or(1);
 
 	// pspautotests is plain homebrew PRXes so do not ship user libraries that a retail disc may carry. 
 	// So we must use HLE, unless we install firmware.
@@ -1320,7 +1336,8 @@ int main(int argc, const char* argv[]) {
 	coreParameter.renderHeight = 272 * coreParameter.renderScaleFactor;
 	coreParameter.pixelWidth = 480 * coreParameter.renderScaleFactor;
 	coreParameter.pixelHeight = 272 * coreParameter.renderScaleFactor;
-	coreParameter.fastForward = true;
+	// Ad hoc tests between instances wait for each other over the real network, so they need PSP speed.
+	coreParameter.fastForward = !cmdLineOptions.realtime.value_or(false);
 
 #if PPSSPP_PLATFORM(IOS)
 	// GetExeDirectory() knows nothing on iOS, but argv[0] is the executable inside the .app,