test.py: Run the two-console ad hoc tests in relay and P2P mode, and in CI

The pair tests that pass go in tests_good, and each runs twice: through
the built-in server's relay and P2P. pair_known_failures lists the ones
that fail in only one mode. A P2P test that needs a loopback address
the machine lacks (macOS has only 127.0.0.1) is skipped with a warning,
and the run fails at the end with the command that adds it. The macOS
CI runners add 127.0.0.2. Each run gets a random port offset.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/build.yml

@@ -550,6 +550,12 @@ jobs:
       timeout-minutes: 30
       run: build/PPSSPPUnitTest ALL
 
+    # The two-console ad hoc tests run in P2P too, where instance N binds to 127.0.0.N. Linux and
+    # Windows route all of 127/8 to the loopback; macOS needs the address added.
+    - name: Add a loopback address for the P2P ad hoc tests
+      if: matrix.extra == 'test' && runner.os == 'macOS'
+      run: sudo ifconfig lo0 alias 127.0.0.2 up
+
     - name: Execute headless tests
       if: matrix.extra == 'test' && runner.os != 'Linux'
       timeout-minutes: 30

Tools/adhoc_multi.py

@@ -17,6 +17,8 @@
 
 import argparse
 import os
+import random
+import socket
 import subprocess
 import sys
 import tempfile
@@ -58,8 +60,33 @@ AdhocServerRelayMode = 1
 """
 
 
+def missing_loopback_aliases(count):
+	# In P2P mode instance N binds to 127.0.0.N. Linux and Windows route all of 127/8 to the loopback;
+	# macOS only has the addresses that were added.
+	missing = []
+	for n in range(2, count + 1):
+		s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+		try:
+			s.bind(("127.0.0.%d" % n, 0))
+		except OSError:
+			missing.append("127.0.0.%d" % n)
+		finally:
+			s.close()
+	return missing
+
+
 def run_test(headless, test, extra, relay_ini):
 	roles = roles_of(test)
+	mode = "relay" if relay_ini else "p2p"
+	if not relay_ini:
+		missing = missing_loopback_aliases(len(roles))
+		if missing:
+			print("FAIL %s [p2p]: %s missing. Add with: %s" % (test, ", ".join(missing), "; ".join("sudo ifconfig lo0 alias %s up" % a for a in missing)))
+			return False
+	# A new port offset each run, so a connection a previous run left in TIME_WAIT doesn't get in the way.
+	# Below 1700 keeps the tests' ports under Linux's ephemeral range (32768 and up).
+	if not any(a.startswith("--port-offset") for a in extra):
+		extra = extra + ["--port-offset=%d" % random.randrange(1, 1700)]
 	memstick = os.path.join(os.path.dirname(headless), "memstick")
 	# A test that hasn't been recorded on PSPs yet has no expected files; then just show what each instance prints.
 	compare = all(os.path.exists(os.path.join(TESTS_DIR, "%s_%s.expected" % (test, role))) for role in roles)
@@ -84,13 +111,13 @@ def run_test(headless, test, extra, relay_ini):
 	for role, proc in procs:
 		out, _ = proc.communicate(timeout=TIMEOUT + 30)
 		if not compare:
-			print("OUTPUT %s_%s (no expected file):" % (test, role))
+			print("OUTPUT %s_%s [%s] (no expected file):" % (test, role, mode))
 			sys.stdout.write(out.decode("utf-8", "replace"))
 			passed = False
 			continue
 		ok = proc.returncode == 0
 		passed = passed and ok
-		print("%s %s_%s" % ("PASS" if ok else "FAIL", test, role))
+		print("%s %s_%s [%s]" % ("PASS" if ok else "FAIL", test, role, mode))
 		if not ok:
 			sys.stdout.write(out.decode("utf-8", "replace"))
 	return passed

docs/pspautotests.md

@@ -87,19 +87,21 @@ needs this run done by hand.
 
 The tests in `tests/net/adhoc/pair` (one host program and one client program each, see
 [pspautotests-hardware.md](pspautotests-hardware.md#tests-that-need-two-psps)) need several emulators
-running at once, so `test.py` doesn't run them. `Tools/adhoc_multi.py` does: it starts one
-PPSSPPHeadless per role with `--realtime` (PSP speed, since the sides wait for each other over the real
-network) and `--instance=N` (its own 127.0.0.N address and MAC), the host with `--adhoc-server`, and
-compares each against its own `.expected`:
+running at once. `test.py` hands them to `Tools/adhoc_multi.py`, which starts one PPSSPPHeadless per
+role with `--realtime` and `--instance=N` (its own 127.0.0.N address and MAC), the host with
+`--adhoc-server`, and compares each against its own `.expected`. Each test runs twice, once in each
+data mode: through the built-in server's relay (aemu_postoffice), and P2P, where the instances send to
+each other directly. A test that fails in only one mode goes in `pair_known_failures` for that mode.
+`--adhoc-mode=relay` or `--adhoc-mode=p2p` runs just one:
 
 ```bash
-python3 Tools/adhoc_multi.py net/adhoc/pair/basic
+python3 test.py -g -m net/adhoc/pair --adhoc-mode=p2p
+python3 Tools/adhoc_multi.py [--relay] net/adhoc/pair/basic
 ```
 
-Instance N binds to 127.0.0.N, which macOS doesn't have by default:
-`sudo ifconfig lo0 alias 127.0.0.2 up` (and .3, ... for more clients). With `--relay`, the instances
-use an aemu_postoffice server running locally (build it from `ext/aemu_postoffice/server_cpp`) in relay
-mode instead, which needs no aliases.
+In P2P mode instance N binds to 127.0.0.N. Linux and Windows route all of 127/8 to the loopback, but
+macOS needs the addresses added, after every reboot: `sudo ifconfig lo0 alias 127.0.0.2 up` (and .3, ...
+for tests with more clients). Without them the P2P run fails and prints that command.
 
 ### What you'll see
 

test.py

@@ -9,6 +9,7 @@ import random
 import threading
 import glob
 import platform
+import socket
 
 
 PPSSPP_EXECUTABLES = [
@@ -587,6 +588,13 @@ tests_good = [
   "video/psmfplayer/setpsmf",
   "video/psmfplayer/settempbuf",
   "video/psmfplayer/stop",
+  # Two-console tests: run on two instances, through the built-in relay and in P2P (see run_pair_tests).
+  "net/adhoc/pair/matching",
+  "net/adhoc/pair/matching2",
+  "net/adhoc/pair/matching3",
+  "net/adhoc/pair/matchingorder",
+  "net/adhoc/pair/ptppoll",
+  "net/adhoc/pair/vanish",
 ]
 
 # Broken tests
@@ -611,6 +619,22 @@ known_failures = {
   "armhf": [],
 }
 
+# Two-console tests that don't pass yet in one of the two ways run_pair_tests runs them. Same rules as above.
+pair_known_failures = {
+  "relay": [
+    # A PSP connects a PTP socket at PtpOpen, so a poll before PtpConnect already reports CONNECT.
+    # The relay doesn't connect until PtpConnect.
+    "net/adhoc/pair/ptppoll",
+  ],
+  "p2p": [
+    # A PTP connection to a peer that has vanished never fails: the host's TCP connection doesn't
+    # notice, and nothing else is watching.
+    "net/adhoc/pair/vanish",
+  ],
+}
+
+ADHOC_MODES = ["relay", "p2p"]
+
 tests_next = [
 # These are the next tests up for fixing. These run by default.
   "cpu/fpu/fcr",
@@ -690,16 +714,11 @@ tests_next = [
   "net/inet/tcp",
   "net/inet/udp",
   "net/primary/ether",
-  # Two-console tests: run on two instances through the built-in relay (see run_pair_tests).
+  # Two-console tests: run on two instances, through the built-in relay and in P2P (see run_pair_tests).
   "net/adhoc/pair/basic",
   "net/adhoc/pair/flow",
   "net/adhoc/pair/gamemode",
   "net/adhoc/pair/lifecycle",
-  "net/adhoc/pair/matching",
-  "net/adhoc/pair/matching2",
-  "net/adhoc/pair/matching3",
-  "net/adhoc/pair/matchingorder",
-  "net/adhoc/pair/vanish",
   "sysmem/partition",
   # These two mbx tests only appeared to work because they papered over bugs 
 
@@ -845,15 +864,56 @@ def run_tests(test_list, args):
 
   return returncode
 
-def run_pair_tests(test_list, args):
+def pair_instances(test):
+  # The host, the client, and client2, client3... if there are more.
+  n = 2
+  while os.path.exists(TEST_ROOT + test + "_client%d.prx" % n):
+    n += 1
+  return n
+
+def missing_loopback_addresses(count):
+  # In P2P mode instance N binds to 127.0.0.N. Linux and Windows route all of 127/8 to the loopback;
+  # macOS only has the addresses that were added.
+  missing = []
+  for n in range(2, count + 1):
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+      s.bind(("127.0.0.%d" % n, 0))
+    except OSError:
+      missing.append("127.0.0.%d" % n)
+    finally:
+      s.close()
+  return missing
+
+# P2P tests that couldn't run for want of a loopback address, reported as a failure at the end.
+pair_tests_without_loopback = []
+
+def run_pair_tests(test_list, args, modes):
   # Each instance needs its own process, so Tools/adhoc_multi.py starts them, the host running the
-  # built-in ad hoc server and relay.
-  cmdline = [sys.executable, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Tools', 'adhoc_multi.py'), '--headless', PPSSPP_EXE, '--relay']
-  cmdline.extend([i for i in args if i not in ['-g', '-m', '-b', '--teamcity']])
-  cmdline.append("--speed=" + os.environ.get("ADHOC_SPEED", "800"))
-  cmdline.extend(test_list)
-  returncode = subprocess.call(cmdline)
-  print("Ran " + ' '.join(cmdline))
+  # built-in ad hoc server. Each test runs through the server's relay (aemu_postoffice) and in P2P,
+  # where the instances talk to each other directly from 127.0.0.1, 127.0.0.2 and so on.
+  returncode = 0
+  for mode in modes:
+    tests = [t for t in test_list if t not in pair_known_failures[mode]]
+    skipped = [t for t in test_list if t in pair_known_failures[mode]]
+    if skipped:
+      print("Skipping %d known failures in %s mode: %s" % (len(skipped), mode, ", ".join(skipped)), flush=True)
+    if mode == "p2p":
+      runnable = [t for t in tests if not missing_loopback_addresses(pair_instances(t))]
+      pair_tests_without_loopback.extend(t for t in tests if t not in runnable)
+      if len(runnable) < len(tests):
+        print("WARNING: skipping %d P2P tests, which need loopback addresses this machine doesn't have." % (len(tests) - len(runnable)), flush=True)
+      tests = runnable
+    if not tests:
+      continue
+    cmdline = [sys.executable, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Tools', 'adhoc_multi.py'), '--headless', PPSSPP_EXE]
+    if mode == "relay":
+      cmdline.append('--relay')
+    cmdline.extend([i for i in args if i not in ['-g', '-m', '-b', '--teamcity']])
+    cmdline.append("--speed=" + os.environ.get("ADHOC_SPEED", "800"))
+    cmdline.extend(tests)
+    returncode = subprocess.call(cmdline) or returncode
+    print("Ran " + ' '.join(cmdline))
   return returncode
 
 def main():
@@ -862,6 +922,7 @@ def main():
   args = []
   teamcity = False
   skip_arch = None
+  adhoc_modes = ADHOC_MODES
   for arg in sys.argv[1:]:
     if arg == '--teamcity':
       args.append(arg)
@@ -872,6 +933,12 @@ def main():
       if skip_arch not in known_failures:
         print("Unknown architecture for --known-failures: " + skip_arch)
         sys.exit(1)
+    elif arg.startswith('--adhoc-mode='):
+      # Ours: which of relay and p2p the two-console tests run in (both by default).
+      adhoc_modes = [arg[len('--adhoc-mode='):]]
+      if adhoc_modes[0] not in ADHOC_MODES:
+        print("Unknown --adhoc-mode: " + adhoc_modes[0])
+        sys.exit(1)
     elif arg[0] == '-':
       args.append(arg)
     else:
@@ -905,7 +972,12 @@ def main():
     if flags_for(test) is not None:
       returncode = run_tests([test], args + flags_for(test)) or returncode
   if pair:
-    returncode = run_pair_tests(pair, args) or returncode
+    returncode = run_pair_tests(pair, args, adhoc_modes) or returncode
+  if pair_tests_without_loopback:
+    missing = missing_loopback_addresses(max(pair_instances(t) for t in pair_tests_without_loopback))
+    print("FAILED: %d P2P tests didn't run, for want of %s: %s" % (len(pair_tests_without_loopback), ", ".join(missing), ", ".join(pair_tests_without_loopback)))
+    print("Add the addresses with: " + "; ".join("sudo ifconfig lo0 alias %s up" % a for a in missing))
+    returncode = 1
   if teamcity:
     return 0
   return returncode