From 074b6bf5478a2ece9017442a1a2ef7b4c2aaa139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 30 Oct 2026 20:43:39 +0100 Subject: [PATCH] test.py: Run the two-console ad hoc tests in relay and P2P mode, and in CI The pair tests that pass go in tests_good, and each runs twice: through the built-in server's relay and P2P. pair_known_failures lists the ones that fail in only one mode. A P2P test that needs a loopback address the machine lacks (macOS has only 127.0.0.1) is skipped with a warning, and the run fails at the end with the command that adds it. The macOS CI runners add 127.0.0.2. Each run gets a random port offset. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 6 +++ Tools/adhoc_multi.py | 31 ++++++++++- docs/pspautotests.md | 20 +++---- pspautotests | 2 +- test.py | 102 ++++++++++++++++++++++++++++++------ 5 files changed, 134 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2d0fc55409..550abb403d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -550,6 +550,12 @@ jobs: timeout-minutes: 30 run: build/PPSSPPUnitTest ALL + # The two-console ad hoc tests run in P2P too, where instance N binds to 127.0.0.N. Linux and + # Windows route all of 127/8 to the loopback; macOS needs the address added. + - name: Add a loopback address for the P2P ad hoc tests + if: matrix.extra == 'test' && runner.os == 'macOS' + run: sudo ifconfig lo0 alias 127.0.0.2 up + - name: Execute headless tests if: matrix.extra == 'test' && runner.os != 'Linux' timeout-minutes: 30 diff --git a/Tools/adhoc_multi.py b/Tools/adhoc_multi.py index 9644757ffc..cdbf8618a4 100644 --- a/Tools/adhoc_multi.py +++ b/Tools/adhoc_multi.py @@ -17,6 +17,8 @@ import argparse import os +import random +import socket import subprocess import sys import tempfile @@ -58,8 +60,33 @@ AdhocServerRelayMode = 1 """ +def missing_loopback_aliases(count): + # In P2P mode instance N binds to 127.0.0.N. Linux and Windows route all of 127/8 to the loopback; + # macOS only has the addresses that were added. + missing = [] + for n in range(2, count + 1): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.bind(("127.0.0.%d" % n, 0)) + except OSError: + missing.append("127.0.0.%d" % n) + finally: + s.close() + return missing + + def run_test(headless, test, extra, relay_ini): roles = roles_of(test) + mode = "relay" if relay_ini else "p2p" + if not relay_ini: + missing = missing_loopback_aliases(len(roles)) + if missing: + print("FAIL %s [p2p]: %s missing. Add with: %s" % (test, ", ".join(missing), "; ".join("sudo ifconfig lo0 alias %s up" % a for a in missing))) + return False + # A new port offset each run, so a connection a previous run left in TIME_WAIT doesn't get in the way. + # Below 1700 keeps the tests' ports under Linux's ephemeral range (32768 and up). + if not any(a.startswith("--port-offset") for a in extra): + extra = extra + ["--port-offset=%d" % random.randrange(1, 1700)] memstick = os.path.join(os.path.dirname(headless), "memstick") # A test that hasn't been recorded on PSPs yet has no expected files; then just show what each instance prints. compare = all(os.path.exists(os.path.join(TESTS_DIR, "%s_%s.expected" % (test, role))) for role in roles) @@ -84,13 +111,13 @@ def run_test(headless, test, extra, relay_ini): for role, proc in procs: out, _ = proc.communicate(timeout=TIMEOUT + 30) if not compare: - print("OUTPUT %s_%s (no expected file):" % (test, role)) + print("OUTPUT %s_%s [%s] (no expected file):" % (test, role, mode)) sys.stdout.write(out.decode("utf-8", "replace")) passed = False continue ok = proc.returncode == 0 passed = passed and ok - print("%s %s_%s" % ("PASS" if ok else "FAIL", test, role)) + print("%s %s_%s [%s]" % ("PASS" if ok else "FAIL", test, role, mode)) if not ok: sys.stdout.write(out.decode("utf-8", "replace")) return passed diff --git a/docs/pspautotests.md b/docs/pspautotests.md index 02bc71681e..a026739137 100644 --- a/docs/pspautotests.md +++ b/docs/pspautotests.md @@ -87,19 +87,21 @@ needs this run done by hand. The tests in `tests/net/adhoc/pair` (one host program and one client program each, see [pspautotests-hardware.md](pspautotests-hardware.md#tests-that-need-two-psps)) need several emulators -running at once, so `test.py` doesn't run them. `Tools/adhoc_multi.py` does: it starts one -PPSSPPHeadless per role with `--realtime` (PSP speed, since the sides wait for each other over the real -network) and `--instance=N` (its own 127.0.0.N address and MAC), the host with `--adhoc-server`, and -compares each against its own `.expected`: +running at once. `test.py` hands them to `Tools/adhoc_multi.py`, which starts one PPSSPPHeadless per +role with `--realtime` and `--instance=N` (its own 127.0.0.N address and MAC), the host with +`--adhoc-server`, and compares each against its own `.expected`. Each test runs twice, once in each +data mode: through the built-in server's relay (aemu_postoffice), and P2P, where the instances send to +each other directly. A test that fails in only one mode goes in `pair_known_failures` for that mode. +`--adhoc-mode=relay` or `--adhoc-mode=p2p` runs just one: ```bash -python3 Tools/adhoc_multi.py net/adhoc/pair/basic +python3 test.py -g -m net/adhoc/pair --adhoc-mode=p2p +python3 Tools/adhoc_multi.py [--relay] net/adhoc/pair/basic ``` -Instance N binds to 127.0.0.N, which macOS doesn't have by default: -`sudo ifconfig lo0 alias 127.0.0.2 up` (and .3, ... for more clients). With `--relay`, the instances -use an aemu_postoffice server running locally (build it from `ext/aemu_postoffice/server_cpp`) in relay -mode instead, which needs no aliases. +In P2P mode instance N binds to 127.0.0.N. Linux and Windows route all of 127/8 to the loopback, but +macOS needs the addresses added, after every reboot: `sudo ifconfig lo0 alias 127.0.0.2 up` (and .3, ... +for tests with more clients). Without them the P2P run fails and prints that command. ### What you'll see diff --git a/pspautotests b/pspautotests index 38cb482639..c6e36d7e03 160000 --- a/pspautotests +++ b/pspautotests @@ -1 +1 @@ -Subproject commit 38cb482639085540b5d70dc79a0aea4054c01861 +Subproject commit c6e36d7e0379cb59df35183d405683f180679257 diff --git a/test.py b/test.py index 649fc9b567..bf0bc8b14f 100755 --- a/test.py +++ b/test.py @@ -9,6 +9,7 @@ import random import threading import glob import platform +import socket PPSSPP_EXECUTABLES = [ @@ -587,6 +588,13 @@ tests_good = [ "video/psmfplayer/setpsmf", "video/psmfplayer/settempbuf", "video/psmfplayer/stop", + # Two-console tests: run on two instances, through the built-in relay and in P2P (see run_pair_tests). + "net/adhoc/pair/matching", + "net/adhoc/pair/matching2", + "net/adhoc/pair/matching3", + "net/adhoc/pair/matchingorder", + "net/adhoc/pair/ptppoll", + "net/adhoc/pair/vanish", ] # Broken tests @@ -611,6 +619,22 @@ known_failures = { "armhf": [], } +# Two-console tests that don't pass yet in one of the two ways run_pair_tests runs them. Same rules as above. +pair_known_failures = { + "relay": [ + # A PSP connects a PTP socket at PtpOpen, so a poll before PtpConnect already reports CONNECT. + # The relay doesn't connect until PtpConnect. + "net/adhoc/pair/ptppoll", + ], + "p2p": [ + # A PTP connection to a peer that has vanished never fails: the host's TCP connection doesn't + # notice, and nothing else is watching. + "net/adhoc/pair/vanish", + ], +} + +ADHOC_MODES = ["relay", "p2p"] + tests_next = [ # These are the next tests up for fixing. These run by default. "cpu/fpu/fcr", @@ -690,16 +714,11 @@ tests_next = [ "net/inet/tcp", "net/inet/udp", "net/primary/ether", - # Two-console tests: run on two instances through the built-in relay (see run_pair_tests). + # Two-console tests: run on two instances, through the built-in relay and in P2P (see run_pair_tests). "net/adhoc/pair/basic", "net/adhoc/pair/flow", "net/adhoc/pair/gamemode", "net/adhoc/pair/lifecycle", - "net/adhoc/pair/matching", - "net/adhoc/pair/matching2", - "net/adhoc/pair/matching3", - "net/adhoc/pair/matchingorder", - "net/adhoc/pair/vanish", "sysmem/partition", # These two mbx tests only appeared to work because they papered over bugs @@ -845,15 +864,56 @@ def run_tests(test_list, args): return returncode -def run_pair_tests(test_list, args): +def pair_instances(test): + # The host, the client, and client2, client3... if there are more. + n = 2 + while os.path.exists(TEST_ROOT + test + "_client%d.prx" % n): + n += 1 + return n + +def missing_loopback_addresses(count): + # In P2P mode instance N binds to 127.0.0.N. Linux and Windows route all of 127/8 to the loopback; + # macOS only has the addresses that were added. + missing = [] + for n in range(2, count + 1): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.bind(("127.0.0.%d" % n, 0)) + except OSError: + missing.append("127.0.0.%d" % n) + finally: + s.close() + return missing + +# P2P tests that couldn't run for want of a loopback address, reported as a failure at the end. +pair_tests_without_loopback = [] + +def run_pair_tests(test_list, args, modes): # Each instance needs its own process, so Tools/adhoc_multi.py starts them, the host running the - # built-in ad hoc server and relay. - cmdline = [sys.executable, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Tools', 'adhoc_multi.py'), '--headless', PPSSPP_EXE, '--relay'] - cmdline.extend([i for i in args if i not in ['-g', '-m', '-b', '--teamcity']]) - cmdline.append("--speed=" + os.environ.get("ADHOC_SPEED", "800")) - cmdline.extend(test_list) - returncode = subprocess.call(cmdline) - print("Ran " + ' '.join(cmdline)) + # built-in ad hoc server. Each test runs through the server's relay (aemu_postoffice) and in P2P, + # where the instances talk to each other directly from 127.0.0.1, 127.0.0.2 and so on. + returncode = 0 + for mode in modes: + tests = [t for t in test_list if t not in pair_known_failures[mode]] + skipped = [t for t in test_list if t in pair_known_failures[mode]] + if skipped: + print("Skipping %d known failures in %s mode: %s" % (len(skipped), mode, ", ".join(skipped)), flush=True) + if mode == "p2p": + runnable = [t for t in tests if not missing_loopback_addresses(pair_instances(t))] + pair_tests_without_loopback.extend(t for t in tests if t not in runnable) + if len(runnable) < len(tests): + print("WARNING: skipping %d P2P tests, which need loopback addresses this machine doesn't have." % (len(tests) - len(runnable)), flush=True) + tests = runnable + if not tests: + continue + cmdline = [sys.executable, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Tools', 'adhoc_multi.py'), '--headless', PPSSPP_EXE] + if mode == "relay": + cmdline.append('--relay') + cmdline.extend([i for i in args if i not in ['-g', '-m', '-b', '--teamcity']]) + cmdline.append("--speed=" + os.environ.get("ADHOC_SPEED", "800")) + cmdline.extend(tests) + returncode = subprocess.call(cmdline) or returncode + print("Ran " + ' '.join(cmdline)) return returncode def main(): @@ -862,6 +922,7 @@ def main(): args = [] teamcity = False skip_arch = None + adhoc_modes = ADHOC_MODES for arg in sys.argv[1:]: if arg == '--teamcity': args.append(arg) @@ -872,6 +933,12 @@ def main(): if skip_arch not in known_failures: print("Unknown architecture for --known-failures: " + skip_arch) sys.exit(1) + elif arg.startswith('--adhoc-mode='): + # Ours: which of relay and p2p the two-console tests run in (both by default). + adhoc_modes = [arg[len('--adhoc-mode='):]] + if adhoc_modes[0] not in ADHOC_MODES: + print("Unknown --adhoc-mode: " + adhoc_modes[0]) + sys.exit(1) elif arg[0] == '-': args.append(arg) else: @@ -905,7 +972,12 @@ def main(): if flags_for(test) is not None: returncode = run_tests([test], args + flags_for(test)) or returncode if pair: - returncode = run_pair_tests(pair, args) or returncode + returncode = run_pair_tests(pair, args, adhoc_modes) or returncode + if pair_tests_without_loopback: + missing = missing_loopback_addresses(max(pair_instances(t) for t in pair_tests_without_loopback)) + print("FAILED: %d P2P tests didn't run, for want of %s: %s" % (len(pair_tests_without_loopback), ", ".join(missing), ", ".join(pair_tests_without_loopback))) + print("Add the addresses with: " + "; ".join("sudo ifconfig lo0 alias %s up" % a for a in missing)) + returncode = 1 if teamcity: return 0 return returncode