diff --git a/Common/System/Display.cpp b/Common/System/Display.cpp index a724cfb653..2c71aa6d17 100644 --- a/Common/System/Display.cpp +++ b/Common/System/Display.cpp @@ -120,7 +120,7 @@ void DisplayProperties::Print() { } // Y-flip compensation is done elsewhere now. -Lin::Matrix4x4 ComputeOrthoMatrix(float xres, float yres, CoordConvention coordConvention) { +Lin::Matrix4x4 ComputeOrthoMatrix(float xres, float yres, CoordConvention coordConvention, bool rotateToDisplay) { using namespace Lin; Matrix4x4 ortho; switch (coordConvention) { @@ -134,7 +134,7 @@ Lin::Matrix4x4 ComputeOrthoMatrix(float xres, float yres, CoordConvention coordC break; } // Compensate for rotated display if needed. - if (g_display.rotation != DisplayRotation::ROTATE_0) { + if (rotateToDisplay && g_display.rotation != DisplayRotation::ROTATE_0) { ortho = ortho * g_display.rot_matrix; } return ortho; diff --git a/Common/System/Display.h b/Common/System/Display.h index edf15c6320..4cd0e61609 100644 --- a/Common/System/Display.h +++ b/Common/System/Display.h @@ -61,7 +61,8 @@ struct DisplayRect { void RotateRectToDisplay(DisplayRect &rect, float rtWidth, float rtHeight); void RotateRectToDisplay(DisplayRect &rect, int rtWidth, int rtHeight); -Lin::Matrix4x4 ComputeOrthoMatrix(float xres, float yres, CoordConvention coordConvention); +// rotateToDisplay applies the primary display's rotation (Android's Vulkan pre-rotation). Not for other outputs. +Lin::Matrix4x4 ComputeOrthoMatrix(float xres, float yres, CoordConvention coordConvention, bool rotateToDisplay = true); // Take this and run through translation. Returns "Portrait", "Landscape" or later "Square" (the latter being the square shape you get when you open a foldable phone). std::string_view DeviceOrientationToString(DeviceOrientation orientation); diff --git a/Common/System/NativeApp.h b/Common/System/NativeApp.h index 6f733bdb14..7b664b054a 100644 --- a/Common/System/NativeApp.h +++ b/Common/System/NativeApp.h @@ -124,6 +124,9 @@ enum class OutputRole { int Native_AddOutput(OutputRole role, const Draw::SwapchainDesc &desc); void Native_ResizeOutput(int id, int width, int height); void Native_RemoveOutput(int id); +// Also waits until the output's swapchain is gone, so its surface can go away right after (as on Android, +// where a surface mustn't be touched once surfaceDestroyed returns). Don't call it from the render thread. +bool Native_RemoveOutputAndWait(int id); AudioBackend *System_CreateAudioBackend(); diff --git a/Common/UI/ScreenManager.cpp b/Common/UI/ScreenManager.cpp index 385d9490d8..f16960f32b 100644 --- a/Common/UI/ScreenManager.cpp +++ b/Common/UI/ScreenManager.cpp @@ -300,7 +300,7 @@ ScreenRenderFlags ScreenManager::Render(std::function afterPreRender) { const float dpHeight = h * g_display.dpi_scale_y; const Bounds primaryBounds = ui->GetBounds(); ui->SetBounds(Bounds(0, 0, dpWidth, dpHeight)); - ui->Draw()->PushDrawMatrix(ComputeOrthoMatrix(dpWidth, dpHeight, draw_->GetDeviceCaps().coordConvention)); + ui->Draw()->PushDrawMatrix(ComputeOrthoMatrix(dpWidth, dpHeight, draw_->GetDeviceCaps().coordConvention, false)); bool drawn = false; for (auto iter = stack_.rbegin(); iter != stack_.rend(); ++iter) { if (iter->screen->renderOutput(output.display)) { diff --git a/UI/NativeApp.cpp b/UI/NativeApp.cpp index 15d7a0662f..6a1ce6b215 100644 --- a/UI/NativeApp.cpp +++ b/UI/NativeApp.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -204,14 +205,22 @@ struct OutputRequest { int height; }; static std::mutex g_outputRequestLock; +static std::condition_variable g_outputRequestsDoneCond; static std::vector g_outputRequests; +// Counts requests queued and handled, for Native_RemoveOutputAndWait. Under g_outputRequestLock. +static uint64_t g_outputRequestsQueued = 0; +static uint64_t g_outputRequestsDone = 0; +// Whether the outputs have swapchains, between NativeInitGraphics and NativeShutdownGraphics. +static bool g_outputsLive = false; static std::atomic g_nextOutputId{1}; static void ProcessOutputRequests() { std::vector requests; + uint64_t queued; { std::lock_guard guard(g_outputRequestLock); requests.swap(g_outputRequests); + queued = g_outputRequestsQueued; } for (const OutputRequest &request : requests) { switch (request.type) { @@ -220,6 +229,15 @@ static void ProcessOutputRequests() { case OutputRequest::Type::REMOVE: g_screenManager->RemoveOutput(request.id); break; } } + std::lock_guard guard(g_outputRequestLock); + g_outputRequestsDone = queued; + g_outputRequestsDoneCond.notify_all(); +} + +static void SetOutputsLive(bool live) { + std::lock_guard guard(g_outputRequestLock); + g_outputsLive = live; + g_outputRequestsDoneCond.notify_all(); } static int renderCounter = 0; @@ -975,6 +993,7 @@ bool NativeInitGraphics(GraphicsContext *graphicsContext) { // An output removed while there was no device can't be recreated, its surface may be gone. ProcessOutputRequests(); g_screenManager->deviceRestored(g_draw); + SetOutputsLive(true); g_audioBackend = System_CreateAudioBackend(); if (g_audioBackend) { @@ -1092,6 +1111,7 @@ void NativeShutdownGraphics(GraphicsContext *graphicsContext) { if (g_screenManager) { g_screenManager->deviceLost(); } + SetOutputsLive(false); g_iconCache.ClearTextures(); if (gpu) { @@ -2044,6 +2064,7 @@ int Native_AddOutput(OutputRole role, const Draw::SwapchainDesc &desc) { request.desc = desc; std::lock_guard guard(g_outputRequestLock); g_outputRequests.push_back(request); + g_outputRequestsQueued++; return request.id; } @@ -2055,6 +2076,7 @@ void Native_ResizeOutput(int id, int width, int height) { request.height = height; std::lock_guard guard(g_outputRequestLock); g_outputRequests.push_back(request); + g_outputRequestsQueued++; } void Native_RemoveOutput(int id) { @@ -2063,6 +2085,21 @@ void Native_RemoveOutput(int id) { request.id = id; std::lock_guard guard(g_outputRequestLock); g_outputRequests.push_back(request); + g_outputRequestsQueued++; +} + +bool Native_RemoveOutputAndWait(int id) { + Native_RemoveOutput(id); + std::unique_lock lock(g_outputRequestLock); + const uint64_t queued = g_outputRequestsQueued; + // Without a device, the outputs have no swapchains to let go of. + const bool done = g_outputRequestsDoneCond.wait_for(lock, std::chrono::seconds(3), [=] { + return !g_outputsLive || g_outputRequestsDone >= queued; + }); + if (!done) { + ERROR_LOG(Log::G3D, "Timed out waiting for output %d to be removed", id); + } + return done; } bool Native_UpdateScreenScale(int pixel_width, int pixel_height, float customScale) { diff --git a/android/jni/app-android.cpp b/android/jni/app-android.cpp index 1324aebeed..2ef41d44be 100644 --- a/android/jni/app-android.cpp +++ b/android/jni/app-android.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -69,6 +70,7 @@ struct JNIEnv {}; #include "Common/GPU/OpenGL/GLFeatures.h" #include "Common/System/Display.h" +#include "Common/GPU/thin3d.h" #include "Common/System/NativeApp.h" #include "Common/System/System.h" #include "Common/System/OSD.h" @@ -126,6 +128,9 @@ static std::vector g_frameCommands; static std::string systemName; static std::string langRegion; static std::string boardName; +// The name the user gave the device (like "Galaxy S26"). Set from the UI thread. +static std::mutex g_deviceNameLock; +static std::string g_deviceName; std::string g_externalDir; // Original external dir (root of Android storage). std::string g_extFilesDir; // App private external dir. @@ -329,6 +334,11 @@ std::string System_GetProperty(SystemProperty prop) { return langRegion; case SYSPROP_BOARDNAME: return boardName; + case SYSPROP_DEVICE_NAME: + { + std::lock_guard guard(g_deviceNameLock); + return g_deviceName; + } case SYSPROP_BUILD_VERSION: return PPSSPP_GIT_VERSION; case SYSPROP_INSTALLER_NAME: @@ -582,6 +592,9 @@ static std::string QueryConfig(std::string_view query) { return g_Config.bSustainedPerformanceMode ? "1" : "0"; } else if (query == "audioMixWithOthers") { return g_Config.bAudioMixWithOthers ? "1" : "0"; + } else if (query == "externalDisplayOutput") { + // Whether the game can go to a second display, see ExternalDisplayPresentation.java. + return g_Config.iGPUBackend == (int)GPUBackend::VULKAN ? "1" : "0"; } else { return ""; } @@ -908,6 +921,58 @@ static void ApplyPendingResize() { } } +extern "C" void JNICALL Java_org_ppsspp_ppsspp_NativeApp_setDeviceName(JNIEnv *env, jclass, jstring jname) { + std::string name = jname ? GetJavaString(env, jname) : ""; + std::lock_guard guard(g_deviceNameLock); + g_deviceName = name; +} + +// Secondary outputs (a TV on HDMI, for one). The surface's window is kept until the output is removed. +static std::mutex g_outputWindowsLock; +static std::map g_outputWindows; + +extern "C" jint JNICALL Java_org_ppsspp_ppsspp_NativeApp_addOutput(JNIEnv *env, jclass, jobject surface, jint width, jint height) { + ANativeWindow *wnd = surface ? ANativeWindow_fromSurface(env, surface) : nullptr; + if (!wnd) { + ERROR_LOG(Log::G3D, "addOutput: no window for the surface"); + return 0; + } + Draw::SwapchainDesc desc{}; + desc.winsys = WINDOWSYSTEM_ANDROID; + desc.data1 = wnd; + desc.width = width; + desc.height = height; + desc.tag = "ExternalDisplay"; + const int id = Native_AddOutput(OutputRole::Game, desc); + INFO_LOG(Log::G3D, "addOutput: %dx%d, output %d", width, height, id); + std::lock_guard guard(g_outputWindowsLock); + g_outputWindows[id] = wnd; + return id; +} + +extern "C" void JNICALL Java_org_ppsspp_ppsspp_NativeApp_resizeOutput(JNIEnv *, jclass, jint id, jint width, jint height) { + INFO_LOG(Log::G3D, "resizeOutput: output %d, %dx%d", id, width, height); + Native_ResizeOutput(id, width, height); +} + +// Returns once the render thread is done with the surface, as surfaceDestroyed requires. +extern "C" void JNICALL Java_org_ppsspp_ppsspp_NativeApp_removeOutput(JNIEnv *, jclass, jint id) { + INFO_LOG(Log::G3D, "removeOutput: output %d", id); + Native_RemoveOutputAndWait(id); + ANativeWindow *wnd = nullptr; + { + std::lock_guard guard(g_outputWindowsLock); + auto iter = g_outputWindows.find(id); + if (iter != g_outputWindows.end()) { + wnd = iter->second; + g_outputWindows.erase(iter); + } + } + if (wnd) { + ANativeWindow_release(wnd); + } +} + extern "C" void JNICALL Java_org_ppsspp_ppsspp_NativeApp_backbufferResize(JNIEnv *, jclass, jint pixel_xres, jint pixel_yres, jint rotation) { INFO_LOG(Log::System, "NativeApp.backbufferResize(%d x %d, rotation %d)", pixel_xres, pixel_yres, rotation); diff --git a/android/src/org/ppsspp/ppsspp/ExternalDisplayPresentation.java b/android/src/org/ppsspp/ppsspp/ExternalDisplayPresentation.java new file mode 100644 index 0000000000..e2e42390b3 --- /dev/null +++ b/android/src/org/ppsspp/ppsspp/ExternalDisplayPresentation.java @@ -0,0 +1,53 @@ +package org.ppsspp.ppsspp; + +import android.app.Presentation; +import android.content.Context; +import android.os.Bundle; +import android.util.Log; +import android.view.Display; +import android.view.SurfaceHolder; +import android.view.SurfaceView; + +// Shows the game on a second display (a TV on HDMI, or a wireless display that apps can draw to), as a +// Game output. The phone keeps the touch controls and the menus. See docs/multi-swapchain.md. +public class ExternalDisplayPresentation extends Presentation implements SurfaceHolder.Callback { + private static final String TAG = "PPSSPPExternalDisplay"; + + private int outputId = 0; + + public ExternalDisplayPresentation(Context outerContext, Display display) { + super(outerContext, display); + } + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + SurfaceView view = new SurfaceView(getContext()); + view.getHolder().addCallback(this); + setContentView(view); + } + + @Override + public void surfaceCreated(SurfaceHolder holder) { + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + if (outputId != 0) { + NativeApp.resizeOutput(outputId, width, height); + return; + } + outputId = NativeApp.addOutput(holder.getSurface(), width, height); + Log.i(TAG, "Showing the game on display " + getDisplay().getDisplayId() + " (" + getDisplay().getName() + "), " + width + "x" + height + ", output " + outputId); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + if (outputId == 0) { + return; + } + Log.i(TAG, "Surface destroyed, removing output " + outputId); + NativeApp.removeOutput(outputId); + outputId = 0; + } +} diff --git a/android/src/org/ppsspp/ppsspp/NativeApp.java b/android/src/org/ppsspp/ppsspp/NativeApp.java index 87d58c1ded..81f77856b0 100644 --- a/android/src/org/ppsspp/ppsspp/NativeApp.java +++ b/android/src/org/ppsspp/ppsspp/NativeApp.java @@ -5,6 +5,7 @@ import android.util.Log; import android.view.InputDevice; import android.view.InputEvent; import android.view.MotionEvent; +import android.view.Surface; import androidx.annotation.RequiresApi; @@ -45,6 +46,13 @@ public class NativeApp { // but the swapchain still has to be recreated, for the pre-rotation. public static native void backbufferResize(int bufferWidth, int bufferHeight, int rotation); + // A second display, like a TV, that the game is shown on. addOutput returns the output's id, or 0. + // removeOutput returns once the native side is done with the surface. + public static native int addOutput(Surface surface, int width, int height); + public static native void resizeOutput(int id, int width, int height); + public static native void removeOutput(int id); + public static native void setDeviceName(String name); + public static native boolean isAtTopLevel(); // These have Android semantics: Resume is always called on bootup, after init diff --git a/android/src/org/ppsspp/ppsspp/PpssppActivity.java b/android/src/org/ppsspp/ppsspp/PpssppActivity.java index 972fa6a53a..7a92bd96d1 100644 --- a/android/src/org/ppsspp/ppsspp/PpssppActivity.java +++ b/android/src/org/ppsspp/ppsspp/PpssppActivity.java @@ -24,6 +24,7 @@ import android.hardware.Sensor; import android.hardware.SensorEvent; import android.hardware.SensorEventListener; import android.hardware.SensorManager; +import android.hardware.display.DisplayManager; import android.hardware.input.InputManager; import android.media.AudioManager; import android.net.Uri; @@ -32,8 +33,10 @@ import android.os.Bundle; import android.os.Environment; import android.os.PowerManager; import android.provider.MediaStore; +import android.provider.Settings; import android.text.InputType; import android.util.Log; +import android.view.Display; import android.view.Gravity; import android.view.HapticFeedbackConstants; import android.view.InputDevice; @@ -116,6 +119,24 @@ public class PpssppActivity extends AppCompatActivity implements SensorEventList private AudioManager audioManager; private InputManager.InputDeviceListener inputDeviceListener; + // The game on a second display, while there's one (and the backend can draw to it) and we're started. + private ExternalDisplayPresentation externalDisplayPresentation; + private final DisplayManager.DisplayListener displayListener = new DisplayManager.DisplayListener() { + @Override + public void onDisplayAdded(int displayId) { + updateExternalDisplay(); + } + + @Override + public void onDisplayRemoved(int displayId) { + updateExternalDisplay(); + } + + @Override + public void onDisplayChanged(int displayId) { + } + }; + // This is to avoid losing the game/menu state etc when we are just // switched-away from or rotated etc. private boolean shuttingDown; @@ -945,6 +966,40 @@ public class PpssppActivity extends AppCompatActivity implements SensorEventList updateSustainedPerformanceMode(); } + // Shows the game on a presentation display (like a TV on HDMI), if there's one and the backend can draw + // to a second surface (Vulkan). Otherwise Android mirrors the phone there. + private void updateExternalDisplay() { + Display display = null; + if ("1".equals(NativeApp.queryConfig("externalDisplayOutput"))) { + DisplayManager displayManager = (DisplayManager)getSystemService(Context.DISPLAY_SERVICE); + Display[] displays = displayManager.getDisplays(DisplayManager.DISPLAY_CATEGORY_PRESENTATION); + if (displays.length > 0) { + display = displays[0]; + } + } + if (externalDisplayPresentation != null && (display == null || externalDisplayPresentation.getDisplay().getDisplayId() != display.getDisplayId())) { + dismissExternalDisplay(); + } + if (externalDisplayPresentation != null || display == null) { + return; + } + Log.i(TAG, "Showing the game on display " + display.getDisplayId() + " (" + display.getName() + ")"); + ExternalDisplayPresentation presentation = new ExternalDisplayPresentation(this, display); + try { + presentation.show(); + externalDisplayPresentation = presentation; + } catch (WindowManager.InvalidDisplayException e) { + Log.w(TAG, "Couldn't show the game on display " + display.getDisplayId() + ": " + e); + } + } + + private void dismissExternalDisplay() { + if (externalDisplayPresentation != null) { + externalDisplayPresentation.dismiss(); + externalDisplayPresentation = null; + } + } + // The render loop thread (EmuThread) is now spawned from the native side. protected void startRenderLoopThread() { synchronized (PpssppActivity.class) { @@ -1080,6 +1135,16 @@ public class PpssppActivity extends AppCompatActivity implements SensorEventList InputManager inputManager = (InputManager)getSystemService(Context.INPUT_SERVICE); inputManager.registerInputDeviceListener(inputDeviceListener, null); + if (checkCurrent("showing the game on a second display")) { + // For the message on the TV while we're in the menus, "In menus on ". + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N_MR1) { + NativeApp.setDeviceName(Settings.Global.getString(getContentResolver(), Settings.Global.DEVICE_NAME)); + } + DisplayManager displayManager = (DisplayManager)getSystemService(Context.DISPLAY_SERVICE); + displayManager.registerDisplayListener(displayListener, null); + updateExternalDisplay(); + } + // In case the surface outlived the stop. Otherwise this waits for surfaceCreated. startRenderLoopThread(); Log.i(TAG, "onStart end"); @@ -1096,6 +1161,11 @@ public class PpssppActivity extends AppCompatActivity implements SensorEventList InputManager inputManager = (InputManager)getSystemService(Context.INPUT_SERVICE); inputManager.unregisterInputDeviceListener(inputDeviceListener); + // Before the render thread stops, so it can let go of the second display's surface. + DisplayManager displayManager = (DisplayManager)getSystemService(Context.DISPLAY_SERVICE); + displayManager.unregisterDisplayListener(displayListener); + dismissExternalDisplay(); + // Normally surfaceDestroyed has done this already, but not every device destroys the surface of a // stopped activity, and an invisible game shouldn't keep running. joinRenderLoopThread(); diff --git a/docs/multi-swapchain.md b/docs/multi-swapchain.md index ee23653578..e44fe37473 100644 --- a/docs/multi-swapchain.md +++ b/docs/multi-swapchain.md @@ -144,8 +144,11 @@ The main swapchain stays where it is, in `VulkanContext` and `FrameDataShared`. recreated at the start of the next frame, on the emu thread. Recreating and destroying stop the render threads and wait for the queue, like a resize of the main swapchain does. Creating doesn't need to, since nothing refers to the new one yet. -- **Not done:** reading back an extra swapchain, and rotation (pre-transform) of one, which Android - will need. `VulkanPresentation` (libretro, offscreen) still only replaces the main swapchain. +- **Rotation:** extra swapchains aren't pre-rotated. They ask for the identity transform, and their + viewports, scissors and UI projection (`ComputeOrthoMatrix(..., false)`) leave out the main + swapchain's rotation, which is Android's. +- **Not done:** reading back an extra swapchain. `VulkanPresentation` (libretro, offscreen) still only + replaces the main swapchain. With only the main swapchain, the frame does what it did before: frametests on Vulkan match. @@ -273,6 +276,23 @@ can be tested on its own. - **AirPlay latency:** this is the same video stream as mirroring, so the same lag. Input stays local, but the picture is late. +## Android + +- **The display:** Android shows a second display to apps as a presentation display: HDMI or USB-C + DisplayPort out (Samsung DeX has to be off, or the TV is a desktop), some wireless displays + (Miracast, like Samsung's Smart View), and Developer options → "Simulate secondary displays". + Casting to a Chromecast or an Android TV is mirroring only. +- **Showing it:** while it's started, the activity watches `DisplayManager` and, when there's a + `DISPLAY_CATEGORY_PRESENTATION` display and the backend is Vulkan (`queryConfig("externalDisplayOutput")`), + shows an `ExternalDisplayPresentation` there. That's a `Presentation` with a `SurfaceView`, whose + surface callbacks add, resize and remove a `Game` output through JNI. Otherwise Android mirrors. + A backend change recreates the activity, which checks again. +- **Removing:** the surface mustn't be touched once `surfaceDestroyed` returns, so it removes the output + with `Native_RemoveOutputAndWait()`. That returns once the render thread has processed the removal, + or right away if the graphics are shut down (the outputs have no swapchains then). The activity + dismisses the presentation in `onStop`, before it stops the render thread. +- **Device name:** "In menus on %1" uses `Settings.Global.DEVICE_NAME` (Android 7.1 and up). + ## Later clients - **ImGui viewports:** `ImGuiConfigFlags_ViewportsEnable`, a platform backend that creates SDL or @@ -285,8 +305,6 @@ can be tested on its own. - **Desktop VR mirror:** with OpenXR on Vulkan, each XR swapchain is a `VulkanPresentation` (acquire maps to `xrAcquireSwapchainImage`/`xrWaitSwapchainImage`, present to `xrReleaseSwapchainImage`), and the desktop window stays primary. VR is GL-only today, so this also needs a Vulkan OpenXR path. -- **Android external display:** a `Presentation` dialog with its own `Surface`, as a `Game` output on - Vulkan. ## Phases @@ -299,8 +317,8 @@ can be tested on its own. can add an offscreen second output, so frametests can check what it shows. 4. **iOS TV mode,** on Metal, tested with the Apple TV. 5. **Vulkan:** done, tested with `--game-window` on Mac, Linux (SDL) and Windows (a second Win32 - window), and on iOS with MoltenVK. Then Android (Vulkan only, through `Presentation`), and D3D11 - when something wants it. + window), and on iOS with MoltenVK. D3D11 when something wants it. +6. **Android,** on Vulkan, through `Presentation`. Open questions: - Does removing the external window resume mirroring?