From e16da314ea9c95d5c72f2ebeb9cd240d9fc878c3 Mon Sep 17 00:00:00 2001 From: Logan McNaughton <848146+loganmc10@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:32:07 +0000 Subject: [PATCH] Android touch controller (#1234) * touch controller for android * use sdl axis values * connect * more * more * more * more --- .../github/gopher64/gopher64/N64Activity.kt | 73 +++- .../gopher64/gopher64/TouchOverlayView.kt | 356 ++++++++++++++++++ src/ui/android.rs | 61 ++- src/ui/gui.rs | 14 +- src/ui/input.rs | 12 + src/ui/netplay.rs | 3 +- 6 files changed, 513 insertions(+), 6 deletions(-) create mode 100644 android-project/app/src/main/java/io/github/gopher64/gopher64/TouchOverlayView.kt diff --git a/android-project/app/src/main/java/io/github/gopher64/gopher64/N64Activity.kt b/android-project/app/src/main/java/io/github/gopher64/gopher64/N64Activity.kt index e7e37d4e..e28288b1 100644 --- a/android-project/app/src/main/java/io/github/gopher64/gopher64/N64Activity.kt +++ b/android-project/app/src/main/java/io/github/gopher64/gopher64/N64Activity.kt @@ -1,13 +1,19 @@ package io.github.gopher64.gopher64 -import android.content.Intent -import android.os.PowerManager import android.content.Context +import android.content.Intent import android.os.Bundle +import android.os.PowerManager import android.util.Log +import android.view.View +import android.view.ViewGroup +import android.widget.RelativeLayout import org.libsdl.app.SDLActivity -class N64Activity : SDLActivity() { +class N64Activity : SDLActivity(), TouchOverlayView.Listener { + private var leftOverlay: TouchOverlayView? = null + private var rightOverlay: TouchOverlayView? = null + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -19,6 +25,64 @@ class N64Activity : SDLActivity() { } else { Log.v("SDL", "Sustained performance mode not supported") } + + if (intent?.getBooleanExtra("show_touch_overlay", false) == true) { + setupTouchOverlay() + } + } + + private fun setupTouchOverlay() { + val layout = mLayout ?: return + val left = TouchOverlayView(this, TouchOverlayView.Side.LEFT, this) + val right = TouchOverlayView(this, TouchOverlayView.Side.RIGHT, this) + + val leftParams = RelativeLayout.LayoutParams(0, ViewGroup.LayoutParams.MATCH_PARENT).apply { + addRule(RelativeLayout.ALIGN_PARENT_LEFT) + addRule(RelativeLayout.ALIGN_PARENT_TOP) + addRule(RelativeLayout.ALIGN_PARENT_BOTTOM) + } + val rightParams = RelativeLayout.LayoutParams(0, ViewGroup.LayoutParams.MATCH_PARENT).apply { + addRule(RelativeLayout.ALIGN_PARENT_RIGHT) + addRule(RelativeLayout.ALIGN_PARENT_TOP) + addRule(RelativeLayout.ALIGN_PARENT_BOTTOM) + } + + layout.addView(left, leftParams) + layout.addView(right, rightParams) + leftOverlay = left + rightOverlay = right + + layout.addOnLayoutChangeListener { _, leftBound, top, rightBound, bottom, _, _, _, _ -> + updateOverlayLayout(rightBound - leftBound, bottom - top) + } + layout.post { + updateOverlayLayout(layout.width, layout.height) + } + } + + private fun updateOverlayLayout(width: Int, height: Int) { + val pillar = TouchOverlayView.pillarWidth(width, height) + val visible = if (pillar > 0) View.VISIBLE else View.GONE + leftOverlay?.let { view -> + view.visibility = visible + val params = view.layoutParams as RelativeLayout.LayoutParams + params.width = pillar + view.layoutParams = params + } + rightOverlay?.let { view -> + view.visibility = visible + val params = view.layoutParams as RelativeLayout.LayoutParams + params.width = pillar + view.layoutParams = params + } + } + + override fun onTouchButton(button: Int, pressed: Boolean) { + nativeTouchButton(button, if (pressed) 1 else 0) + } + + override fun onTouchAxis(x: Int, y: Int) { + nativeTouchAxis(x, y) } override fun getLibraries(): Array = arrayOf( @@ -45,4 +109,7 @@ class N64Activity : SDLActivity() { setResult(RESULT_OK, dataIntent) return args } + + private external fun nativeTouchButton(button: Int, pressed: Int) + private external fun nativeTouchAxis(x: Int, y: Int) } diff --git a/android-project/app/src/main/java/io/github/gopher64/gopher64/TouchOverlayView.kt b/android-project/app/src/main/java/io/github/gopher64/gopher64/TouchOverlayView.kt new file mode 100644 index 00000000..bea5fefb --- /dev/null +++ b/android-project/app/src/main/java/io/github/gopher64/gopher64/TouchOverlayView.kt @@ -0,0 +1,356 @@ +package io.github.gopher64.gopher64 + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.PointF +import android.util.TypedValue +import android.view.MotionEvent +import android.view.View +import kotlin.math.hypot +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt + +/** + * On-screen N64 controls for one side of the screen (left or right pillar). + * + * Button indices match gopher64's N64 bit order (R_DPAD=0 … L_TRIG=13). + */ +class TouchOverlayView( + context: Context, + private val side: Side, + private val listener: Listener, +) : View(context) { + + enum class Side { LEFT, RIGHT } + + interface Listener { + fun onTouchButton(button: Int, pressed: Boolean) + fun onTouchAxis(x: Int, y: Int) + } + + private data class ButtonControl( + val button: Int, + val label: String, + val center: PointF, + val radius: Float, + val fillColor: Int = Color.argb(90, 255, 255, 255), + val pressedColor: Int = Color.argb(160, 255, 255, 255), + var pointerId: Int = -1, + ) + + private data class StickControl( + val center: PointF, + val radius: Float, + var knobOffsetX: Float = 0f, + var knobOffsetY: Float = 0f, + var pointerId: Int = -1, + ) + + private val fillPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + style = Paint.Style.FILL + } + private val strokePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + style = Paint.Style.STROKE + strokeWidth = dp(2f) + color = Color.argb(200, 255, 255, 255) + } + private val textPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = Color.argb(220, 255, 255, 255) + textAlign = Paint.Align.CENTER + isFakeBoldText = true + } + + private var buttons: List = emptyList() + private var stick: StickControl? = null + + override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { + super.onSizeChanged(w, h, oldw, oldh) + layoutControls(w.toFloat(), h.toFloat()) + } + + private fun layoutControls(w: Float, h: Float) { + if (w <= 0f || h <= 0f) { + buttons = emptyList() + stick = null + return + } + + val pad = dp(8f) + val usableW = max(1f, w - pad * 2f) + val cx = w / 2f + val buttonR = min(usableW * 0.22f, h * 0.055f) + textPaint.textSize = buttonR * 0.9f + + when (side) { + Side.LEFT -> { + // Keep stick and D-pad separated: D-pad upper-middle, stick lower. + val stickR = min(usableW * 0.38f, h * 0.13f) + val dpadCy = h * 0.34f + val dpadSpread = buttonR * 1.85f + val dpadBottom = dpadCy + dpadSpread + buttonR + val zR = buttonR + val zGap = dp(10f) + // Leave room above the stick for Z between D-pad and stick. + val stickCy = max( + h * 0.78f, + dpadBottom + zR * 2f + zGap * 2f + stickR + dp(8f), + ) + val stickCenter = PointF(cx, min(stickCy, h - stickR - pad)) + stick = StickControl(stickCenter, stickR) + + // Z sits above the stick on its left side, with a clear gap. + val zCenter = PointF( + max(pad + zR, stickCenter.x - stickR * 0.65f), + stickCenter.y - stickR - zR - zGap, + ) + + buttons = listOf( + ButtonControl(L_TRIG, "L", PointF(cx, pad + buttonR), buttonR * 1.1f), + ButtonControl(Z_TRIG, "Z", zCenter, zR), + ButtonControl(U_DPAD, "↑", PointF(cx, dpadCy - dpadSpread), buttonR), + ButtonControl(D_DPAD, "↓", PointF(cx, dpadCy + dpadSpread), buttonR), + ButtonControl(L_DPAD, "←", PointF(cx - dpadSpread, dpadCy), buttonR), + ButtonControl(R_DPAD, "→", PointF(cx + dpadSpread, dpadCy), buttonR), + ) + } + Side.RIGHT -> { + stick = null + val rR = buttonR * 1.1f + val startR = buttonR * 0.85f + val cR = buttonR * 0.85f + val faceR = buttonR * 1.15f + val cSpread = buttonR * 1.55f + val faceSpread = buttonR * 1.55f + val gap = dp(10f) + val faceCGap = dp(28f) + + // Anchor A/B and C as low as possible, with clear space between them. + val faceY = h - pad - faceR + val cY = faceY - faceR - faceCGap - cSpread + val cTop = cY - cSpread - cR + + // Keep R and Start clearly separated in the upper area. + val rY = pad + rR + val startY = min(rY + rR + gap + startR, cTop - gap - startR) + + buttons = listOf( + ButtonControl(R_TRIG, "R", PointF(cx, rY), rR), + ButtonControl( + START_BUTTON, "S", PointF(cx, startY), startR, + fillColor = Color.argb(120, 220, 40, 40), + pressedColor = Color.argb(200, 255, 70, 70), + ), + ButtonControl( + U_CBUTTON, "C↑", PointF(cx, cY - cSpread), cR, + fillColor = Color.argb(120, 230, 200, 40), + pressedColor = Color.argb(200, 255, 230, 80), + ), + ButtonControl( + D_CBUTTON, "C↓", PointF(cx, cY + cSpread), cR, + fillColor = Color.argb(120, 230, 200, 40), + pressedColor = Color.argb(200, 255, 230, 80), + ), + ButtonControl( + L_CBUTTON, "C←", PointF(cx - cSpread, cY), cR, + fillColor = Color.argb(120, 230, 200, 40), + pressedColor = Color.argb(200, 255, 230, 80), + ), + ButtonControl( + R_CBUTTON, "C→", PointF(cx + cSpread, cY), cR, + fillColor = Color.argb(120, 230, 200, 40), + pressedColor = Color.argb(200, 255, 230, 80), + ), + ButtonControl( + B_BUTTON, "B", PointF(cx - faceSpread, faceY), faceR, + fillColor = Color.argb(120, 40, 180, 70), + pressedColor = Color.argb(200, 70, 220, 100), + ), + ButtonControl( + A_BUTTON, "A", PointF(cx + faceSpread, faceY), faceR, + fillColor = Color.argb(120, 50, 90, 220), + pressedColor = Color.argb(200, 80, 120, 255), + ), + ) + } + } + invalidate() + } + + override fun onDraw(canvas: Canvas) { + for (button in buttons) { + fillPaint.color = if (button.pointerId >= 0) button.pressedColor else button.fillColor + canvas.drawCircle(button.center.x, button.center.y, button.radius, fillPaint) + canvas.drawCircle(button.center.x, button.center.y, button.radius, strokePaint) + val textY = button.center.y - (textPaint.descent() + textPaint.ascent()) / 2f + canvas.drawText(button.label, button.center.x, textY, textPaint) + } + + stick?.let { s -> + fillPaint.color = if (s.pointerId >= 0) { + Color.argb(160, 255, 255, 255) + } else { + Color.argb(90, 255, 255, 255) + } + canvas.drawCircle(s.center.x, s.center.y, s.radius, fillPaint) + canvas.drawCircle(s.center.x, s.center.y, s.radius, strokePaint) + val knobR = s.radius * 0.38f + val kx = s.center.x + s.knobOffsetX + val ky = s.center.y + s.knobOffsetY + fillPaint.color = if (s.pointerId >= 0) { + Color.argb(180, 255, 255, 255) + } else { + Color.argb(120, 255, 255, 255) + } + canvas.drawCircle(kx, ky, knobR, fillPaint) + canvas.drawCircle(kx, ky, knobR, strokePaint) + } + } + + override fun onTouchEvent(event: MotionEvent): Boolean { + var handled = false + when (event.actionMasked) { + MotionEvent.ACTION_DOWN, MotionEvent.ACTION_POINTER_DOWN -> { + val index = event.actionIndex + handled = pressAt(event.getPointerId(index), event.getX(index), event.getY(index)) + } + MotionEvent.ACTION_MOVE -> { + for (i in 0 until event.pointerCount) { + if (movePointer(event.getPointerId(i), event.getX(i), event.getY(i))) { + handled = true + } + } + } + MotionEvent.ACTION_UP, MotionEvent.ACTION_POINTER_UP -> { + handled = releasePointer(event.getPointerId(event.actionIndex)) + } + MotionEvent.ACTION_CANCEL -> { + releaseAll() + return true + } + } + if (handled) { + invalidate() + } + return handled + } + + private fun pressAt(pointerId: Int, x: Float, y: Float): Boolean { + stick?.let { s -> + if (s.pointerId < 0 && hypot(x - s.center.x, y - s.center.y) <= s.radius * 1.15f) { + s.pointerId = pointerId + updateStick(s, x, y) + return true + } + } + for (button in buttons) { + if (button.pointerId < 0 && + hypot(x - button.center.x, y - button.center.y) <= button.radius * 1.15f + ) { + button.pointerId = pointerId + listener.onTouchButton(button.button, true) + return true + } + } + return false + } + + private fun movePointer(pointerId: Int, x: Float, y: Float): Boolean { + stick?.let { s -> + if (s.pointerId == pointerId) { + updateStick(s, x, y) + return true + } + } + return buttons.any { it.pointerId == pointerId } + } + + private fun releasePointer(pointerId: Int): Boolean { + stick?.let { s -> + if (s.pointerId == pointerId) { + s.pointerId = -1 + s.knobOffsetX = 0f + s.knobOffsetY = 0f + listener.onTouchAxis(0, 0) + return true + } + } + for (button in buttons) { + if (button.pointerId == pointerId) { + button.pointerId = -1 + listener.onTouchButton(button.button, false) + return true + } + } + return false + } + + private fun releaseAll() { + stick?.let { s -> + if (s.pointerId >= 0) { + s.pointerId = -1 + s.knobOffsetX = 0f + s.knobOffsetY = 0f + listener.onTouchAxis(0, 0) + } + } + for (button in buttons) { + if (button.pointerId >= 0) { + button.pointerId = -1 + listener.onTouchButton(button.button, false) + } + } + invalidate() + } + + private fun updateStick(s: StickControl, x: Float, y: Float) { + var dx = x - s.center.x + var dy = y - s.center.y + val len = hypot(dx, dy) + if (len > s.radius) { + dx = dx / len * s.radius + dy = dy / len * s.radius + } + s.knobOffsetX = dx + s.knobOffsetY = dy + // Match SDL axis range (i16): +X right, +Y up. Android Y grows downward. + val axisX = ((dx / s.radius) * Short.MAX_VALUE) + .roundToInt() + .coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) + val axisY = ((-dy / s.radius) * Short.MAX_VALUE) + .roundToInt() + .coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) + listener.onTouchAxis(axisX, axisY) + } + + private fun dp(value: Float): Float = + TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, value, resources.displayMetrics) + + companion object { + const val R_DPAD = 0 + const val L_DPAD = 1 + const val D_DPAD = 2 + const val U_DPAD = 3 + const val START_BUTTON = 4 + const val Z_TRIG = 5 + const val B_BUTTON = 6 + const val A_BUTTON = 7 + const val R_CBUTTON = 8 + const val L_CBUTTON = 9 + const val D_CBUTTON = 10 + const val U_CBUTTON = 11 + const val R_TRIG = 12 + const val L_TRIG = 13 + + /** Left/right pillar widths for a centered 4:3 game area. */ + fun pillarWidth(viewWidth: Int, viewHeight: Int): Int { + if (viewWidth <= 0 || viewHeight <= 0) { + return 0 + } + val gameW = min(viewWidth.toFloat(), viewHeight * 4f / 3f) + return ((viewWidth - gameW) / 2f).roundToInt().coerceAtLeast(0) + } + } +} diff --git a/src/ui/android.rs b/src/ui/android.rs index 1c3363ae..72b8c08a 100644 --- a/src/ui/android.rs +++ b/src/ui/android.rs @@ -13,6 +13,18 @@ const REQUEST_SELECT_ROM: jint = 1; const CONFIGURE_INPUT_PROFILE: jint = 2; const RUN_ROM: jint = 3; +pub struct TouchOverlay { + pub buttons: u32, + pub x: i16, + pub y: i16, +} + +pub static TOUCH_OVERLAY: std::sync::Mutex = std::sync::Mutex::new(TouchOverlay { + buttons: 0, + x: 0, + y: 0, +}); + pub static ANDROID_APP: std::sync::Mutex> = std::sync::Mutex::new(None); @@ -82,6 +94,10 @@ bind_java_type! { sig = (extra: JString, value: JString[]) -> AndroidIntent, name = "putExtra", }, + fn put_extra_boolean { + sig = (extra: JString, value: jboolean) -> AndroidIntent, + name = "putExtra", + }, fn get_string_extra(name: JString) -> JString, fn set_class_name(package_name: JString, class_name: JString) -> AndroidIntent, }, @@ -241,13 +257,22 @@ pub fn run_rom( file_path: std::path::PathBuf, game_settings: ui::GameSettings, netplay: Option, + show_touch_overlay: bool, weak: slint::Weak, ) { if let Ok(app) = ANDROID_APP.lock() && let Some(app) = app.as_ref() { if let Err(err) = get_vm(app).attach_current_thread(|env| { - start_run_rom_on_jvm(env, app, file_path, game_settings, netplay, weak) + start_run_rom_on_jvm( + env, + app, + file_path, + game_settings, + netplay, + show_touch_overlay, + weak, + ) }) { eprintln!("JNI error while starting N64Activity: {err:?}"); } @@ -260,6 +285,7 @@ fn start_run_rom_on_jvm( file_path: std::path::PathBuf, game_settings: ui::GameSettings, netplay: Option, + show_touch_overlay: bool, weak: slint::Weak, ) -> jni::errors::Result<()> { let raw_activity_global = app.activity_as_ptr() as jni::sys::jobject; @@ -314,10 +340,14 @@ fn start_run_rom_on_jvm( let file_path_string = JString::from_str(env, file_path)?; let cheats_path_string = JString::from_str(env, cheats_path.to_str().unwrap())?; + + let show_touch_overlay_key = JString::from_str(env, "show_touch_overlay")?; + let intent = AndroidIntent::new(env)? .set_class_name(env, &package_name, &class_name)? .put_extra_string(env, &file_path_key, &file_path_string)? .put_extra_string(env, &cheats_path_key, &cheats_path_string)? + .put_extra_boolean(env, &show_touch_overlay_key, show_touch_overlay)? .put_extra_string_array(env, &args_key, &j_args)?; weak.upgrade_in_event_loop(move |handle| handle.set_game_running(true)) @@ -329,6 +359,35 @@ fn start_run_rom_on_jvm( Ok(()) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_io_github_gopher64_gopher64_N64Activity_nativeTouchButton<'caller>( + _unowned_env: EnvUnowned<'caller>, + _this: JObject<'caller>, + button: jint, + pressed: jint, +) { + if let Ok(mut touch_overlay) = TOUCH_OVERLAY.lock() { + if pressed == 1 { + touch_overlay.buttons |= 1 << button; + } else { + touch_overlay.buttons &= !(1 << button); + } + } +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_io_github_gopher64_gopher64_N64Activity_nativeTouchAxis<'caller>( + _unowned_env: EnvUnowned<'caller>, + _this: JObject<'caller>, + x: jint, + y: jint, +) { + if let Ok(mut touch_overlay) = TOUCH_OVERLAY.lock() { + touch_overlay.x = x as i16; + touch_overlay.y = y as i16; + } +} + fn get_vm(app: &slint::android::AndroidApp) -> JavaVM { unsafe { JavaVM::from_raw(app.vm_as_ptr().cast()) } } diff --git a/src/ui/gui.rs b/src/ui/gui.rs index 2fd2928c..c32485b2 100644 --- a/src/ui/gui.rs +++ b/src/ui/gui.rs @@ -84,6 +84,7 @@ fn run_with_path(weak: slint::Weak, path: std::path::PathBuf) { load_savestate_slot: None, }, None, + show_touch_overlay(&handle), weak2, ); }) @@ -570,10 +571,11 @@ pub fn run_rom( file_path: std::path::PathBuf, game_settings: ui::GameSettings, netplay: Option, + _show_touch_overlay: bool, weak: slint::Weak, ) { #[cfg(target_os = "android")] - ui::android::run_rom(file_path, game_settings, netplay, weak); + ui::android::run_rom(file_path, game_settings, netplay, _show_touch_overlay, weak); #[cfg(not(target_os = "android"))] tokio::spawn(async move { @@ -784,6 +786,7 @@ fn open_rom(app: &AppWindow) { let overclock = app.get_overclock_n64_cpu(); let disable_expansion_pak = app.get_disable_expansion_pak(); + let show_touch_overlay = show_touch_overlay(app); let weak = app.as_weak(); tokio::spawn(async move { @@ -797,8 +800,17 @@ fn open_rom(app: &AppWindow) { load_savestate_slot: None, }, None, + show_touch_overlay, weak, ); } }); } + +pub fn show_touch_overlay(app: &AppWindow) -> bool { + let selected = app.get_selected_controller().row_data(0).unwrap_or(0); + app.get_controller_paths() + .row_data(selected as usize) + .map(|p| p.is_empty()) + .unwrap_or(true) +} diff --git a/src/ui/input.rs b/src/ui/input.rs index 036f08d7..2c3c29c8 100644 --- a/src/ui/input.rs +++ b/src/ui/input.rs @@ -417,6 +417,18 @@ pub fn get(ui: &mut ui::Ui, channel: usize) -> InputData { ); let (mut x, mut y) = set_axis(profile, joystick, controller, ui.input.keyboard_state); + + #[cfg(target_os = "android")] + { + if channel == 0 + && ui.config.input.controller_assignment[0].is_none() + && let Ok(touch_overlay) = ui::android::TOUCH_OVERLAY.lock() + { + x = normalize_axis_position(touch_overlay.x); + y = normalize_axis_position(touch_overlay.y); + keys = touch_overlay.buttons; + } + } bound_axis(&mut x, &mut y); keys |= (x.round() as i8 as u8 as u32) << X_AXIS_SHIFT; diff --git a/src/ui/netplay.rs b/src/ui/netplay.rs index 2519c4b8..b8f029f6 100644 --- a/src/ui/netplay.rs +++ b/src/ui/netplay.rs @@ -1,7 +1,7 @@ use crate::device; use crate::netplay::RtcIceServerConfig; use crate::ui; -use crate::ui::gui::{AppWindow, open_uri, run_rom, save_settings}; +use crate::ui::gui::{AppWindow, open_uri, run_rom, save_settings, show_touch_overlay}; use futures::{SinkExt, StreamExt}; use sha2::digest::Digest; use slint::ComponentHandle; @@ -824,6 +824,7 @@ fn setup_wait_window( number_of_players: players.row_count(), input_delay, }), + show_touch_overlay(&handle), weak_app2, ); handle.invoke_netplay_close();