Android: on-screen keyboard over the running game, and a setting to enable it

Uses the Android system IME rather than a drawn key grid, so layouts, languages, prediction
and emoji come for free and it is the keyboard users already know. Bound to a new
KEYBOARD_TOGGLE hotkey, so it can be raised and dismissed mid-game without opening
settings.

The IME only opens for a focused view that accepts input, so a zero-size transparent
EditText owns focus on demand. Its InputConnection does the real work, because an IME
reports typing in two different ways and only one of them is a key event:

  sendKeyEvent            backspace, enter, arrows -- forward the keycode as-is
  commitText              ordinary characters, with NO key event behind them
  deleteSurroundingText   some IMEs delete by range instead of sending backspace

commitText is synthesised with KeyCharacterMap.getEvents, which produces the shift presses
capitals and symbols need rather than guessing a keycode per character. Characters no
keycode can produce -- emoji, CJK picked from a candidate list -- are still delivered with
their unicode and KEYCODE_UNKNOWN, since the guest reads the unicode field and that is the
honest keycode for a character with no key behind it.

Also adds the Emulated Keyboard setting, without which all of this was inert: the core
defaults to keyboard_handler::null, so cellKb told games no keyboard was attached no matter
what was typed. Off by default, matching the core, because a game that sees a keyboard can
behave differently. The hotkey says so in its toast when the guest cannot receive keys,
rather than silently showing an IME that goes nowhere.

android/armsx3-ui/app/src/main/java/com/armsx2/config/Settings.kt

@@ -193,6 +193,9 @@ data class Ps3Settings(
     /** Stops the core writing any log after startup. A real performance lever on games that
      *  log heavily, but it also destroys the only artifact a bug report can carry, so it is
      *  off by default and the UI says so plainly. */
+    // Emulated keyboard. Off by default, matching the core's keyboard_handler::null: a game that
+    // sees a keyboard attached can behave differently, so this is opt-in rather than assumed.
+    val emulatedKeyboard: Boolean = false,
     val silenceAllLogs: Boolean = false,
     val netEnabled: Boolean = false,
     val psnStatus: Boolean = false,
@@ -1065,6 +1068,7 @@ data class Settings(
         // surface layout. Keeping them in sync stops "Stretch" looking inert.
         put("PS3/Video", "Stretch To Display Area", "bool", (displayFitMode == 1).toString())
         put("PS3/Video", "Display Aspect Override", "int", ps3.displayAspect.coerceIn(0, 4000).toString())
+        put("PS3/Input", "Keyboard", "bool", ps3.emulatedKeyboard.toString())
         put("PS3/Misc", "Silence All Logs", "bool", ps3.silenceAllLogs.toString())
         put("PS3/Overlay", "Enabled", "bool", ps3.overlayEnabled.toString())
         put("PS3/Overlay", "Detail level", "enum", ps3.overlayDetail.toString())
@@ -2060,6 +2064,7 @@ data class Settings(
         put("ps3FrameGenFlowScale", ps3.frameGenFlowScale)
         put("ps3WriteColorBuffers", ps3.writeColorBuffers)
         put("ps3GpuTurbo", ps3.gpuTurbo)
+        put("ps3EmulatedKeyboard", ps3.emulatedKeyboard)
         put("ps3SilenceAllLogs", ps3.silenceAllLogs)
         put("ps3WriteDepthBuffer", ps3.writeDepthBuffer)
         put("ps3ReadColorBuffers", ps3.readColorBuffers)
@@ -2407,6 +2412,7 @@ data class Settings(
                     frameGenFlowScale = json.optInt("ps3FrameGenFlowScale", def.ps3.frameGenFlowScale),
                     writeColorBuffers = json.optBoolean("ps3WriteColorBuffers", def.ps3.writeColorBuffers),
                     gpuTurbo = json.optBoolean("ps3GpuTurbo", def.ps3.gpuTurbo),
+                    emulatedKeyboard = json.optBoolean("ps3EmulatedKeyboard", def.ps3.emulatedKeyboard),
                     silenceAllLogs = json.optBoolean("ps3SilenceAllLogs", def.ps3.silenceAllLogs),
                     writeDepthBuffer = json.optBoolean("ps3WriteDepthBuffer", def.ps3.writeDepthBuffer),
                     readColorBuffers = json.optBoolean("ps3ReadColorBuffers", def.ps3.readColorBuffers),
@@ -2734,6 +2740,7 @@ data class Settings(
             if (current.ps3.frameGenFlowScale != base.ps3.frameGenFlowScale) j.put("ps3FrameGenFlowScale", current.ps3.frameGenFlowScale)
             if (current.ps3.writeColorBuffers != base.ps3.writeColorBuffers) j.put("ps3WriteColorBuffers", current.ps3.writeColorBuffers)
             if (current.ps3.gpuTurbo != base.ps3.gpuTurbo) j.put("ps3GpuTurbo", current.ps3.gpuTurbo)
+            if (current.ps3.emulatedKeyboard != base.ps3.emulatedKeyboard) j.put("ps3EmulatedKeyboard", current.ps3.emulatedKeyboard)
             if (current.ps3.silenceAllLogs != base.ps3.silenceAllLogs) j.put("ps3SilenceAllLogs", current.ps3.silenceAllLogs)
             if (current.ps3.writeDepthBuffer != base.ps3.writeDepthBuffer) j.put("ps3WriteDepthBuffer", current.ps3.writeDepthBuffer)
             if (current.ps3.readColorBuffers != base.ps3.readColorBuffers) j.put("ps3ReadColorBuffers", current.ps3.readColorBuffers)
@@ -3042,6 +3049,7 @@ data class Settings(
                     frameGenFlowScale = if (overrides.has("ps3FrameGenFlowScale")) overrides.getInt("ps3FrameGenFlowScale") else base.ps3.frameGenFlowScale,
                     writeColorBuffers = if (overrides.has("ps3WriteColorBuffers")) overrides.getBoolean("ps3WriteColorBuffers") else base.ps3.writeColorBuffers,
                     gpuTurbo = if (overrides.has("ps3GpuTurbo")) overrides.getBoolean("ps3GpuTurbo") else base.ps3.gpuTurbo,
+                    emulatedKeyboard = if (overrides.has("ps3EmulatedKeyboard")) overrides.getBoolean("ps3EmulatedKeyboard") else base.ps3.emulatedKeyboard,
                     silenceAllLogs = if (overrides.has("ps3SilenceAllLogs")) overrides.getBoolean("ps3SilenceAllLogs") else base.ps3.silenceAllLogs,
                     writeDepthBuffer = if (overrides.has("ps3WriteDepthBuffer")) overrides.getBoolean("ps3WriteDepthBuffer") else base.ps3.writeDepthBuffer,
                     readColorBuffers = if (overrides.has("ps3ReadColorBuffers")) overrides.getBoolean("ps3ReadColorBuffers") else base.ps3.readColorBuffers,

android/armsx3-ui/app/src/main/java/com/armsx2/i18n/I18n.kt

@@ -1080,6 +1080,9 @@ val EN: Map<String, String> = mapOf(
     "perf.framegen.x3" to "x3",
     "perf.framegen.x4" to "x4",
     "perf.framegen.description" to "EXPERIMENTAL. Insert generated frames between the ones the game actually draws. Costs GPU time and adds latency, so it helps when the CPU is the limit and hurts when the GPU already is.\n\nIt works best from a steady framerate. Interpolating a game that is already struggling tends to look worse rather than better \u2014 generated frames land at the wrong moment when the real interval keeps changing, which reads as judder. A locked 25 usually looks better than a wandering 28.\n\nOn-screen text shimmers or flickers while this is on \u2014 the overlay and the game\u0027s own menus get interpolated along with everything else, and fine text is what that looks worst on. That is how frame generation behaves, not a fault. Turning it off restores steady text. Switching it on or off during a game also pauses for a few seconds while the shaders are prepared.\n\nThis does nothing until you import Lossless.dll below. It is part of Lossless Scaling on Steam \u2014 you need your own copy, and nothing is bundled or downloaded. On Windows the file sits in steamapps\\common\\Lossless Scaling\\Lossless.dll; copy it to your device and pick it with the button below. Only the shaders are kept, and your copy of the file is deleted afterwards.",
+    "pad.section.keyboard" to "Keyboard",
+    "pad.emulatedKeyboard.label" to "Emulated Keyboard",
+    "pad.emulatedKeyboard.description" to "Tell games a keyboard is attached. Needed by titles with keyboard support \u2014 online text chat, Counter-Strike, and the debug menus in some beta builds. With this on, a USB or Bluetooth keyboard works directly, and the \"Toggle Keyboard\" hotkey raises the Android keyboard over the game. Off by default because a game that sees a keyboard can behave differently.",
     "perf.ppuDecoder.label" to "PPU Decoder",
     "perf.ppuDecoder.description" to "How the PS3's main CPU (PPU) is executed. LLVM recompiles PowerPC to native ARM64 and is enormously faster \u2014 keep it unless you are debugging. Interpreter is only for diagnosing a game LLVM gets wrong.",
     "perf.spuDecoder.label" to "SPU Decoder",

android/armsx3-ui/app/src/main/java/com/armsx2/input/ControllerMappings.kt

@@ -887,6 +887,12 @@ object ControllerMappings {
         // "only while aiming" binding (gyro live only while the button is held, so the
         // phone can sit still the rest of the time). Both drive
         // MainActivityRuntime.gyroActive and are session-only, never persisted.
+        // Raises and dismisses the ANDROID system keyboard over the running game, for titles
+        // that want real keyboard input -- online text chat, Counter-Strike, NFS Most Wanted's
+        // beta debug menu. The system IME is used rather than a drawn key grid so layouts,
+        // languages, prediction and emoji all come for free. Keys reach the guest through
+        // android_keyboard_handler; see MainActivityRuntime.toggleGuestKeyboard.
+        KEYBOARD_TOGGLE("pad.keyboardtoggle.keycode", "Toggle Keyboard"),
         GYRO_TOGGLE("pad.gyrotoggle.keycode", "Gyro On/Off (toggle)"),
         GYRO_HOLD("pad.gyrohold.keycode", "Gyro (hold to aim)"),
         // Raises/drops the Android IME over the running game and routes what it types to the

android/armsx3-ui/app/src/main/java/com/armsx2/runtime/MainActivityRuntime.kt

@@ -3235,6 +3235,10 @@ open class MainActivityRuntime : ComponentActivity() {
                     if (down && event.repeatCount == 0) hotkeyToast(InGameOverlay.cycleOsd())
                     return true
                 }
+                ControllerMappings.SysHotkey.KEYBOARD_TOGGLE -> {
+                    if (down && event.repeatCount == 0) toggleGuestKeyboard()
+                    return true
+                }
                 ControllerMappings.SysHotkey.GYRO_TOGGLE -> {
                     if (down && event.repeatCount == 0) toggleGyro()
                     return true
@@ -3403,6 +3407,134 @@ open class MainActivityRuntime : ComponentActivity() {
      *  hotkey and the on-screen fast-forward touch button (FastForwardWidget). Restores
      *  the user's base limiter mode when turning off so it stays in sync with the
      *  frame-limit toggle. */
+    // ---- Guest keyboard (Android IME over the running game) --------------------------------
+    //
+    // The system IME is deliberately used instead of a drawn key grid: layouts, languages,
+    // prediction and emoji all come for free, and it is the keyboard users already know. Keys
+    // reach the guest through android_keyboard_handler, which registers Android keycodes directly.
+    //
+    // The IME will only open for a focused view that accepts input, so a zero-size EditText is
+    // parked in the content view and focused on demand. Its InputConnection is where the work
+    // happens, because an IME reports typing in two different ways and only one of them is a key
+    // event:
+    //   * sendKeyEvent  -- backspace, enter, arrows: forward the keycode as-is
+    //   * commitText    -- ordinary typed characters, no key event at all, so one has to be
+    //                     synthesised per character
+    // KeyCharacterMap.getEvents does that synthesis properly, including the shift presses needed
+    // for capitals and symbols, rather than guessing a keycode per char.
+    private var guestKeyboardView: android.widget.EditText? = null
+
+    private fun guestKeyboardTarget(): android.widget.EditText {
+        guestKeyboardView?.let { return it }
+
+        val view = object : android.widget.EditText(this) {
+            override fun onCreateInputConnection(outAttrs: android.view.inputmethod.EditorInfo): android.view.inputmethod.InputConnection {
+                outAttrs.inputType = android.text.InputType.TYPE_CLASS_TEXT
+                outAttrs.imeOptions = android.view.inputmethod.EditorInfo.IME_FLAG_NO_FULLSCREEN or
+                    android.view.inputmethod.EditorInfo.IME_FLAG_NO_EXTRACT_UI
+
+                return object : android.view.inputmethod.BaseInputConnection(this, false) {
+                    override fun sendKeyEvent(event: KeyEvent): Boolean {
+                        sendToGuest(event.keyCode, event.action == KeyEvent.ACTION_DOWN, event.unicodeChar)
+                        return true
+                    }
+
+                    override fun commitText(text: CharSequence?, newCursorPosition: Int): Boolean {
+                        val chars = text?.toString() ?: return true
+                        val map = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
+                        val events = map.getEvents(chars.toCharArray())
+
+                        if (events == null) {
+                            // No keycode produces this character on the virtual layout (emoji, CJK
+                            // from an IME candidate list). Still deliver the text: the guest reads
+                            // the unicode field, and CELL_KEYC_NO_EVENT is the right keycode for
+                            // "a character arrived with no key behind it".
+                            for (ch in chars) {
+                                sendToGuest(KeyEvent.KEYCODE_UNKNOWN, true, ch.code)
+                                sendToGuest(KeyEvent.KEYCODE_UNKNOWN, false, ch.code)
+                            }
+
+                            return true
+                        }
+
+                        for (ev in events) {
+                            sendToGuest(ev.keyCode, ev.action == KeyEvent.ACTION_DOWN, ev.unicodeChar)
+                        }
+
+                        return true
+                    }
+
+                    override fun deleteSurroundingText(before: Int, after: Int): Boolean {
+                        // Some IMEs delete by range rather than by sending backspace.
+                        repeat(before.coerceAtLeast(0)) {
+                            sendToGuest(KeyEvent.KEYCODE_DEL, true, 0)
+                            sendToGuest(KeyEvent.KEYCODE_DEL, false, 0)
+                        }
+
+                        return true
+                    }
+                }
+            }
+        }
+
+        view.isFocusable = true
+        view.isFocusableInTouchMode = true
+        // Zero-size and transparent: it exists to own IME focus, never to be seen or to steal a
+        // touch from the game or the on-screen pad.
+        view.layoutParams = android.view.ViewGroup.LayoutParams(0, 0)
+        view.setBackgroundColor(0)
+        view.alpha = 0f
+
+        runCatching {
+            (findViewById<android.view.ViewGroup>(android.R.id.content)).addView(view)
+        }
+
+        guestKeyboardView = view
+        return view
+    }
+
+    private fun sendToGuest(keyCode: Int, pressed: Boolean, unicode: Int) {
+        runCatching { net.rpcsx.RPCSX.instance.keyboardKey(keyCode, pressed, unicode) }
+    }
+
+    /**
+     * Raise or dismiss the Android keyboard over the running game (KEYBOARD_TOGGLE hotkey).
+     *
+     * Reports through a toast when the guest has no keyboard to receive the keys, which is the
+     * common case: the core's Keyboard setting defaults to Null, and silently showing an IME that
+     * goes nowhere is worse than saying so.
+     */
+    fun toggleGuestKeyboard() {
+        val imm = getSystemService(android.content.Context.INPUT_METHOD_SERVICE)
+            as? android.view.inputmethod.InputMethodManager ?: return
+
+        if (guestKeyboardShown) {
+            guestKeyboardShown = false
+            runCatching { imm.hideSoftInputFromWindow(guestKeyboardTarget().windowToken, 0) }
+            guestKeyboardView?.clearFocus()
+            hotkeyToast("Keyboard hidden")
+            return
+        }
+
+        // Probe before showing: keyboardKey returns false when no keyboard is active, so a
+        // harmless no-op key tells us whether anything would receive input at all.
+        val reachable = runCatching {
+            net.rpcsx.RPCSX.instance.keyboardKey(KeyEvent.KEYCODE_UNKNOWN, false, 0)
+        }.getOrDefault(false)
+
+        val view = guestKeyboardTarget()
+        view.requestFocus()
+        guestKeyboardShown = true
+        runCatching { imm.showSoftInput(view, android.view.inputmethod.InputMethodManager.SHOW_IMPLICIT) }
+
+        hotkeyToast(
+            if (reachable) "Keyboard shown"
+            else "Keyboard shown (set Keyboard to Basic for the game to see it)",
+        )
+    }
+
+    private var guestKeyboardShown = false
+
     /** Flip the runtime gyro enable (issue #337). Shared by the GYRO_TOGGLE hotkey and the
      *  edge-triggered (stick/combo) path. Only silences the sensor for this session — the
      *  user's Gyro Mode setting is untouched, so re-enabling restores their configured mode. */
@@ -4582,6 +4714,7 @@ open class MainActivityRuntime : ComponentActivity() {
                 runCatching { NativeApp.speedhackLimitermode(if (on) ffLimiterMode() else baseLimiterMode()) }
                 hotkeyToast(if (on) "Fast Forward ON" else "Fast Forward OFF")
             }
+            ControllerMappings.SysHotkey.KEYBOARD_TOGGLE -> toggleGuestKeyboard()
             ControllerMappings.SysHotkey.GYRO_TOGGLE -> toggleGyro()
             // GYRO_HOLD needs key up/down edges, which this edge-triggered path (stick
             // directions / combos) doesn't provide — behave as a toggle here rather than

android/armsx3-ui/app/src/main/java/com/armsx2/ui/settings/PadTab.kt

@@ -466,6 +466,25 @@ fun PadTab(state: MutableState<Settings>) {
             )
         }
 
+        CollapsibleSection(str("pad.section.keyboard"), initiallyExpanded = false) {
+            // Emulated keyboard (keyboard_handler). Off by default, matching the core: a game that
+            // sees a keyboard attached can behave differently, so this is opt-in.
+            //
+            // With it on, a physical USB/BT keyboard reaches the guest directly, and the
+            // "Toggle Keyboard" hotkey raises the Android keyboard over the running game.
+            SegmentedRow(
+                label = str("pad.emulatedKeyboard.label"),
+                options = listOf(str("common.off"), str("common.on")),
+                selectedIndex = if (state.value.ps3.emulatedKeyboard) 1 else 0,
+                description = str("pad.emulatedKeyboard.description"),
+                onChange = { idx ->
+                    com.armsx2.ui.InGameOverlay.saveSettings(
+                        state.value.copy(ps3 = state.value.ps3.copy(emulatedKeyboard = idx == 1)),
+                    )
+                },
+            )
+        }
+
         CollapsibleSection(str("pad.section.buttonMapping"), initiallyExpanded = false) {
             ControllerMappings.actions.forEach { action ->
                 val physical = ControllerMappings.physicalForScope(action, editPlayer.intValue, editSerial)

android/armsx3-ui/app/src/main/java/com/armsx2/ui/settingshub/SettingsSearchIndex.kt

@@ -42,6 +42,7 @@ internal val SETTINGS_SEARCH_INDEX: List<SettingsSearchEntry> = listOf(
     SettingsSearchEntry("perf.sustainedPerformance.label", true, SettingsCategory.Performance),
     SettingsSearchEntry("perf.adpf.label", true, SettingsCategory.Performance),
     SettingsSearchEntry("perf.scheduler.label", true, SettingsCategory.Performance),
+    SettingsSearchEntry("pad.emulatedKeyboard.label", true, SettingsCategory.Controls),
     SettingsSearchEntry("perf.ppuDecoder.label", true, SettingsCategory.Performance),
     SettingsSearchEntry("perf.spuDecoder.label", true, SettingsCategory.Performance),
     SettingsSearchEntry("perf.spuBlockSize.label", true, SettingsCategory.Performance),

android/armsx3-ui/app/src/main/java/com/armsx3/Rpcs3Bridge.kt

@@ -612,6 +612,11 @@ object Rpcs3Bridge {
             //
             // Values cross as indices and Rpcs3Settings turns them into the core's enum NAMES;
             // see the tables there for why an index cannot be mapped arithmetically.
+            "PS3/Input" -> when (key) {
+                "Keyboard" -> Rpcs3Settings.setEmulatedKeyboard(asBool(value))
+                else -> return false
+            }
+
             "PS3/System" -> when (key) {
                 "Language" -> Rpcs3Settings.setConsoleLanguage(asInt(value))
                 "License Area" -> Rpcs3Settings.setConsoleRegion(asInt(value))

android/armsx3-ui/app/src/main/java/com/armsx3/Rpcs3Settings.kt

@@ -108,6 +108,15 @@ object Rpcs3Settings {
     fun setKeyboardType(index: Int) = setIndexedEnum("$SYSTEM@@Keyboard Type", KEYBOARD_TYPES, index, 0)
     fun setDateFormat(index: Int) = setIndexedEnum("$SYSTEM@@Date Format", DATE_FORMATS, index, 1)
     fun setTimeFormat(index: Int) = setIndexedEnum("$SYSTEM@@Time Format", TIME_FORMATS, index, 1)
+    /**
+     * keyboard_handler: exactly "Null" | "Basic".
+     *
+     * Basic makes cellKb report a keyboard attached, which is what lets games with keyboard
+     * support see input at all -- online text chat, Counter-Strike, NFS Most Wanted's debug menu.
+     * Off by default because a game that sees a keyboard can behave differently.
+     */
+    fun setEmulatedKeyboard(on: Boolean) = setEnum("$IO@@Keyboard", if (on) "Basic" else "Null")
+
     fun setEnterButtonAssign(index: Int) =
         setIndexedEnum("$SYSTEM@@Enter button assignment", ENTER_BUTTONS, index, 1)