feat: implement Android capability backends

app/build.gradle.kts

@@ -26,6 +26,7 @@ android {
 }
 
 dependencies {
+    implementation("androidx.core:core-ktx:1.15.0")
     implementation("com.squareup.okhttp3:okhttp:4.12.0")
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.10.1")
 }

app/src/main/AndroidManifest.xml

@@ -3,9 +3,11 @@
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
     <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
     <uses-permission android:name="android.permission.CAMERA" />
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
+    <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
     <uses-permission android:name="android.permission.QUERY_ALL_PACKAGES" />
 
     <application
@@ -25,6 +27,18 @@
         <service
             android:name=".HermesNodeService"
             android:exported="false"
-            android:foregroundServiceType="dataSync" />
+            android:foregroundServiceType="dataSync|microphone" />
+
+        <service
+            android:name=".HermesAccessibilityService"
+            android:exported="true"
+            android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
+            <intent-filter>
+                <action android:name="android.accessibilityservice.AccessibilityService" />
+            </intent-filter>
+            <meta-data
+                android:name="android.accessibilityservice"
+                android:resource="@xml/hermes_accessibility_service" />
+        </service>
     </application>
 </manifest>

app/src/main/java/net/nexlab/hermesnodeandroid/AndroidRuntimeGrants.kt

+/*
+ * Hermes Node Android
+ * Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+package net.nexlab.hermesnodeandroid
+
+import android.content.Intent
+
+/**
+ * Process-local holder for Android consent grants that cannot be stored durably.
+ * MediaProjection grants are intentionally volatile and must be provided by user consent.
+ */
+object AndroidRuntimeGrants {
+    @Volatile var mediaProjectionResultCode: Int? = null
+    @Volatile var mediaProjectionData: Intent? = null
+
+    fun setMediaProjection(resultCode: Int, data: Intent?) {
+        mediaProjectionResultCode = resultCode
+        mediaProjectionData = data
+    }
+
+    fun hasMediaProjection(): Boolean = mediaProjectionResultCode != null && mediaProjectionData != null
+}

app/src/main/java/net/nexlab/hermesnodeandroid/HermesAccessibilityService.kt

+/*
+ * Hermes Node Android
+ * Copyright (C) 2026 Stefy Lanza <stefy@nexlab.net>
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+package net.nexlab.hermesnodeandroid
+
+import android.accessibilityservice.AccessibilityService
+import android.accessibilityservice.GestureDescription
+import android.graphics.Path
+import android.os.Build
+import android.os.Bundle
+import android.view.accessibility.AccessibilityEvent
+import android.view.accessibility.AccessibilityNodeInfo
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+
+class HermesAccessibilityService : AccessibilityService() {
+    override fun onServiceConnected() {
+        instance = this
+    }
+
+    override fun onAccessibilityEvent(event: AccessibilityEvent?) {
+        lastPackageName = event?.packageName?.toString() ?: lastPackageName
+        lastClassName = event?.className?.toString() ?: lastClassName
+    }
+
+    override fun onInterrupt() = Unit
+
+    override fun onDestroy() {
+        if (instance === this) instance = null
+        super.onDestroy()
+    }
+
+    fun activeWindowInfo(): Map<String, Any?> {
+        val root = rootInActiveWindow
+        return mapOf(
+            "package" to (root?.packageName?.toString() ?: lastPackageName),
+            "class" to (root?.className?.toString() ?: lastClassName),
+            "text" to root?.text?.toString(),
+            "content_description" to root?.contentDescription?.toString(),
+            "source" to if (root != null) "accessibility_root" else "last_accessibility_event",
+        )
+    }
+
+    fun click(x: Int, y: Int, durationMs: Long = 60L): Boolean = gesture {
+        val p = Path().apply { moveTo(x.toFloat(), y.toFloat()) }
+        addStroke(GestureDescription.StrokeDescription(p, 0, durationMs))
+    }
+
+    fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, durationMs: Long = 300L): Boolean = gesture {
+        val p = Path().apply {
+            moveTo(x1.toFloat(), y1.toFloat())
+            lineTo(x2.toFloat(), y2.toFloat())
+        }
+        addStroke(GestureDescription.StrokeDescription(p, 0, durationMs))
+    }
+
+    fun longPress(x: Int, y: Int, durationMs: Long = 700L): Boolean = click(x, y, durationMs)
+
+    fun globalAction(action: Int): Boolean = performGlobalAction(action)
+
+    fun typeText(text: String): Boolean {
+        val node = rootInActiveWindow?.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: return false
+        val args = Bundle().apply { putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) }
+        return node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args)
+    }
+
+    private fun gesture(build: GestureDescription.Builder.() -> Unit): Boolean {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return false
+        val latch = CountDownLatch(1)
+        var ok = false
+        val desc = GestureDescription.Builder().apply(build).build()
+        dispatchGesture(desc, object : GestureResultCallback() {
+            override fun onCompleted(gestureDescription: GestureDescription?) {
+                ok = true
+                latch.countDown()
+            }
+            override fun onCancelled(gestureDescription: GestureDescription?) {
+                ok = false
+                latch.countDown()
+            }
+        }, null)
+        latch.await(2, TimeUnit.SECONDS)
+        return ok
+    }
+
+    companion object {
+        @Volatile private var instance: HermesAccessibilityService? = null
+        @Volatile private var lastPackageName: String? = null
+        @Volatile private var lastClassName: String? = null
+
+        fun current(): HermesAccessibilityService? = instance
+    }
+}

app/src/main/java/net/nexlab/hermesnodeandroid/MainActivity.kt

@@ -12,8 +12,10 @@ import android.content.Context
 import android.content.Intent
 import android.content.IntentFilter
 import android.content.pm.PackageManager
+import android.media.projection.MediaProjectionManager
 import android.os.Build
 import android.os.Bundle
+import android.provider.Settings
 import android.view.View
 import android.widget.Button
 import android.widget.CheckBox
@@ -120,11 +122,24 @@ class MainActivity : Activity() {
             text = "Refresh status"
             setOnClickListener { refreshStatus() }
         }
+        val permissions = Button(this).apply {
+            text = "Grant camera/microphone permissions"
+            setOnClickListener { requestRuntimePermissions() }
+        }
+        val accessibility = Button(this).apply {
+            text = "Open Accessibility settings"
+            setOnClickListener { startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)) }
+        }
+        val screenshotConsent = Button(this).apply {
+            text = "Grant screenshot consent"
+            setOnClickListener { requestScreenshotConsent() }
+        }
 
         listOf<View>(
             title, subtitle, gatewayUrl, nodeName, token, insecureTls,
             computerControl, desktopObserve, audioControl, cameraControl,
-            browserControl, execControl, save, start, stop, refresh, statusView,
+            browserControl, execControl, save, start, stop, refresh,
+            permissions, accessibility, screenshotConsent, statusView,
         ).forEach { root.addView(it) }
         setContentView(ScrollView(this).apply { addView(root) })
     }
@@ -181,4 +196,34 @@ class MainActivity : Activity() {
             requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), 100)
         }
     }
+
+    private fun requestRuntimePermissions() {
+        val needed = mutableListOf<String>()
+        if (checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) needed += Manifest.permission.CAMERA
+        if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) needed += Manifest.permission.RECORD_AUDIO
+        if (Build.VERSION.SDK_INT >= 33 && checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) needed += Manifest.permission.POST_NOTIFICATIONS
+        if (needed.isEmpty()) {
+            Toast.makeText(this, "Camera/microphone permissions already granted", Toast.LENGTH_SHORT).show()
+        } else {
+            requestPermissions(needed.toTypedArray(), 101)
+        }
+    }
+
+    private fun requestScreenshotConsent() {
+        val mgr = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+        startActivityForResult(mgr.createScreenCaptureIntent(), 200)
+    }
+
+    @Deprecated("Deprecated platform callback, still the compatible MediaProjection result path")
+    override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
+        super.onActivityResult(requestCode, resultCode, data)
+        if (requestCode == 200) {
+            if (resultCode == RESULT_OK && data != null) {
+                AndroidRuntimeGrants.setMediaProjection(resultCode, data)
+                Toast.makeText(this, "Screenshot consent granted for this app process", Toast.LENGTH_SHORT).show()
+            } else {
+                Toast.makeText(this, "Screenshot consent denied", Toast.LENGTH_SHORT).show()
+            }
+        }
+    }
 }

app/src/main/java/net/nexlab/hermesnodeandroid/capabilities/AndroidAudioControl.kt

@@ -5,10 +5,24 @@
  */
 package net.nexlab.hermesnodeandroid.capabilities
 
+import android.Manifest
 import android.content.Context
+import android.content.pm.PackageManager
+import android.media.AudioDeviceInfo
+import android.media.AudioFormat
 import android.media.AudioManager
+import android.media.AudioRecord
+import android.media.MediaRecorder
+import android.os.Build
+import androidx.core.content.ContextCompat
 import net.nexlab.hermesnodeandroid.CapabilityResult
+import org.json.JSONArray
 import org.json.JSONObject
+import java.io.ByteArrayOutputStream
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.util.Base64
+import kotlin.math.min
 
 class AndroidAudioControl(private val context: Context) {
     private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
@@ -16,15 +30,17 @@ class AndroidAudioControl(private val context: Context) {
     fun capabilityInfo(): JSONObject = JSONObject()
         .put("available", true)
         .put("can_get_status", true)
+        .put("can_list_devices", Build.VERSION.SDK_INT >= Build.VERSION_CODES.M)
         .put("can_play_audio", true)
-        .put("capture_input_requires_microphone_permission", true)
+        .put("can_capture_input", hasRecordAudioPermission())
+        .put("capture_input_requires_microphone_permission", !hasRecordAudioPermission())
         .put("capture_output_ready", false)
 
     fun handle(action: String?, params: JSONObject): CapabilityResult = when (action) {
-        "list_audio_devices" -> CapabilityResult.unsupported("list_audio_devices requires API-specific AudioDeviceInfo wiring")
+        "list_audio_devices" -> listAudioDevices()
         "get_audio_status" -> getAudioStatus()
-        "play_audio" -> CapabilityResult.unsupported("play_audio scaffolded; needs download/cache + MediaPlayer/ExoPlayer backend")
-        "capture_input" -> CapabilityResult.unsupported("capture_input requires RECORD_AUDIO permission and foreground UX")
+        "play_audio" -> CapabilityResult.unsupported("play_audio needs download/cache + MediaPlayer/ExoPlayer backend")
+        "capture_input" -> captureInput(params)
         "capture_output" -> CapabilityResult.unsupported("capture_output is not generally available to normal Android apps")
         else -> CapabilityResult.unsupported("Unknown audio_control action: $action")
     }
@@ -36,4 +52,98 @@ class AndroidAudioControl(private val context: Context) {
         "speakerphone_on" to audioManager.isSpeakerphoneOn,
         "microphone_mute" to audioManager.isMicrophoneMute,
     )
+
+    private fun listAudioDevices(): CapabilityResult {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return CapabilityResult.unsupported("AudioDeviceInfo requires Android M+")
+        val arr = JSONArray()
+        audioManager.getDevices(AudioManager.GET_DEVICES_ALL).forEach { device ->
+            arr.put(JSONObject()
+                .put("id", device.id)
+                .put("type", audioDeviceTypeName(device.type))
+                .put("is_source", device.isSource)
+                .put("is_sink", device.isSink)
+                .put("product_name", device.productName?.toString() ?: "")
+                .put("sample_rates", JSONArray(device.sampleRates.toList()))
+                .put("channel_counts", JSONArray(device.channelCounts.toList()))
+            )
+        }
+        return CapabilityResult.ok("devices" to arr, "count" to arr.length())
+    }
+
+    private fun captureInput(params: JSONObject): CapabilityResult {
+        if (!hasRecordAudioPermission()) return CapabilityResult.unsupported("capture_input requires RECORD_AUDIO permission; grant Microphone permission in Android settings/app prompt")
+        val durationMs = min(params.optLong("duration_ms", params.optLong("duration", 1000L)), 5000L)
+        val sampleRate = params.optInt("sample_rate", 16000)
+        val minBuffer = AudioRecord.getMinBufferSize(
+            sampleRate,
+            AudioFormat.CHANNEL_IN_MONO,
+            AudioFormat.ENCODING_PCM_16BIT,
+        )
+        if (minBuffer <= 0) return CapabilityResult.unsupported("AudioRecord does not support ${sampleRate}Hz mono PCM16 on this device")
+        val targetBytes = ((sampleRate * durationMs / 1000L) * 2L).toInt()
+        val bufferSize = maxOf(minBuffer * 2, 4096)
+        val audioRecord = AudioRecord(
+            MediaRecorder.AudioSource.MIC,
+            sampleRate,
+            AudioFormat.CHANNEL_IN_MONO,
+            AudioFormat.ENCODING_PCM_16BIT,
+            bufferSize,
+        )
+        return try {
+            audioRecord.startRecording()
+            val out = ByteArrayOutputStream(targetBytes)
+            val buf = ByteArray(bufferSize)
+            while (out.size() < targetBytes) {
+                val n = audioRecord.read(buf, 0, min(buf.size, targetBytes - out.size()))
+                if (n <= 0) break
+                out.write(buf, 0, n)
+            }
+            audioRecord.stop()
+            val pcm = out.toByteArray()
+            val wav = wavWrap(pcm, sampleRate)
+            CapabilityResult.ok(
+                "format" to "wav",
+                "sample_rate" to sampleRate,
+                "channels" to 1,
+                "duration_ms" to durationMs,
+                "encoding" to "base64",
+                "data" to Base64.getEncoder().encodeToString(wav),
+                "bytes" to wav.size,
+            )
+        } catch (e: SecurityException) {
+            CapabilityResult.unsupported("capture_input permission denied: ${e.message}")
+        } catch (e: Exception) {
+            CapabilityResult.unsupported("capture_input failed: ${e.message}")
+        } finally {
+            runCatching { audioRecord.release() }
+        }
+    }
+
+    private fun hasRecordAudioPermission(): Boolean =
+        ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
+
+    private fun wavWrap(pcm: ByteArray, sampleRate: Int): ByteArray {
+        val header = ByteArrayOutputStream(44 + pcm.size)
+        fun ascii(s: String) = header.write(s.toByteArray(Charsets.US_ASCII))
+        fun le32(v: Int) = header.write(ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(v).array())
+        fun le16(v: Int) = header.write(ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN).putShort(v.toShort()).array())
+        ascii("RIFF"); le32(36 + pcm.size); ascii("WAVE")
+        ascii("fmt "); le32(16); le16(1); le16(1); le32(sampleRate); le32(sampleRate * 2); le16(2); le16(16)
+        ascii("data"); le32(pcm.size); header.write(pcm)
+        return header.toByteArray()
+    }
+
+    private fun audioDeviceTypeName(type: Int): String = when (type) {
+        AudioDeviceInfo.TYPE_BUILTIN_EARPIECE -> "builtin_earpiece"
+        AudioDeviceInfo.TYPE_BUILTIN_SPEAKER -> "builtin_speaker"
+        AudioDeviceInfo.TYPE_WIRED_HEADSET -> "wired_headset"
+        AudioDeviceInfo.TYPE_WIRED_HEADPHONES -> "wired_headphones"
+        AudioDeviceInfo.TYPE_BLUETOOTH_SCO -> "bluetooth_sco"
+        AudioDeviceInfo.TYPE_BLUETOOTH_A2DP -> "bluetooth_a2dp"
+        AudioDeviceInfo.TYPE_BUILTIN_MIC -> "builtin_mic"
+        AudioDeviceInfo.TYPE_USB_DEVICE -> "usb_device"
+        AudioDeviceInfo.TYPE_USB_HEADSET -> "usb_headset"
+        AudioDeviceInfo.TYPE_HDMI -> "hdmi"
+        else -> "type_$type"
+    }
 }

app/src/main/java/net/nexlab/hermesnodeandroid/capabilities/AndroidCameraControl.kt

@@ -5,26 +5,40 @@
  */
 package net.nexlab.hermesnodeandroid.capabilities
 
+import android.Manifest
 import android.content.Context
+import android.content.pm.PackageManager
+import android.graphics.ImageFormat
+import android.hardware.camera2.CameraAccessException
+import android.hardware.camera2.CameraCaptureSession
 import android.hardware.camera2.CameraCharacteristics
+import android.hardware.camera2.CameraDevice
 import android.hardware.camera2.CameraManager
+import android.media.ImageReader
+import android.os.Handler
+import android.os.HandlerThread
+import androidx.core.content.ContextCompat
 import net.nexlab.hermesnodeandroid.CapabilityResult
 import org.json.JSONArray
 import org.json.JSONObject
+import java.util.Base64
+import java.util.concurrent.ArrayBlockingQueue
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
 
 class AndroidCameraControl(private val context: Context) {
     fun capabilityInfo(): JSONObject = JSONObject()
         .put("available", true)
-        .put("can_capture", true)
+        .put("can_capture", hasCameraPermission())
         .put("can_inject", false)
-        .put("capture_requires_camera_permission", true)
-        .put("backend", "Camera2/CameraX planned")
+        .put("capture_requires_camera_permission", !hasCameraPermission())
+        .put("backend", "Camera2")
 
     fun handle(action: String?, params: JSONObject): CapabilityResult = when (action) {
         "list_cameras" -> listCameras()
-        "get_camera_status" -> CapabilityResult.ok("available" to true, "backend" to "Camera2")
-        "capture_frame" -> CapabilityResult.unsupported("capture_frame requires CAMERA permission and CameraX/Camera2 capture session")
-        "capture_video" -> CapabilityResult.unsupported("capture_video requires CAMERA/RECORD_AUDIO permissions and visible UX")
+        "get_camera_status" -> CapabilityResult.ok("available" to true, "backend" to "Camera2", "can_capture" to hasCameraPermission())
+        "capture_frame" -> captureFrame(params)
+        "capture_video" -> CapabilityResult.unsupported("capture_video requires a longer visible foreground UX; frame capture is implemented first")
         "inject_video" -> CapabilityResult.unsupported("inject_video is not a normal Android capability without rooted/vendor-specific backend")
         else -> CapabilityResult.unsupported("Unknown camera_control action: $action")
     }
@@ -40,8 +54,82 @@ class AndroidCameraControl(private val context: Context) {
                 CameraCharacteristics.LENS_FACING_EXTERNAL -> "external"
                 else -> "unknown"
             }
-            arr.put(JSONObject().put("id", id).put("facing", facing))
+            val sizes = chars.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
+                ?.getOutputSizes(ImageFormat.JPEG)
+                ?.take(10)
+                ?.map { JSONObject().put("width", it.width).put("height", it.height) }
+                ?: emptyList()
+            arr.put(JSONObject().put("id", id).put("facing", facing).put("jpeg_sizes", JSONArray(sizes)))
         }
         return CapabilityResult.ok("cameras" to arr, "count" to arr.length())
     }
+
+    private fun captureFrame(params: JSONObject): CapabilityResult {
+        if (!hasCameraPermission()) return CapabilityResult.unsupported("capture_frame requires CAMERA permission; grant Camera permission in Android settings/app prompt")
+        val manager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
+        val cameraId = params.optString("camera_id", params.optString("id", "0"))
+        val width = params.optInt("width", 640)
+        val height = params.optInt("height", 480)
+        val thread = HandlerThread("HermesCameraCapture").apply { start() }
+        val handler = Handler(thread.looper)
+        val imageReader = ImageReader.newInstance(width, height, ImageFormat.JPEG, 1)
+        val jpegQueue = ArrayBlockingQueue<ByteArray>(1)
+        imageReader.setOnImageAvailableListener({ reader ->
+            val image = reader.acquireLatestImage() ?: return@setOnImageAvailableListener
+            image.use {
+                val buffer = it.planes[0].buffer
+                val bytes = ByteArray(buffer.remaining())
+                buffer.get(bytes)
+                jpegQueue.offer(bytes)
+            }
+        }, handler)
+
+        var camera: CameraDevice? = null
+        var session: CameraCaptureSession? = null
+        return try {
+            val opened = CountDownLatch(1)
+            manager.openCamera(cameraId, object : CameraDevice.StateCallback() {
+                override fun onOpened(device: CameraDevice) { camera = device; opened.countDown() }
+                override fun onDisconnected(device: CameraDevice) { device.close(); opened.countDown() }
+                override fun onError(device: CameraDevice, error: Int) { device.close(); opened.countDown() }
+            }, handler)
+            if (!opened.await(3, TimeUnit.SECONDS) || camera == null) return CapabilityResult.unsupported("capture_frame timed out opening camera $cameraId")
+
+            val configured = CountDownLatch(1)
+            camera!!.createCaptureSession(listOf(imageReader.surface), object : CameraCaptureSession.StateCallback() {
+                override fun onConfigured(captureSession: CameraCaptureSession) { session = captureSession; configured.countDown() }
+                override fun onConfigureFailed(captureSession: CameraCaptureSession) { configured.countDown() }
+            }, handler)
+            if (!configured.await(3, TimeUnit.SECONDS) || session == null) return CapabilityResult.unsupported("capture_frame timed out configuring camera $cameraId")
+
+            val request = camera!!.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE).apply {
+                addTarget(imageReader.surface)
+            }.build()
+            session!!.capture(request, null, handler)
+            val jpeg = jpegQueue.poll(5, TimeUnit.SECONDS) ?: return CapabilityResult.unsupported("capture_frame timed out waiting for JPEG")
+            CapabilityResult.ok(
+                "camera_id" to cameraId,
+                "format" to "jpg",
+                "encoding" to "base64",
+                "width" to width,
+                "height" to height,
+                "data" to Base64.getEncoder().encodeToString(jpeg),
+                "bytes" to jpeg.size,
+            )
+        } catch (e: SecurityException) {
+            CapabilityResult.unsupported("capture_frame permission denied: ${e.message}")
+        } catch (e: CameraAccessException) {
+            CapabilityResult.unsupported("capture_frame camera access failed: ${e.message}")
+        } catch (e: Exception) {
+            CapabilityResult.unsupported("capture_frame failed: ${e.message}")
+        } finally {
+            runCatching { session?.close() }
+            runCatching { camera?.close() }
+            runCatching { imageReader.close() }
+            thread.quitSafely()
+        }
+    }
+
+    private fun hasCameraPermission(): Boolean =
+        ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED
 }

app/src/main/java/net/nexlab/hermesnodeandroid/capabilities/AndroidComputerControl.kt

@@ -5,11 +5,13 @@
  */
 package net.nexlab.hermesnodeandroid.capabilities
 
+import android.accessibilityservice.AccessibilityService
 import android.content.Context
 import android.content.Intent
 import android.net.Uri
 import android.view.KeyEvent
 import net.nexlab.hermesnodeandroid.CapabilityResult
+import net.nexlab.hermesnodeandroid.HermesAccessibilityService
 import org.json.JSONObject
 
 class AndroidComputerControl(private val context: Context) {
@@ -21,18 +23,22 @@ class AndroidComputerControl(private val context: Context) {
             "tap", "swipe", "long_press", "type_text", "get_active_window",
             "screenshot", "mouse_move", "mouse_click", "mouse_position"
         ).joinToString(","))
-        .put("requires_accessibility_for_input", true)
-        .put("notes", "Reuses gateway computer_control name for Android-native phone control")
+        .put("accessibility_available", HermesAccessibilityService.current() != null)
+        .put("requires_accessibility_for_input", HermesAccessibilityService.current() == null)
+        .put("notes", "Uses Android intents plus AccessibilityService gestures when explicitly enabled by the user")
 
     fun handle(action: String?, params: JSONObject): CapabilityResult = when (action) {
         "launch_app" -> launchApp(params.optString("package"))
         "open_url" -> openUrl(params.optString("url"))
         "open_intent" -> openIntent(params)
         "key_press" -> keyPress(params.optString("key"))
-        "tap", "mouse_click", "mouse_move", "swipe", "long_press", "type_text" ->
-            CapabilityResult.unsupported("$action requires AccessibilityService gesture/input backend; scaffolded but not wired yet")
+        "tap", "mouse_click" -> tap(params)
+        "mouse_move" -> CapabilityResult.ok("note" to "Android has no persistent pointer; mouse_move is treated as a no-op")
+        "swipe" -> swipe(params)
+        "long_press" -> longPress(params)
+        "type_text" -> typeText(params.optString("text"))
         "get_active_window", "screenshot", "mouse_position" ->
-            CapabilityResult.unsupported("$action belongs to AndroidDesktopObserve backend; route via desktop_observe or wire delegation later")
+            CapabilityResult.unsupported("$action belongs to AndroidDesktopObserve backend; route via desktop_observe")
         else -> CapabilityResult.unsupported("Unknown computer_control action: $action")
     }
 
@@ -62,14 +68,55 @@ class AndroidComputerControl(private val context: Context) {
     }
 
     private fun keyPress(key: String): CapabilityResult {
-        val code = when (key.lowercase()) {
-            "back" -> KeyEvent.KEYCODE_BACK
-            "home" -> KeyEvent.KEYCODE_HOME
-            "enter" -> KeyEvent.KEYCODE_ENTER
-            "volume_up" -> KeyEvent.KEYCODE_VOLUME_UP
-            "volume_down" -> KeyEvent.KEYCODE_VOLUME_DOWN
+        val service = accessibility() ?: return CapabilityResult.unsupported("key_press requires enabling Hermes Node Accessibility Service")
+        val action = when (key.lowercase()) {
+            "back" -> AccessibilityService.GLOBAL_ACTION_BACK
+            "home" -> AccessibilityService.GLOBAL_ACTION_HOME
+            "recents", "overview" -> AccessibilityService.GLOBAL_ACTION_RECENTS
+            "notifications" -> AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS
+            "quick_settings" -> AccessibilityService.GLOBAL_ACTION_QUICK_SETTINGS
+            "power_dialog" -> AccessibilityService.GLOBAL_ACTION_POWER_DIALOG
+            "enter" -> return CapabilityResult.unsupported("enter key injection is not available through public Accessibility global actions")
+            "volume_up" -> return CapabilityResult.unsupported("volume_up requires privileged input injection; use audio_control/playback APIs instead")
+            "volume_down" -> return CapabilityResult.unsupported("volume_down requires privileged input injection; use audio_control/playback APIs instead")
             else -> return CapabilityResult.unsupported("Unsupported Android key: $key")
         }
-        return CapabilityResult.unsupported("key_press($code) requires AccessibilityService or privileged input injection")
+        return CapabilityResult.ok("key" to key, "performed" to service.globalAction(action))
     }
+
+    private fun tap(params: JSONObject): CapabilityResult {
+        val service = accessibility() ?: return CapabilityResult.unsupported("tap requires enabling Hermes Node Accessibility Service")
+        val x = params.optInt("x", params.optInt("screen_x", -1))
+        val y = params.optInt("y", params.optInt("screen_y", -1))
+        if (x < 0 || y < 0) return CapabilityResult.unsupported("tap requires params.x and params.y")
+        return CapabilityResult.ok("x" to x, "y" to y, "performed" to service.click(x, y))
+    }
+
+    private fun swipe(params: JSONObject): CapabilityResult {
+        val service = accessibility() ?: return CapabilityResult.unsupported("swipe requires enabling Hermes Node Accessibility Service")
+        val x1 = params.optInt("x1", -1)
+        val y1 = params.optInt("y1", -1)
+        val x2 = params.optInt("x2", -1)
+        val y2 = params.optInt("y2", -1)
+        val duration = params.optLong("duration_ms", 300L)
+        if (listOf(x1, y1, x2, y2).any { it < 0 }) return CapabilityResult.unsupported("swipe requires x1,y1,x2,y2")
+        return CapabilityResult.ok("performed" to service.swipe(x1, y1, x2, y2, duration), "duration_ms" to duration)
+    }
+
+    private fun longPress(params: JSONObject): CapabilityResult {
+        val service = accessibility() ?: return CapabilityResult.unsupported("long_press requires enabling Hermes Node Accessibility Service")
+        val x = params.optInt("x", -1)
+        val y = params.optInt("y", -1)
+        val duration = params.optLong("duration_ms", 700L)
+        if (x < 0 || y < 0) return CapabilityResult.unsupported("long_press requires params.x and params.y")
+        return CapabilityResult.ok("x" to x, "y" to y, "performed" to service.longPress(x, y, duration))
+    }
+
+    private fun typeText(text: String): CapabilityResult {
+        val service = accessibility() ?: return CapabilityResult.unsupported("type_text requires enabling Hermes Node Accessibility Service")
+        if (text.isEmpty()) return CapabilityResult.unsupported("type_text requires params.text")
+        return CapabilityResult.ok("performed" to service.typeText(text), "length" to text.length)
+    }
+
+    private fun accessibility(): HermesAccessibilityService? = HermesAccessibilityService.current()
 }

app/src/main/java/net/nexlab/hermesnodeandroid/capabilities/AndroidDesktopObserve.kt

@@ -5,27 +5,37 @@
  */
 package net.nexlab.hermesnodeandroid.capabilities
 
+import android.content.ClipData
+import android.content.ClipboardManager
 import android.content.Context
 import android.content.res.Configuration
 import android.os.PowerManager
 import android.util.DisplayMetrics
-import android.view.WindowManager
+import net.nexlab.hermesnodeandroid.AndroidRuntimeGrants
 import net.nexlab.hermesnodeandroid.CapabilityResult
+import net.nexlab.hermesnodeandroid.HermesAccessibilityService
 import org.json.JSONObject
 
 class AndroidDesktopObserve(private val context: Context) {
     fun capabilityInfo(): JSONObject = JSONObject()
         .put("available", true)
         .put("screen_info", true)
-        .put("active_window_requires_usage_stats_or_accessibility", true)
-        .put("screenshot_requires_media_projection", true)
+        .put("active_window_available", HermesAccessibilityService.current() != null)
+        .put("active_window_requires_accessibility", HermesAccessibilityService.current() == null)
+        .put("screenshot_available", AndroidRuntimeGrants.hasMediaProjection())
+        .put("screenshot_requires_media_projection", !AndroidRuntimeGrants.hasMediaProjection())
+        .put("clipboard_get", true)
 
     fun handle(action: String?, params: JSONObject): CapabilityResult = when (action) {
         "screen_info" -> screenInfo()
-        "active_window", "get_active_window" -> CapabilityResult.unsupported("active_window requires UsageStats or AccessibilityService grant")
-        "screenshot", "region_screenshot" -> CapabilityResult.unsupported("screenshot requires MediaProjection consent flow")
-        "clipboard_get" -> CapabilityResult.unsupported("clipboard_get is Android-version/foreground constrained; not wired yet")
+        "active_window", "get_active_window" -> activeWindow()
+        "screenshot", "region_screenshot" -> screenshotStatus()
+        "clipboard_get" -> clipboardGet()
         "cursor_position", "mouse_position" -> CapabilityResult.ok("x" to 0, "y" to 0, "note" to "Android has touch focus, not a persistent mouse cursor")
+        "list_windows" -> activeWindow().let { res ->
+            if (!res.success) res else CapabilityResult.ok("windows" to org.json.JSONArray().put(res.data), "count" to 1)
+        }
+        "window_geometry" -> screenInfo()
         else -> CapabilityResult.unsupported("Unknown desktop_observe action: $action")
     }
 
@@ -46,4 +56,24 @@ class AndroidDesktopObserve(private val context: Context) {
             "interactive" to pm.isInteractive,
         )
     }
+
+    private fun activeWindow(): CapabilityResult {
+        val service = HermesAccessibilityService.current()
+            ?: return CapabilityResult.unsupported("active_window requires enabling Hermes Node Accessibility Service")
+        return CapabilityResult.ok(*service.activeWindowInfo().map { it.key to it.value }.toTypedArray())
+    }
+
+    private fun screenshotStatus(): CapabilityResult {
+        if (!AndroidRuntimeGrants.hasMediaProjection()) {
+            return CapabilityResult.unsupported("screenshot requires MediaProjection consent; tap 'Grant screenshot consent' in the Android app")
+        }
+        return CapabilityResult.unsupported("MediaProjection grant present, but bitmap capture pipeline is not exposed over the gateway yet")
+    }
+
+    private fun clipboardGet(): CapabilityResult {
+        val clipboard = context.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
+        val clip: ClipData? = clipboard.primaryClip
+        val text = if (clip != null && clip.itemCount > 0) clip.getItemAt(0).coerceToText(context)?.toString() else ""
+        return CapabilityResult.ok("text" to (text ?: ""), "available" to (clip != null))
+    }
 }

app/src/main/res/values/strings.xml

 <resources>
     <string name="app_name">Hermes Node</string>
+    <string name="accessibility_service_description">Allows Hermes Node to observe the foreground app and perform explicit remote gestures.</string>
 </resources>

app/src/main/res/xml/hermes_accessibility_service.xml

+<?xml version="1.0" encoding="utf-8"?>
+<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
+    android:accessibilityEventTypes="typeAllMask"
+    android:accessibilityFeedbackType="feedbackGeneric"
+    android:accessibilityFlags="flagReportViewIds|flagRetrieveInteractiveWindows"
+    android:canPerformGestures="true"
+    android:canRetrieveWindowContent="true"
+    android:description="@string/accessibility_service_description"
+    android:notificationTimeout="50" />

gradle.properties

+android.useAndroidX=true
+android.nonTransitiveRClass=true