From 2ad2bef3040d154ba0163821ed357e3571046a45 Mon Sep 17 00:00:00 2001 From: sspanak Date: Mon, 10 Mar 2025 16:37:58 +0200 Subject: [PATCH] More robust voice input on Android 12+. It will now choose dynamically between offline and online mode and download the necessary files automatically, instead of just failing --- .../github/sspanak/tt9/ime/VoiceHandler.java | 10 +- .../voice/SpeechRecognizerSupportLegacy.java | 24 ++++ .../voice/SpeechRecognizerSupportModern.java | 91 ++++++++++++++++ .../tt9/ime/voice/VoiceInputError.java | 5 + .../sspanak/tt9/ime/voice/VoiceInputOps.java | 103 ++++++++++++++---- 5 files changed, 209 insertions(+), 24 deletions(-) create mode 100644 app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportLegacy.java create mode 100644 app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportModern.java diff --git a/app/src/main/java/io/github/sspanak/tt9/ime/VoiceHandler.java b/app/src/main/java/io/github/sspanak/tt9/ime/VoiceHandler.java index 4ac34334..fdca4f1f 100644 --- a/app/src/main/java/io/github/sspanak/tt9/ime/VoiceHandler.java +++ b/app/src/main/java/io/github/sspanak/tt9/ime/VoiceHandler.java @@ -75,7 +75,15 @@ abstract class VoiceHandler extends TypingHandler { private void onVoiceInputError(VoiceInputError error) { - if (error.isIrrelevantToUser()) { + if (error.isLanguageMissing()) { + if (voiceInputOps.downloadLanguage(mLanguage)) { + Logger.i(LOG_TAG, "Downloading voice input language '" + mLanguage.getName() + "'"); + resetStatus(); + } else { + Logger.e(LOG_TAG, "Could not start download for voice input language '" + mLanguage.getName() + "'"); + statusBar.setError(error.toString()); + } + } else if (error.isIrrelevantToUser()) { Logger.i(LOG_TAG, "Ignoring voice input. " + error.debugMessage); resetStatus(); // re-enable the function keys } else { diff --git a/app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportLegacy.java b/app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportLegacy.java new file mode 100644 index 00000000..21a73dd0 --- /dev/null +++ b/app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportLegacy.java @@ -0,0 +1,24 @@ +package io.github.sspanak.tt9.ime.voice; + +import android.content.Context; +import android.speech.SpeechRecognizer; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import io.github.sspanak.tt9.hacks.DeviceInfo; +import io.github.sspanak.tt9.languages.Language; + +public class SpeechRecognizerSupportLegacy { + final boolean isOnDeviceRecognitionAvailable; + final boolean isRecognitionAvailable; + + SpeechRecognizerSupportLegacy(@NonNull Context ims) { + isOnDeviceRecognitionAvailable = DeviceInfo.AT_LEAST_ANDROID_12 && SpeechRecognizer.isOnDeviceRecognitionAvailable(ims); + isRecognitionAvailable = SpeechRecognizer.isRecognitionAvailable(ims); + } + + SpeechRecognizerSupportLegacy setLanguage(Language l) { return this; } + void checkOfflineSupport(@NonNull Runnable onSupportChecked) { onSupportChecked.run(); } + boolean isLanguageSupportedOffline(@Nullable Language l) { return false; } +} diff --git a/app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportModern.java b/app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportModern.java new file mode 100644 index 00000000..c2d2b4b0 --- /dev/null +++ b/app/src/main/java/io/github/sspanak/tt9/ime/voice/SpeechRecognizerSupportModern.java @@ -0,0 +1,91 @@ +package io.github.sspanak.tt9.ime.voice; + +import android.content.Context; +import android.os.Build; +import android.speech.RecognitionSupport; +import android.speech.RecognitionSupportCallback; +import android.speech.SpeechRecognizer; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.Executors; + +import io.github.sspanak.tt9.hacks.DeviceInfo; +import io.github.sspanak.tt9.languages.Language; + +@RequiresApi(api = Build.VERSION_CODES.TIRAMISU) +class SpeechRecognizerSupportModern extends SpeechRecognizerSupportLegacy implements RecognitionSupportCallback { + private final HashSet missingOfflineLanguages = new HashSet<>(); + private final HashSet availableOfflineLanguages = new HashSet<>(); + + private final Context ims; + private String locale; + private Runnable onSupportChecked; + private SpeechRecognizer recognizer; + + + SpeechRecognizerSupportModern(@NonNull Context ims) { + super(ims); + this.ims = ims; + } + + + @Override + SpeechRecognizerSupportModern setLanguage(@Nullable Language language) { + locale = language == null ? null : VoiceInputOps.getLocale(language); + return this; + } + + + @Override + void checkOfflineSupport(@NonNull Runnable onSupportChecked) { + if ( + locale == null + || !DeviceInfo.AT_LEAST_ANDROID_13 + || !isOnDeviceRecognitionAvailable + || missingOfflineLanguages.contains(locale) + || availableOfflineLanguages.contains(locale) + ) { + onSupportChecked.run(); + return; + } + + this.onSupportChecked = onSupportChecked; + + recognizer = SpeechRecognizer.createOnDeviceSpeechRecognizer(ims); + recognizer.checkRecognitionSupport(VoiceInputOps.createIntent(locale), Executors.newSingleThreadExecutor(), this); + } + + + @Override + boolean isLanguageSupportedOffline(@Nullable Language language) { + return language != null && isOnDeviceRecognitionAvailable && availableOfflineLanguages.contains(VoiceInputOps.getLocale(language)); + } + + + public void onSupportResult(@NonNull RecognitionSupport recognitionSupport) { + recognizer.destroy(); + + List locales = recognitionSupport.getSupportedOnDeviceLanguages(); + if (locales.contains(locale)) { + availableOfflineLanguages.add(locale); + missingOfflineLanguages.remove(locale); + } else { + availableOfflineLanguages.remove(locale); + missingOfflineLanguages.add(locale); + } + + onSupportChecked.run(); + } + + + public void onError(int error) { + recognizer.destroy(); + missingOfflineLanguages.add(locale); + onSupportChecked.run(); + } +} diff --git a/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputError.java b/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputError.java index 3b0fb132..2c862df2 100644 --- a/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputError.java +++ b/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputError.java @@ -41,6 +41,11 @@ public class VoiceInputError { } + public boolean isLanguageMissing() { + return DeviceInfo.AT_LEAST_ANDROID_12 && code == SpeechRecognizer.ERROR_LANGUAGE_UNAVAILABLE; + } + + @NonNull @Override public String toString() { diff --git a/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputOps.java b/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputOps.java index 8bff2b2a..03d0c56c 100644 --- a/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputOps.java +++ b/app/src/main/java/io/github/sspanak/tt9/ime/voice/VoiceInputOps.java @@ -2,10 +2,13 @@ package io.github.sspanak.tt9.ime.voice; import android.content.Context; import android.content.Intent; +import android.os.Handler; +import android.os.Looper; import android.speech.RecognizerIntent; import android.speech.SpeechRecognizer; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import java.util.ArrayList; @@ -16,14 +19,13 @@ import io.github.sspanak.tt9.util.ConsumerCompat; import io.github.sspanak.tt9.util.Logger; public class VoiceInputOps { - private final boolean isOnDeviceRecognitionAvailable; - private final boolean isRecognitionAvailable; - + private final static String LOG_TAG = VoiceInputOps.class.getSimpleName(); private final Context ims; private Language language; - private SpeechRecognizer speechRecognizer; private final VoiceListener listener; + private final SpeechRecognizerSupportLegacy recognizerSupport; + private SpeechRecognizer speechRecognizer; private final ConsumerCompat onStopListening; private final ConsumerCompat onListeningError; @@ -35,9 +37,8 @@ public class VoiceInputOps { ConsumerCompat onStop, ConsumerCompat onError ) { - isOnDeviceRecognitionAvailable = DeviceInfo.AT_LEAST_ANDROID_12 && SpeechRecognizer.isOnDeviceRecognitionAvailable(ims); - isRecognitionAvailable = SpeechRecognizer.isRecognitionAvailable(ims); listener = new VoiceListener(ims, onStart, this::onStop, this::onError); + recognizerSupport = DeviceInfo.AT_LEAST_ANDROID_13 ? new SpeechRecognizerSupportModern(ims) : new SpeechRecognizerSupportLegacy(ims); onStopListening = onStop != null ? onStop : result -> {}; onListeningError = onError != null ? onError : error -> {}; @@ -46,12 +47,33 @@ public class VoiceInputOps { } - private void createRecognizer() { - if (DeviceInfo.AT_LEAST_ANDROID_12 && isOnDeviceRecognitionAvailable) { + static String getLocale(@NonNull Language lang) { + return lang.getLocale().toString().replace("_", "-"); + } + + + static Intent createIntent(@NonNull String locale) { + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, locale); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); + return intent; + } + + + static Intent createIntent(@NonNull Language language) { + return createIntent(getLocale(language)); + } + + + private void createRecognizer(@Nullable Language language) { + if (DeviceInfo.AT_LEAST_ANDROID_13 && recognizerSupport.isLanguageSupportedOffline(language)) { + Logger.d(LOG_TAG, "Creating on-device SpeechRecognizer..."); speechRecognizer = SpeechRecognizer.createOnDeviceSpeechRecognizer(ims); - } else if (isRecognitionAvailable) { + } else if (recognizerSupport.isRecognitionAvailable) { + Logger.d(LOG_TAG, "Creating online SpeechRecognizer..."); speechRecognizer = SpeechRecognizer.createSpeechRecognizer(ims); } else { + Logger.d(LOG_TAG, "Cannot create SpeechRecognizer, recognition not available."); return; } @@ -60,7 +82,7 @@ public class VoiceInputOps { public boolean isAvailable() { - return isRecognitionAvailable || isOnDeviceRecognitionAvailable; + return recognizerSupport.isRecognitionAvailable || recognizerSupport.isOnDeviceRecognitionAvailable; } @@ -69,7 +91,7 @@ public class VoiceInputOps { } - public void listen(Language language) { + public void listen(@Nullable Language language) { if (language == null) { onListeningError.accept(new VoiceInputError(ims, VoiceInputError.ERROR_INVALID_LANGUAGE)); return; @@ -85,20 +107,37 @@ public class VoiceInputOps { return; } - createRecognizer(); - this.language = language; - String locale = language.getLocale().toString().replace("_", "-"); + recognizerSupport.setLanguage(language).checkOfflineSupport(this::listenAsync); + } - Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); - intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, locale); - intent.putExtra(RecognizerIntent.EXTRA_PROMPT, toString()); + + private void listenAsync() { + Handler mainHandler = new Handler(Looper.getMainLooper()); + mainHandler.post(this::listen); + } + + + private void listen() { + if (!isAvailable()) { + onListeningError.accept(new VoiceInputError(ims, VoiceInputError.ERROR_NOT_AVAILABLE)); + return; + } + + if (isListening()) { + onListeningError.accept(new VoiceInputError(ims, SpeechRecognizer.ERROR_RECOGNIZER_BUSY)); + return; + } + + createRecognizer(language); + + String locale = getLocale(language); try { - speechRecognizer.startListening(intent); - Logger.d(getClass().getSimpleName(), "SpeechRecognizer started for locale: " + locale); + speechRecognizer.startListening(createIntent(locale)); + Logger.d(LOG_TAG, "SpeechRecognizer started for locale: " + locale); } catch (SecurityException e) { - Logger.e(getClass().getSimpleName(), "SpeechRecognizer start failed due to a SecurityException. " + e.getMessage()); + Logger.e(LOG_TAG, "SpeechRecognizer start failed due to a SecurityException. " + e.getMessage()); onError(new VoiceInputError(ims, VoiceInputError.ERROR_CANNOT_BIND_TO_VOICE_SERVICE)); } } @@ -122,19 +161,36 @@ public class VoiceInputOps { speechRecognizer.destroy(); } catch (IllegalArgumentException e) { if (i < 2) { - Logger.e(getClass().getSimpleName(), "SpeechRecognizer destroy failed. " + e.getMessage() + ". Retrying..."); + Logger.e(LOG_TAG, "SpeechRecognizer destroy failed. " + e.getMessage() + ". Retrying..."); continue; } else { - Logger.e(getClass().getSimpleName(), "SpeechRecognizer destroy failed. " + e.getMessage() + ". Giving up and just nulling the reference."); + Logger.e(LOG_TAG, "SpeechRecognizer destroy failed. " + e.getMessage() + ". Giving up and just nulling the reference."); } } speechRecognizer = null; - Logger.d(getClass().getSimpleName(), "SpeechRecognizer destroyed"); + Logger.d(LOG_TAG, "SpeechRecognizer destroyed"); } } + public boolean downloadLanguage(Language language) { + if (!DeviceInfo.AT_LEAST_ANDROID_13 || !recognizerSupport.isLanguageSupportedOffline(language) || isListening()) { + return false; + } + + createRecognizer(language); + if (speechRecognizer == null) { + return false; + } + + speechRecognizer.triggerModelDownload(createIntent(language)); + destroy(); + + return true; + } + + private void onStop(ArrayList results) { destroy(); onStopListening.accept(results.isEmpty() ? null : results.get(0)); @@ -146,6 +202,7 @@ public class VoiceInputOps { onListeningError.accept(error); } + @NonNull @Override public String toString() {