diff --git a/app/src/main/java/io/github/sspanak/tt9/ime/TypingHandler.java b/app/src/main/java/io/github/sspanak/tt9/ime/TypingHandler.java
index b024285d..fb42220b 100644
--- a/app/src/main/java/io/github/sspanak/tt9/ime/TypingHandler.java
+++ b/app/src/main/java/io/github/sspanak/tt9/ime/TypingHandler.java
@@ -319,9 +319,11 @@ public abstract class TypingHandler extends KeyPadHandler {
 	 * Since Japanese is unique with its 3 alphabets, we need to setup the input modes separately.
 	 */
 	private void determineJapaneseInputModes() {
-		allowedInputModes.add(InputMode.MODE_HIRAGANA);
-		allowedInputModes.add(InputMode.MODE_KATAKANA);
-		allowedInputModes.remove((Integer) InputMode.MODE_ABC);
+		if (allowedInputModes.contains(InputMode.MODE_ABC)) {
+			allowedInputModes.remove((Integer) InputMode.MODE_ABC);
+			allowedInputModes.add(InputMode.MODE_HIRAGANA);
+			allowedInputModes.add(InputMode.MODE_KATAKANA);
+		}
 		if (!settings.getPredictiveMode()) {
 			allowedInputModes.remove((Integer) InputMode.MODE_PREDICTIVE);
 		}