small fixes 27 (#404)

* fix deleting words with apostrophes and filtering in general

* fixed typing non-existing Ukrainian and Hebrew words not working after filtering is applied

* fixed the suggestions sometimes appearing in the wrong order, when some of them are generated

* fixed lowercase being incorrectly forced sometimes

* fixed attempting to add a word while still typing it, and the suggestions are visible, causing the word to be erased

src/io/github/sspanak/tt9/ime/TraditionalT9.java

@@ -357,7 +357,7 @@ public class TraditionalT9 extends KeyPadHandler {
 		}
 
 		cancelAutoAccept();
-		clearSuggestions();
+		acceptIncompleteSuggestion();
 
 		String word = textField.getSurroundingWord(mLanguage);
 		if (word.isEmpty()) {
@@ -421,10 +421,10 @@ public class TraditionalT9 extends KeyPadHandler {
 			filter = getComposingText();
 		}
 
-		if (mInputMode.setWordStem(filter, repeat)) {
-			mInputMode.loadSuggestions(this::getSuggestions, filter);
-		} else if (filter.length() == 0) {
+		if (filter.isEmpty()) {
 			mInputMode.reset();
+		} else if (mInputMode.setWordStem(filter, repeat)) {
+			mInputMode.loadSuggestions(this::getSuggestions, filter);
 		}
 
 		return true;

src/io/github/sspanak/tt9/ime/helpers/TextField.java

@@ -185,8 +185,8 @@ public class TextField {
 		Matcher before;
 		Matcher after;
 
-		if (language != null && language.isUkrainian()) {
-			// Ukrainian uses apostrophes as letters
+		if (language != null && (language.isHebrew() || language.isUkrainian())) {
+			// Hebrew and Ukrainian use apostrophes as letters
 			before = beforeCursorUkrainianRegex.matcher(getTextBeforeCursor());
 			after = afterCursorUkrainianRegex.matcher(getTextAfterCursor());
 		} else {

src/io/github/sspanak/tt9/ime/modes/InputMode.java

@@ -138,7 +138,7 @@ abstract public class InputMode {
 
 	// Stem filtering.
 	// Where applicable, return "true" if the mode supports it and the operation was possible.
-	public boolean clearWordStem() { return false; }
+	public boolean clearWordStem() { return setWordStem("", true); }
 	public boolean isStemFilterFuzzy() { return false; }
 	public String getWordStem() { return ""; }
 	public boolean setWordStem(String stem, boolean exact) { return false; }

src/io/github/sspanak/tt9/ime/modes/ModePredictive.java

@@ -16,6 +16,8 @@ import io.github.sspanak.tt9.languages.Language;
 import io.github.sspanak.tt9.preferences.SettingsStore;
 
 public class ModePredictive extends InputMode {
+	private final String LOG_TAG = getClass().getSimpleName();
+
 	private final SettingsStore settings;
 
 	public int getId() { return MODE_PREDICTIVE; }
@@ -35,6 +37,7 @@ public class ModePredictive extends InputMode {
 	private final AutoSpace autoSpace;
 	private final AutoTextCase autoTextCase;
 	private final Predictions predictions;
+	private boolean isCursorDirectionForward = false;
 
 
 	ModePredictive(SettingsStore settings, Language lang) {
@@ -51,6 +54,8 @@ public class ModePredictive extends InputMode {
 
 	@Override
 	public boolean onBackspace() {
+		isCursorDirectionForward = false;
+
 		if (digitSequence.length() < 1) {
 			clearWordStem();
 			return false;
@@ -60,7 +65,7 @@ public class ModePredictive extends InputMode {
 		if (digitSequence.length() == 0) {
 			clearWordStem();
 		} else if (stem.length() > digitSequence.length()) {
-			stem = stem.substring(0, digitSequence.length() - 1);
+			stem = stem.substring(0, digitSequence.length());
 		}
 
 		return true;
@@ -69,6 +74,8 @@ public class ModePredictive extends InputMode {
 
 	@Override
 	public boolean onNumber(int number, boolean hold, int repeat) {
+		isCursorDirectionForward = true;
+
 		if (hold) {
 			// hold to type any digit
 			reset();
@@ -136,23 +143,6 @@ public class ModePredictive extends InputMode {
 	}
 
 
-	/**
-	 * clearWordStem
-	 * Do not filter the suggestions by the word set using "setWordStem()", use only the digit sequence.
-	 */
-	@Override
-	public boolean clearWordStem() {
-		if (stem.length() == 0) {
-			return false;
-		}
-
-		stem = "";
-		Logger.d("setWordStem", "Stem filter cleared");
-
-		return true;
-	}
-
-
 	/**
 	 * setWordStem
 	 * Filter the possible suggestions by the given stem.
@@ -165,30 +155,30 @@ public class ModePredictive extends InputMode {
 	 * added to the suggestions list, even if they make no sense.
 	 * For example: "exac_" -> "exac", "exact", "exacu", "exacv", {database suggestions...}
 	 *
-	 *
 	 * Note that you need to manually get the suggestions again to obtain a filtered list.
 	 */
 	@Override
 	public boolean setWordStem(String newStem, boolean exact) {
-		String sanitizedStem = TextTools.removeNonLetters(newStem);
-		if (language == null || sanitizedStem == null || sanitizedStem.length() < 1) {
-			return false;
+		if (newStem == null || newStem.isEmpty()) {
+			isStemFuzzy = false;
+			stem = "";
+
+			Logger.d(LOG_TAG, "Stem filter cleared");
+			return true;
 		}
 
 		try {
-			// digitSequence = "the raw input", so that everything the user typed is preserved visually
-			// stem = "the sanitized input", because filtering by anything that is not a letter makes no sense
 			digitSequence = language.getDigitSequenceForWord(newStem);
-			stem = sanitizedStem.toLowerCase(language.getLocale());
 			isStemFuzzy = !exact;
+			stem = newStem.toLowerCase(language.getLocale());
 
-			Logger.d("setWordStem", "Stem is now: " + stem + (isStemFuzzy ? " (fuzzy)" : ""));
+			Logger.d(LOG_TAG, "Stem is now: " + stem + (isStemFuzzy ? " (fuzzy)" : ""));
 			return true;
 		} catch (Exception e) {
 			isStemFuzzy = false;
 			stem = "";
 
-			Logger.w("setWordStem", "Ignoring invalid stem: " + newStem + ". " + e.getMessage());
+			Logger.w("setWordStem", "Ignoring invalid stem: " + newStem + " in language: " + language + ". " + e.getMessage());
 			return false;
 		}
 	}
@@ -270,7 +260,7 @@ public class ModePredictive extends InputMode {
 		stem = "";
 
 		if (currentWord.isEmpty()) {
-			Logger.i("acceptCurrentSuggestion", "Current word is empty. Nothing to accept.");
+			Logger.i(LOG_TAG, "Current word is empty. Nothing to accept.");
 			return;
 		}
 
@@ -284,7 +274,7 @@ public class ModePredictive extends InputMode {
 				DictionaryDb.incrementWordFrequency(language, currentWord, sequence);
 			}
 		} catch (Exception e) {
-			Logger.e("ModePredictive", "Failed incrementing priority of word: '" + currentWord + "'. " + e.getMessage());
+			Logger.e(LOG_TAG, "Failed incrementing priority of word: '" + currentWord + "'. " + e.getMessage());
 		}
 	}
 
@@ -314,8 +304,8 @@ public class ModePredictive extends InputMode {
 
 	/**
 	 * shouldAcceptPreviousSuggestion
-	 * In this mode, In addition to confirming the suggestion in the input field,
-	 * we also increase its' priority. This function determines whether we want to do all this or not.
+	 * Automatic space assistance. Spaces (and special chars) cause suggestions to be accepted
+	 * automatically. This is used for analysis before processing the incoming pressed key.
 	 */
 	@Override
 	public boolean shouldAcceptPreviousSuggestion(int nextKey) {
@@ -333,14 +323,29 @@ public class ModePredictive extends InputMode {
 	 */
 	@Override
 	public boolean shouldAcceptPreviousSuggestion() {
+		// backspace never breaks words
+		if (!isCursorDirectionForward) {
+			return false;
+		}
+
+		// special characters always break words
+		if (autoAcceptTimeout == 0 && !digitSequence.startsWith("0")) {
+			return true;
+		}
+
+		// allow apostrophes in the middle or at the end of Hebrew and Ukrainian words
+		if (language.isHebrew() || language.isUkrainian()) {
+			return
+				predictions.noDbWords()
+				&& digitSequence.equals("1");
+		}
+
+		// punctuation breaks words, unless there are database matches ('s, qu', по-, etc...)
 		return
-			(autoAcceptTimeout == 0 && !digitSequence.startsWith("0"))
-			|| (
-				!digitSequence.isEmpty()
-				&& !predictions.areThereDbWords()
-				&& digitSequence.contains("1")
-				&& TextTools.containsOtherThan1(digitSequence)
-			);
+			!digitSequence.isEmpty()
+			&& predictions.noDbWords()
+			&& digitSequence.contains("1")
+			&& TextTools.containsOtherThan1(digitSequence);
 	}
 
 

src/io/github/sspanak/tt9/ime/modes/helpers/AutoTextCase.java

@@ -29,9 +29,7 @@ public class AutoTextCase {
 			case InputMode.CASE_LOWER:
 				return word.toLowerCase(language.getLocale());
 			case InputMode.CASE_CAPITALIZE:
-				return language.isMixedCaseWord(word) ? word : language.capitalize(word);
-			case InputMode.CASE_DICTIONARY:
-				return language.isMixedCaseWord(word) ? word : word.toLowerCase(language.getLocale());
+				return language.isMixedCaseWord(word) || language.isUpperCaseWord(word) ? word : language.capitalize(word);
 			default:
 				return word;
 		}

src/io/github/sspanak/tt9/ime/modes/helpers/Predictions.java

@@ -24,7 +24,7 @@ public class Predictions {
 
 	// data
 	private boolean areThereDbWords = false;
-	private final ArrayList<String> words = new ArrayList<>();
+	private ArrayList<String> words = new ArrayList<>();
 
 	// punctuation/emoji
 	private final Pattern containsOnly1Regex = Pattern.compile("^1+$");
@@ -83,8 +83,8 @@ public class Predictions {
 		return words;
 	}
 
-	public boolean areThereDbWords() {
-		return areThereDbWords;
+	public boolean noDbWords() {
+		return !areThereDbWords;
 	}
 
 
@@ -212,20 +212,20 @@ public class Predictions {
 
 		if (dbWords.isEmpty() && !digitSequence.isEmpty()) {
 			emptyDbWarning.emitOnce(language);
-			dbWords = generatePossibleCompletions(inputWord);
 		}
 
 		words.clear();
 		suggestStem();
 		suggestMissingWords(generatePossibleStemVariations(dbWords));
-		suggestMissingWords(insertPunctuationCompletions(dbWords));
+		suggestMissingWords(dbWords.isEmpty() ? generateWordVariations(inputWord) : dbWords);
+		words = insertPunctuationCompletions(words);
 
 		onWordsChanged.run();
 	}
 
 
 	/**
-	 * generatePossibleCompletions
+	 * generateWordVariations
 	 * When there are no matching suggestions after the last key press, generate a list of possible
 	 * ones, so that the user can complete a missing word that is completely different from the ones
 	 * in the dictionary.
@@ -233,7 +233,7 @@ public class Predictions {
 	 * For example, if the word is "missin_" and the last pressed key is "4", the results would be:
 	 * | missing | missinh | missini |
 	 */
-	private ArrayList<String> generatePossibleCompletions(String baseWord) {
+	private ArrayList<String> generateWordVariations(String baseWord) {
 		ArrayList<String> generatedWords = new ArrayList<>();
 
 		// Make sure the displayed word and the digit sequence, we will be generating suggestions from,
@@ -257,7 +257,7 @@ public class Predictions {
 
 	/**
 	 * insertPunctuationCompletions
-	 * When given: "you'", for example, this also generates all other 1-key alternatives, like:
+	 * When given: "you'", for example, this inserts all other 1-key alternatives, like:
 	 * "you.", "you?", "you!" and so on. The generated words will be inserted after the direct
 	 * database matches and before the fuzzy matches, as if they were direct matches with low frequency.
 	 * This is to preserve the sorting by length and frequency.
@@ -278,7 +278,7 @@ public class Predictions {
 		}
 
 		// generated "exact matches"
-		for (String w : generatePossibleCompletions(dbWords.get(0))) {
+		for (String w : generateWordVariations(dbWords.get(0))) {
 			if (!dbWords.contains(w) && !dbWords.contains(w.toLowerCase(language.getLocale()))) {
 				complementedWords.add(w);
 			}
@@ -310,12 +310,9 @@ public class Predictions {
 	 */
 	private ArrayList<String> generatePossibleStemVariations(ArrayList<String> dbWords) {
 		ArrayList<String> variations = new ArrayList<>();
-		if (stem.isEmpty()) {
-			return variations;
-		}
 
-		if (isStemFuzzy && stem.length() == digitSequence.length() - 1) {
-			ArrayList<String> allPossibleVariations = generatePossibleCompletions(stem);
+		if (isStemFuzzy && !stem.isEmpty() && stem.length() == digitSequence.length() - 1) {
+			ArrayList<String> allPossibleVariations = generateWordVariations(stem);
 
 			// first add the known words, because it makes more sense to see them first
 			for (String variation : allPossibleVariations) {

src/io/github/sspanak/tt9/languages/Language.java

@@ -150,26 +150,21 @@ public class Language {
 	/**
 	 * isLatinBased
 	 * Returns "true" when the language is based on the Latin alphabet or "false" otherwise.
-	 * WARNING: This performs somewhat resource-intensive operations every time, so consider
-	 * caching the result.
 	 */
 	public boolean isLatinBased() {
-		ArrayList<String> letters = getKeyCharacters(2, false);
-		return letters.contains("a");
+		return getKeyCharacters(2, false).contains("a");
 	}
 
-	/**
-	 * isGreek
-	 * Similar to "isLatinBased()", this returns "true" when the language is based on the Greek alphabet.
-	 */
 	public boolean isGreek() {
-		ArrayList<String> letters = getKeyCharacters(2, false);
-		return letters.contains("α");
+		return getKeyCharacters(2, false).contains("α");
 	}
 
 	public boolean isUkrainian() {
-		ArrayList<String> letters = getKeyCharacters(4, false);
-		return letters.contains("ї");
+		return getKeyCharacters(3, false).contains("є");
+	}
+
+	public boolean isHebrew() {
+		return getKeyCharacters(3, false).contains("א");
 	}
 
 	/* ************ utility ************ */
@@ -230,11 +225,14 @@ public class Language {
 	}
 
 	public boolean isMixedCaseWord(String word) {
-		return word != null
-			&& (
-				(word.length() == 1 && word.toUpperCase(locale).equals(word))
-				|| (!word.toLowerCase(locale).equals(word) && !word.toUpperCase(locale).equals(word))
-			);
+		return
+			word != null
+			&& !word.toLowerCase(locale).equals(word)
+			&& !word.toUpperCase(locale).equals(word);
+	}
+
+	public boolean isUpperCaseWord(String word) {
+		return word != null && word.toUpperCase(locale).equals(word);
 	}
 
 	public ArrayList<String> getKeyCharacters(int key, boolean includeDigit) {