Easier contraction typing (#289)

* removed isPunctiationPartOfWords hack and created a new contraction input method that allows typing just anything, instead of predefined list coming from the dictionary

* updated the common compound words and contractions in Bulgarian, Dutch, English and French

* removed some non-sense and rarely used English words

* fixed crashing when trying to find words with apostrophes in the database

* fixed a crash when trying to capitalize single character strings

* improved dictionary validation at build time: spaces are now disallowed

src/io/github/sspanak/tt9/TextTools.java

@@ -0,0 +1,43 @@
+package io.github.sspanak.tt9;
+
+import java.util.regex.Pattern;
+
+public class TextTools {
+	private static final Pattern containsOtherThan1 = Pattern.compile("[02-9]");
+	private static final Pattern previousIsLetter = Pattern.compile("\\p{L}$");
+	private static final Pattern nextIsPunctuation = Pattern.compile("^\\p{Punct}");
+	private static final Pattern nextToWord = Pattern.compile("\\b$");
+	private static final Pattern startOfSentence = Pattern.compile("(?<!\\.)(^|[.?!¿¡])\\s*$");
+
+	public static boolean containsOtherThan1(String str) {
+		return str != null && containsOtherThan1.matcher(str).find();
+	}
+
+	public static boolean isNextToWord(String str) {
+		return str != null && nextToWord.matcher(str).find();
+	}
+
+	public static boolean isStartOfSentence(String str) {
+		return str != null && startOfSentence.matcher(str).find();
+	}
+
+	public static boolean nextIsPunctuation(String str) {
+		return str != null && nextIsPunctuation.matcher(str).find();
+	}
+
+	public static boolean previousIsLetter(String str) {
+		return str != null && previousIsLetter.matcher(str).find();
+	}
+
+	public static boolean startsWithWhitespace(String str) {
+		return str != null && !str.isEmpty() && (str.charAt(0) == ' ' || str.charAt(0) == '\n' || str.charAt(0) == '\t');
+	}
+
+	public static boolean startsWithNumber(String str) {
+		return str != null && !str.isEmpty() && (str.charAt(0) >= '0' && str.charAt(0) <= '9');
+	}
+
+	public static String removeNonLetters(String str) {
+		return str != null ? str.replaceAll("\\P{L}", "") : null;
+	}
+}

src/io/github/sspanak/tt9/db/DictionaryDb.java

@@ -191,24 +191,13 @@ public class DictionaryDb {
 
 				// In case the user has changed the text case, there would be no match.
 				// Try again with the lowercase equivalent.
-				String lowercaseWord = "";
 				if (affectedRows == 0) {
-					lowercaseWord = word.toLowerCase(language.getLocale());
+					String lowercaseWord = word.toLowerCase(language.getLocale());
 					affectedRows = getInstance().wordsDao().incrementFrequency(language.getId(), lowercaseWord, sequence);
 
 					Logger.d("incrementWordFrequency", "Attempting to increment frequency for lowercase variant: " + lowercaseWord);
 				}
 
-				// Some languages permit appending the punctuation to the end of the words, like so: "try,".
-				// But there are no such words in the dictionary, so try without the punctuation mark.
-				if (affectedRows == 0 && language.isPunctuationPartOfWords() && sequence.endsWith("1")) {
-					String truncatedWord = lowercaseWord.substring(0, word.length() - 1);
-					String truncatedSequence = sequence.substring(0, sequence.length() - 1);
-					affectedRows = getInstance().wordsDao().incrementFrequency(language.getId(), truncatedWord, truncatedSequence);
-
-					Logger.d("incrementWordFrequency", "Attempting to increment frequency with stripped punctuation: " + truncatedWord);
-				}
-
 				Logger.d("incrementWordFrequency", "Affected rows: " + affectedRows);
 			} catch (Exception e) {
 				Logger.e(

src/io/github/sspanak/tt9/db/migrations/DB11.java

@@ -0,0 +1,39 @@
+package io.github.sspanak.tt9.db.migrations;
+
+import androidx.annotation.NonNull;
+import androidx.room.migration.Migration;
+import androidx.sqlite.db.SupportSQLiteDatabase;
+
+import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.languages.definitions.Dutch;
+import io.github.sspanak.tt9.languages.definitions.English;
+
+public class DB11 {
+	public static final Migration MIGRATION = new Migration(10, 11) {
+		@Override
+		public void migrate(@NonNull SupportSQLiteDatabase database) {
+			final String enWords = "'I''d','I''m','d''annunzio','I''ll','I''ve','prud''hon','an''t','bo''s''n','bo''s''ns','bo''sun','bo''suns','bos''n','bos''ns','br''er','ca''canny','could''ve','d''arezzo','d''estaing','e''en','e''er','fo''c''s''le','fo''c''s''les','fo''c''sle','fo''c''sles','ha''penny','he''d','he''ll','how''d','how''re','howe''er','it''d','it''ll','might''ve','must''ve','n''importe','ne''er','nor''easter','nor''wester','o''er','rec''d','sec''y','she''d','she''ll','should''ve','sou''wester','ta''en','that''d','that''ll','they''d','they''ll','they''re','they''ve','we''d','we''ll','we''re','we''ve','whate''er','whatsoe''er','whene''er','where''er','who''d','who''ll','who''re','who''ve','why''d','would''ve','you''d','you''ll','you''re','you''ve','Ch''in','L''Amour','L''Enfant','L''Oreal','L''Ouverture','T''ang','Xi''an'";
+			final String nlWords = "'''s-Graveland','''s-Gravendeel','''s-Gravenhaags','''s-Gravenhage','''s-Gravenhagenaar','''s-Gravenmoer','''s-Gravenzande','''s-Gravenzander','''s-Gravenzands','''s-Hertogenbosch','''t','A.D.','az.','chin.','d.v.','h.k.h.','h.m.','l.b.','mgr.','n.b.','n.h.','n.n.','n.o.','n.v.','n.w.','ned.','o.l.v.','openoffice.org','r.i.p.','st.-eustatius','st.-maarten','stct.','w.','w.v.str.','z.h.','z.k.h.','a.d.h.v.','a.g.v.','a.h.w.','a.j.b.','a.m.','a.s.','a.u.b.','aanw.','afb.','afd.','afz.','an.','arr.','b.d.','b.g.g.','b.v.d.','bc.','bett.','bijl.','bijv.','blz.','bv.','bw.','c.q.','c.s.','ca.','d.d.','d.i.','d.m.v.','d.w.z.','dd.','dhr.','div.','dr.','dra.','drs.','drs.-titel','ds.','e.a.','e.d.','e.e.a.','e.o.','e.v.','e.v.a.','enz.','etc.','evt.','excl.','fa.','fam.','fig.','fr.','g.g.d.','geb.','gem.','get.','i.c.','i.c.m.','i.e.','i.h.a.','i.h.b.','i.m.','i.o.','i.o.v.','i.p.v.','i.s.m.','i.t.t.','i.v.m.','i.z.g.st.','incl.','ing.','ir.','jhr.','jkvr.','jl.','jr.','k.k.','lic.','m.','m.a.w.','m.b.t.','m.b.v.','m.i.','m.i.v.','m.m.','m.m.v.','m.n.','m.u.v.','max.','mevr.','min.','mld.','mln.','mr.','mw.','n.a.v.','n.o.t.k.','n.v.t.','nl.','nl.openoffice.org','no.','nr.','nrs.','o.a.','o.b.v.','o.i.','o.i.d.','o.m.','o.t.t.','o.v.t.','o.v.v.','ong.','p.','p.a.','p.m.','p.o.','p.p.','p.w.','pag.','plm.','plv.','prof.','q.e.d.','q.q.','r.-k.','red.','resp.','s.j.','s.v.p.','sr.','t.a.v.','t.b.v.','t.g.v.','t.h.t.','t.h.v.','t.n.v.','t.o.v.','t.w.','t.w.v.','t.z.t.','v.','v.chr.','v.d.','v.h.','v.l.n.r.','v.r.n.l.','v.t.t.','v.v.','v.v.t.','v.w.b.','verg.','vgl.','vnl.','vnw.','voorz.','vs.','w.o.','w.v.t.t.k.','ww.','z.g.','z.g.a.n.','z.i.','z.o.z.','z.s.m.','zgn.'";
+
+			try {
+				database.beginTransaction();
+				database.execSQL(getDeleteEnglishSwordsQuery());
+				database.execSQL(getDeleteWordsQuery(new English().getId(), enWords));
+				database.execSQL(getDeleteWordsQuery(new Dutch().getId(), nlWords));
+				database.setTransactionSuccessful();
+			} catch (Exception e) {
+				Logger.e("Migrate to DB11", "Migration failed. " + e.getMessage());
+			} finally {
+				database.endTransaction();
+			}
+		}
+	};
+
+	private static String getDeleteEnglishSwordsQuery() {
+		return "DELETE FROM words WHERE lang=" + new English().getId() + " AND word LIKE '%''s'";
+	}
+
+	private static String getDeleteWordsQuery(int langId, String wordList) {
+		return "DELETE FROM words WHERE lang=" + langId + " AND word IN(" + wordList + ")";
+	}
+}

src/io/github/sspanak/tt9/db/room/TT9Room.java

@@ -8,12 +8,13 @@ import androidx.room.RoomDatabase;
 import androidx.sqlite.db.SimpleSQLiteQuery;
 
 import io.github.sspanak.tt9.db.migrations.DB10;
+import io.github.sspanak.tt9.db.migrations.DB11;
 import io.github.sspanak.tt9.db.migrations.DB6;
 import io.github.sspanak.tt9.db.migrations.DB7;
 import io.github.sspanak.tt9.db.migrations.DB8;
 import io.github.sspanak.tt9.db.migrations.DB9;
 
-@Database(version = 10, entities = Word.class, exportSchema = false)
+@Database(version = 11, entities = Word.class, exportSchema = false)
 public abstract class TT9Room extends RoomDatabase {
 	public abstract WordsDao wordsDao();
 
@@ -25,7 +26,8 @@ public abstract class TT9Room extends RoomDatabase {
 				new DB7().getMigration(context),
 				DB8.MIGRATION,
 				DB9.MIGRATION,
-				DB10.MIGRATION
+				DB10.MIGRATION,
+				DB11.MIGRATION
 			)
 			.build();
 	}
@@ -41,7 +43,7 @@ public abstract class TT9Room extends RoomDatabase {
 			" LIMIT " + limit;
 
 		if (word != null) {
-			sql = sql.replace("WHERE 1", "WHERE 1 AND word LIKE '" + word + "%'");
+			sql = sql.replace("WHERE 1", "WHERE 1 AND word LIKE '" + word.replace("'", "''") + "%'");
 		}
 
 		return new SimpleSQLiteQuery(sql);

src/io/github/sspanak/tt9/db/room/WordsDao.java

@@ -18,7 +18,7 @@ public interface WordsDao {
 	@Query("SELECT COUNT(id) FROM words WHERE :langId < 0 OR lang = :langId")
 	int count(int langId);
 
-	@Query("DELETE FROM words WHERE LANG IN(:langIds)")
+	@Query("DELETE FROM words WHERE lang IN(:langIds)")
 	void deleteByLanguage(ArrayList<Integer> langIds);
 
 	@Query("SELECT COUNT(id) FROM words WHERE lang = :langId AND word = :word")

src/io/github/sspanak/tt9/ime/TraditionalT9.java

@@ -262,18 +262,16 @@ public class TraditionalT9 extends KeyPadHandler {
 		cancelAutoAccept();
 		forceShowWindowIfHidden();
 
-		String currentWord = getComposingText();
-
-		// Automatically accept the current word, when the next one is a space or punctuation,
+		// Automatically accept the previous word, when the next one is a space or punctuation,
 		// instead of requiring "OK" before that.
-		if (mInputMode.shouldAcceptCurrentSuggestion(key, hold, repeat > 0)) {
+		// First pass, analyze the incoming key press and decide whether it could be the start of
+		// a new word.
+		if (mInputMode.shouldAcceptPreviousSuggestion(key)) {
 			autoCorrectSpace(acceptIncompleteSuggestion(), false, key);
-			currentWord = "";
 		}
 
 		// Auto-adjust the text case before each word, if the InputMode supports it.
-		// We don't do it too often, because it is somewhat resource-intensive.
-		if (currentWord.length() == 0) {
+		if (getComposingText().isEmpty()) {
 			mInputMode.determineNextWordTextCase(textField.isThereText(), textField.getTextBeforeCursor());
 		}
 
@@ -543,11 +541,11 @@ public class TraditionalT9 extends KeyPadHandler {
 	}
 
 	private void commitCurrentSuggestion(boolean entireSuggestion) {
-		if (!isSuggestionViewHidden() && currentInputConnection != null) {
+		if (!isSuggestionViewHidden()) {
 			if (entireSuggestion) {
 				textField.setComposingText(suggestionBar.getCurrentSuggestion());
 			}
-			currentInputConnection.finishComposingText();
+			textField.finishComposingText();
 		}
 
 		setSuggestions(null);
@@ -556,11 +554,8 @@ public class TraditionalT9 extends KeyPadHandler {
 
 	private void clearSuggestions() {
 		setSuggestions(null);
-
-		if (currentInputConnection != null) {
-			textField.setComposingText("");
-			currentInputConnection.finishComposingText();
-		}
+		textField.setComposingText("");
+		textField.finishComposingText();
 	}
 
 
@@ -570,10 +565,22 @@ public class TraditionalT9 extends KeyPadHandler {
 
 
 	private void handleSuggestions() {
+		// Automatically accept the previous word, without requiring OK. This is similar to what
+		// Second pass, analyze the available suggestions and decide if combining them with the
+		// last key press makes up a compound word like: (it)'s, (I)'ve, l'(oiseau), or it is
+		// just the end of a sentence, like: "word." or "another?"
+		if (mInputMode.shouldAcceptPreviousSuggestion()) {
+			String lastComposingText = getComposingText(mInputMode.getSequenceLength() - 1);
+			commitCurrentSuggestion(false);
+			mInputMode.onAcceptSuggestion(lastComposingText, true);
+			autoCorrectSpace(lastComposingText, false, -1);
+			mInputMode.determineNextWordTextCase(textField.isThereText(), textField.getTextBeforeCursor());
+		}
+
 		// key code "suggestions" take priority over words
 		if (mInputMode.getKeyCode() > 0) {
 			sendDownUpKeyEvents(mInputMode.getKeyCode());
-			mInputMode.onAcceptSuggestion("");
+			mInputMode.reset();
 			return;
 		}
 
@@ -605,16 +612,27 @@ public class TraditionalT9 extends KeyPadHandler {
 	}
 
 
-	private String getComposingText() {
+	private String getComposingText(int maxLength) {
+		if (maxLength == 0) {
+			return "";
+		}
+
+		maxLength = maxLength > 0 ? Math.min(maxLength, mInputMode.getSequenceLength()) : mInputMode.getSequenceLength();
+
 		String text = suggestionBar.getCurrentSuggestion();
-		if (text.length() > 0 && text.length() > mInputMode.getSequenceLength()) {
-			text = text.substring(0, mInputMode.getSequenceLength());
+		if (text.length() > 0 && text.length() > maxLength) {
+			text = text.substring(0, maxLength);
 		}
 
 		return text;
 	}
 
 
+	private String getComposingText() {
+		return getComposingText(-1);
+	}
+
+
 	private void refreshComposingText() {
 		textField.setComposingText(getComposingText());
 	}
@@ -729,11 +747,11 @@ public class TraditionalT9 extends KeyPadHandler {
 
 
 	private void showAddWord() {
-		if (currentInputConnection == null) {
+		if (shouldBeOff()) {
 			return;
 		}
 
-		currentInputConnection.finishComposingText();
+		textField.finishComposingText();
 		clearSuggestions();
 
 		UI.showAddWordDialog(this, mLanguage.getId(), textField.getSurroundingWord());

src/io/github/sspanak/tt9/ime/modes/InputMode.java

@@ -56,7 +56,8 @@ abstract public class InputMode {
 	abstract public boolean onOtherKey(int key);
 
 	// Suggestions
-	public void onAcceptSuggestion(@NonNull String suggestion) {}
+	public void onAcceptSuggestion(@NonNull String word) { onAcceptSuggestion(word, false); }
+	public void onAcceptSuggestion(@NonNull String word, boolean preserveWordList) {}
 
 	/**
 	 * loadSuggestions
@@ -97,7 +98,8 @@ abstract public class InputMode {
 	}
 
 	// Interaction with the IME. Return "true" if it should perform the respective action.
-	public boolean shouldAcceptCurrentSuggestion(int key, boolean hold, boolean repeat) { return false; }
+	public boolean shouldAcceptPreviousSuggestion() { return false; }
+	public boolean shouldAcceptPreviousSuggestion(int nextKey) { return false; }
 	public boolean shouldAddAutoSpace(InputType inputType, TextField textField, boolean isWordAcceptedManually, int nextKey) { return false; }
 	public boolean shouldDeletePrecedingSpace(InputType inputType) { return false; }
 	public boolean shouldSelectNextSuggestion() { return false; }

src/io/github/sspanak/tt9/ime/modes/ModeABC.java

@@ -70,7 +70,7 @@ public class ModeABC extends InputMode {
 	@Override final public boolean isABC() { return true; }
 	@Override public int getSequenceLength() { return 1; }
 
-	@Override public boolean shouldAcceptCurrentSuggestion(int key, boolean hold, boolean repeat) { return hold || !repeat; }
+	@Override public boolean shouldAcceptPreviousSuggestion() { return autoAcceptTimeout == 0 || !shouldSelectNextLetter; }
 	@Override public boolean shouldTrackUpDown() { return true; }
 	@Override public boolean shouldTrackLeftRight() { return true; }
 	@Override public boolean shouldSelectNextSuggestion() {

src/io/github/sspanak/tt9/ime/modes/ModePredictive.java

@@ -2,7 +2,10 @@ package io.github.sspanak.tt9.ime.modes;
 
 import androidx.annotation.NonNull;
 
+import java.util.ArrayList;
+
 import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.TextTools;
 import io.github.sspanak.tt9.db.DictionaryDb;
 import io.github.sspanak.tt9.ime.helpers.InputType;
 import io.github.sspanak.tt9.ime.helpers.TextField;
@@ -19,7 +22,6 @@ public class ModePredictive extends InputMode {
 
 	private String digitSequence = "";
 	private String lastAcceptedWord = "";
-	private String lastAcceptedSequence = "";
 
 	// stem filter
 	private boolean isStemFuzzy = false;
@@ -123,6 +125,31 @@ public class ModePredictive extends InputMode {
 	}
 
 
+	/**
+	 * clearLastAcceptedWord
+	 * Removes the last accepted word from the suggestions list and the "digitSequence"
+	 * or stops silently, when there is nothing to do.
+	 */
+	private void clearLastAcceptedWord() {
+		if (
+			lastAcceptedWord.isEmpty()
+			|| suggestions.isEmpty()
+			|| !suggestions.get(0).toLowerCase(language.getLocale()).startsWith(lastAcceptedWord.toLowerCase(language.getLocale()))
+		) {
+			return;
+		}
+
+		int lastAcceptedWordLength = lastAcceptedWord.length();
+		digitSequence = digitSequence.length() > lastAcceptedWordLength ? digitSequence.substring(lastAcceptedWordLength) : "";
+
+		ArrayList<String> lastSuggestions = new ArrayList<>(suggestions);
+		suggestions.clear();
+		for (String s : lastSuggestions) {
+			suggestions.add(s.length() >= lastAcceptedWordLength ? s.substring(lastAcceptedWordLength) : "");
+		}
+	}
+
+
 	/**
 	 * clearWordStem
 	 * Do not filter the suggestions by the word set using "setWordStem()", use only the digit sequence.
@@ -156,15 +183,18 @@ public class ModePredictive extends InputMode {
 	 * Note that you need to manually get the suggestions again to obtain a filtered list.
 	 */
 	@Override
-	public boolean setWordStem(String wordStem, boolean exact) {
-		if (language == null || wordStem == null || wordStem.length() < 1) {
+	public boolean setWordStem(String newStem, boolean exact) {
+		String sanitizedStem = TextTools.removeNonLetters(newStem);
+		if (language == null || sanitizedStem == null || sanitizedStem.length() < 1) {
 			return false;
 		}
 
 		try {
-			digitSequence = language.getDigitSequenceForWord(wordStem);
+			// digitSequence = "the raw input", so that everything the user typed is preserved visually
+			// stem = "the sanitized input", because filtering by anything that is not a letter makes no sense
+			digitSequence = language.getDigitSequenceForWord(newStem);
+			stem = sanitizedStem.toLowerCase(language.getLocale());
 			isStemFuzzy = !exact;
-			stem = digitSequence.startsWith("0") || digitSequence.startsWith("1") ? "" : wordStem.toLowerCase(language.getLocale());
 
 			Logger.d("tt9/setWordStem", "Stem is now: " + stem + (isStemFuzzy ? " (fuzzy)" : ""));
 			return true;
@@ -172,7 +202,7 @@ public class ModePredictive extends InputMode {
 			isStemFuzzy = false;
 			stem = "";
 
-			Logger.w("tt9/setWordStem", "Ignoring invalid stem: " + wordStem + ". " + e.getMessage());
+			Logger.w("tt9/setWordStem", "Ignoring invalid stem: " + newStem + ". " + e.getMessage());
 			return false;
 		}
 	}
@@ -239,19 +269,26 @@ public class ModePredictive extends InputMode {
 
 	/**
 	 * onAcceptSuggestion
-	 * Bring this word up in the suggestions list next time.
+	 * Bring this word up in the suggestions list next time and if necessary preserves the suggestion list
+	 * with "currentWord" cleaned from them.
 	 */
 	@Override
-	public void onAcceptSuggestion(@NonNull String currentWord) {
+	public void onAcceptSuggestion(@NonNull String currentWord, boolean preserveWords) {
 		lastAcceptedWord = currentWord;
-		lastAcceptedSequence = digitSequence;
-		reset();
 
-		if (currentWord.length() == 0) {
+		if (preserveWords) {
+			clearLastAcceptedWord();
+		} else {
+			reset();
+		}
+		stem = "";
+
+		if (currentWord.isEmpty()) {
 			Logger.i("acceptCurrentSuggestion", "Current word is empty. Nothing to accept.");
 			return;
 		}
 
+		// increment the frequency of the given word
 		try {
 			String sequence = language.getDigitSequenceForWord(currentWord);
 
@@ -284,22 +321,34 @@ public class ModePredictive extends InputMode {
 
 
 	/**
-	 * shouldAcceptCurrentSuggestion
+	 * shouldAcceptPreviousSuggestion
 	 * In this mode, In addition to confirming the suggestion in the input field,
 	 * we also increase its' priority. This function determines whether we want to do all this or not.
 	 */
 	@Override
-	public boolean shouldAcceptCurrentSuggestion(int key, boolean hold, boolean repeat) {
+	public boolean shouldAcceptPreviousSuggestion(int nextKey) {
 		return
-			hold
-			// Quickly accept suggestions using "space" instead of pressing "ok" then "space"
-			|| (key == 0 && !repeat)
-			// Punctuation is considered "a word", so that we can increase the priority as needed
-			// Also, it must break the current word.
-			|| (!language.isPunctuationPartOfWords() && key == 1 && digitSequence.length() > 0 && !digitSequence.endsWith("1"))
-			// On the other hand, letters also "break" punctuation.
-			|| (!language.isPunctuationPartOfWords() && key != 1 && digitSequence.endsWith("1"))
-			|| (digitSequence.endsWith("0") && key != 0);
+			!digitSequence.isEmpty() && (
+				(nextKey == 0 && digitSequence.charAt(digitSequence.length() - 1) != '0')
+				|| (nextKey != 0 && digitSequence.charAt(digitSequence.length() - 1) == '0')
+			);
+	}
+
+
+		/**
+	 * shouldAcceptPreviousSuggestion
+	 * Variant for post suggestion load analysis.
+	 */
+	@Override
+	public boolean shouldAcceptPreviousSuggestion() {
+		return
+			(autoAcceptTimeout == 0 && !digitSequence.startsWith("0"))
+			|| (
+				!digitSequence.isEmpty()
+				&& !predictions.areThereDbWords()
+				&& digitSequence.contains("1")
+				&& TextTools.containsOtherThan1(digitSequence)
+			);
 	}
 
 
@@ -307,11 +356,10 @@ public class ModePredictive extends InputMode {
 	public boolean shouldAddAutoSpace(InputType inputType, TextField textField, boolean isWordAcceptedManually, int nextKey) {
 		return autoSpace
 			.setLastWord(lastAcceptedWord)
-			.setLastSequence(lastAcceptedSequence)
+			.setLastSequence()
 			.setInputType(inputType)
 			.setTextField(textField)
 			.shouldAddAutoSpace(isWordAcceptedManually, nextKey);
-
 	}
 
 
@@ -319,7 +367,6 @@ public class ModePredictive extends InputMode {
 	public boolean shouldDeletePrecedingSpace(InputType inputType) {
 		return autoSpace
 			.setLastWord(lastAcceptedWord)
-			.setLastSequence(lastAcceptedSequence)
 			.setInputType(inputType)
 			.setTextField(null)
 			.shouldDeletePrecedingSpace();

src/io/github/sspanak/tt9/ime/modes/helpers/AutoSpace.java

@@ -1,22 +1,16 @@
 package io.github.sspanak.tt9.ime.modes.helpers;
 
-import java.util.regex.Pattern;
-
+import io.github.sspanak.tt9.TextTools;
 import io.github.sspanak.tt9.ime.helpers.InputType;
 import io.github.sspanak.tt9.ime.helpers.TextField;
 import io.github.sspanak.tt9.preferences.SettingsStore;
 
 public class AutoSpace {
-	private final Pattern isNumber = Pattern.compile("\\s*\\d+\\s*");
-	private final Pattern nextIsLetter = Pattern.compile("^\\p{L}+");
-	private final Pattern nextIsPunctuation = Pattern.compile("^\\p{Punct}");
-
 	private final SettingsStore settings;
 
 	private InputType inputType;
 	private TextField textField;
 	private String lastWord;
-	private String lastSequence;
 
 	public AutoSpace(SettingsStore settingsStore) {
 		settings = settingsStore;
@@ -37,8 +31,7 @@ public class AutoSpace {
 		return this;
 	}
 
-	public AutoSpace setLastSequence(String lastSequence) {
-		this.lastSequence = lastSequence;
+	public AutoSpace setLastSequence() {
 		return this;
 	}
 
@@ -57,12 +50,11 @@ public class AutoSpace {
 		return
 			settings.getAutoSpace()
 			&& !inputType.isSpecialized()
-			&& !nextChars.startsWith(" ")
-			&& !isNumber.matcher(previousChars).find()
-			&& !nextIsPunctuation.matcher(nextChars).find()
+			&& nextKey != 0
+			&& !TextTools.startsWithWhitespace(nextChars)
 			&& (
-				shouldAddAfterPunctuation(previousChars, nextKey)
-				|| shouldAddAfterWord(isWordAcceptedManually, nextChars)
+				shouldAddAfterWord(isWordAcceptedManually, previousChars, nextChars, nextKey)
+				|| shouldAddAfterPunctuation(previousChars, nextChars, nextKey)
 			);
 	}
 
@@ -73,24 +65,23 @@ public class AutoSpace {
 	 * The rules are similar to the ones in the standard Android keyboard (with some exceptions,
 	 * because we are not using a QWERTY keyboard here).
 	 */
-	private boolean shouldAddAfterPunctuation(String previousChars, int nextKey) {
-		return
-			// no space after whitespace or special characters
-			!previousChars.endsWith(" ") && !previousChars.endsWith("\n") && !previousChars.endsWith("\t") // previous whitespace
-			&& !lastSequence.equals("0") // previous previous math/special char
-			&& nextKey != 0 // composing (upcoming) whitespace or special character
+	private boolean shouldAddAfterPunctuation(String previousChars, String nextChars, int nextKey) {
+		char previousChar = previousChars.isEmpty() ? 0 : previousChars.charAt(previousChars.length() - 1);
 
-			// add space after the these
+		return
+			nextKey != 1
+			&& !TextTools.nextIsPunctuation(nextChars)
+			&& !TextTools.startsWithNumber(nextChars)
 			&& (
-				previousChars.endsWith(".")
-				|| previousChars.endsWith(",")
-				|| previousChars.endsWith(";")
-				|| previousChars.endsWith(":")
-				|| previousChars.endsWith("!")
-				|| previousChars.endsWith("?")
-				|| previousChars.endsWith(")")
-				|| previousChars.endsWith("]")
-				|| previousChars.endsWith("%")
+				previousChar == '.'
+				|| previousChar == ','
+				|| previousChar == ';'
+				|| previousChar == ':'
+				|| previousChar == '!'
+				|| previousChar == '?'
+				|| previousChar == ')'
+				|| previousChar == ']'
+				|| previousChar == '%'
 				|| previousChars.endsWith(" -")
 				|| previousChars.endsWith(" /")
 			);
@@ -98,15 +89,15 @@ public class AutoSpace {
 
 
 	/**
-	 * shouldAddAfterPunctuation
+	 * shouldAddAfterWord
 	 * Similar to "shouldAddAfterPunctuation()", but determines whether to add a space after words.
 	 */
-	private boolean shouldAddAfterWord(boolean isWordAcceptedManually, String nextChars) {
+	private boolean shouldAddAfterWord(boolean isWordAcceptedManually, String previousChars, String nextChars, int nextKey) {
 		return
-			// Do not add space when auto-accepting words, because it feels very confusing when typing.
-			isWordAcceptedManually
-			// Right before another word
-			&& !nextIsLetter.matcher(nextChars).find();
+			isWordAcceptedManually // Do not add space when auto-accepting words, because it feels very confusing when typing.
+			&& nextKey != 1
+			&& nextChars.isEmpty()
+			&& TextTools.previousIsLetter(previousChars);
 	}
 
 

src/io/github/sspanak/tt9/ime/modes/helpers/AutoTextCase.java

@@ -1,14 +1,11 @@
 package io.github.sspanak.tt9.ime.modes.helpers;
 
-import java.util.regex.Pattern;
-
+import io.github.sspanak.tt9.TextTools;
 import io.github.sspanak.tt9.ime.modes.InputMode;
 import io.github.sspanak.tt9.languages.Language;
 import io.github.sspanak.tt9.preferences.SettingsStore;
 
 public class AutoTextCase {
-	private final Pattern nextToWordRegex = Pattern.compile("\\b$");
-	private final Pattern startOfSentenceRegex = Pattern.compile("(?<!\\.)(^|[.?!¿¡])\\s*$");
 	private final SettingsStore settings;
 
 
@@ -68,11 +65,11 @@ public class AutoTextCase {
 		}
 
 		// start of sentence, excluding after "..."
-		if (startOfSentenceRegex.matcher(textBeforeCursor).find()) {
+		if (TextTools.isStartOfSentence(textBeforeCursor)) {
 			return InputMode.CASE_CAPITALIZE;
 		}
 
-		if (nextToWordRegex.matcher(textBeforeCursor).find()) {
+		if (TextTools.isNextToWord(textBeforeCursor)) {
 			return InputMode.CASE_LOWER;
 		}
 

src/io/github/sspanak/tt9/ime/modes/helpers/Predictions.java

@@ -23,6 +23,7 @@ public class Predictions {
 	private Runnable onWordsChanged = () -> {};
 
 	// data
+	private boolean areThereDbWords = false;
 	private final ArrayList<String> words = new ArrayList<>();
 
 	// punctuation/emoji
@@ -82,6 +83,10 @@ public class Predictions {
 		return words;
 	}
 
+	public boolean areThereDbWords() {
+		return areThereDbWords;
+	}
+
 
 	/**
 	 * suggestStem
@@ -89,7 +94,7 @@ public class Predictions {
 	 * the user has pressed X keys (otherwise, it makes no sense to add it).
 	 */
 	private void suggestStem() {
-		if (stem.length() > 0 && stem.length() == digitSequence.length()) {
+		if (!stem.isEmpty() && stem.length() == digitSequence.length()) {
 			words.add(stem);
 		}
 	}
@@ -114,7 +119,7 @@ public class Predictions {
 	 * sequence or loads the static ones.
 	 */
 	public void load() {
-		if (digitSequence == null || digitSequence.length() == 0) {
+		if (digitSequence == null || digitSequence.isEmpty()) {
 			words.clear();
 			onWordsChanged.run();
 			return;
@@ -124,7 +129,7 @@ public class Predictions {
 			onWordsChanged.run();
 		} else {
 			DictionaryDb.getWords(
-				this::onDbWords,
+				(words) -> onDbWords(words, true),
 				language,
 				digitSequence,
 				stem,
@@ -170,6 +175,23 @@ public class Predictions {
 		return true;
 	}
 
+	private void loadWithoutLeadingPunctuation() {
+		DictionaryDb.getWords(
+			(dbWords) -> {
+				char firstChar = inputWord.charAt(0);
+				for (int i = 0; i < dbWords.size(); i++) {
+					dbWords.set(i, firstChar + dbWords.get(i));
+				}
+				onDbWords(dbWords, false);
+			},
+			language,
+			digitSequence.substring(1),
+			stem.length() > 1 ? stem.substring(1) : "",
+			settings.getSuggestionsMin(),
+			settings.getSuggestionsMax()
+		);
+	}
+
 
 	/**
 	 * dbWordsHandler
@@ -177,8 +199,18 @@ public class Predictions {
 	 * they will be generated based on the "inputWord". After the word list is compiled, it notifies the
 	 * external handler it is now possible to use it with "getList()".
 	 */
-	private void onDbWords (ArrayList<String> dbWords) {
-		if (dbWords.size() == 0 && digitSequence.length() > 0) {
+	private void onDbWords(ArrayList<String> dbWords, boolean isRetryAllowed) {
+		// only the first round matters, the second one is just for getting the letters for a given key
+		areThereDbWords = !dbWords.isEmpty() && isRetryAllowed;
+
+		// If there were no database words for ",a", try getting the letters only (e.g. "a", "b", "c").
+		// We do this to display them in the correct order.
+		if (dbWords.isEmpty() && isRetryAllowed && digitSequence.length() == 2 && digitSequence.charAt(0) == '1') {
+			loadWithoutLeadingPunctuation();
+			return;
+		}
+
+		if (dbWords.isEmpty() && !digitSequence.isEmpty()) {
 			emptyDbWarning.emitOnce(language);
 			dbWords = generatePossibleCompletions(inputWord);
 		}
@@ -186,7 +218,7 @@ public class Predictions {
 		words.clear();
 		suggestStem();
 		suggestMissingWords(generatePossibleStemVariations(dbWords));
-		suggestMissingWords(dbWords);
+		suggestMissingWords(insertPunctuationCompletions(dbWords));
 
 		onWordsChanged.run();
 	}
@@ -206,20 +238,16 @@ public class Predictions {
 
 		// Make sure the displayed word and the digit sequence, we will be generating suggestions from,
 		// have the same length, to prevent visual discrepancies.
-		baseWord = (baseWord != null && baseWord.length() > 0) ? baseWord.substring(0, Math.min(digitSequence.length() - 1, baseWord.length())) : "";
+		baseWord = (baseWord != null && !baseWord.isEmpty()) ? baseWord.substring(0, Math.min(digitSequence.length() - 1, baseWord.length())) : "";
 
 		// append all letters for the last digit in the sequence (the last pressed key)
 		int lastSequenceDigit = digitSequence.charAt(digitSequence.length() - 1) - '0';
-		for (String keyLetter : language.getKeyCharacters(lastSequenceDigit)) {
-			// let's skip numbers, because it's weird, for example:
-			// | weird | weire | weirf | weir2 |
-			if (keyLetter.charAt(0) < '0' || keyLetter.charAt(0) > '9') {
-				generatedWords.add(baseWord + keyLetter);
-			}
+		for (String keyLetter : language.getKeyCharacters(lastSequenceDigit, false)) {
+			generatedWords.add(baseWord + keyLetter);
 		}
 
 		// if there are no letters for this key, just append the number
-		if (generatedWords.size() == 0) {
+		if (generatedWords.isEmpty()) {
 			generatedWords.add(baseWord + digitSequence.charAt(digitSequence.length() - 1));
 		}
 
@@ -227,6 +255,46 @@ public class Predictions {
 	}
 
 
+	/**
+	 * insertPunctuationCompletions
+	 * When given: "you'", for example, this also generates all other 1-key alternatives, like:
+	 * "you.", "you?", "you!" and so on. The generated words will be inserted after the direct
+	 * database matches and before the fuzzy matches, as if they were direct matches with low frequency.
+	 * This is to preserve the sorting by length and frequency.
+	 */
+	private ArrayList<String> insertPunctuationCompletions(ArrayList<String> dbWords) {
+		if (!stem.isEmpty() || dbWords.isEmpty() || digitSequence.length() < 2 || !digitSequence.endsWith("1")) {
+			return dbWords;
+		}
+
+		ArrayList<String> complementedWords = new ArrayList<>();
+		int exactMatchLength = digitSequence.length();
+
+		// shortest database words (exact matches)
+		for (String w : dbWords) {
+			if (w.length() <= exactMatchLength) {
+				complementedWords.add(w);
+			}
+		}
+
+		// generated "exact matches"
+		for (String w : generatePossibleCompletions(dbWords.get(0))) {
+			if (!dbWords.contains(w) && !dbWords.contains(w.toLowerCase(language.getLocale()))) {
+				complementedWords.add(w);
+			}
+		}
+
+		// longer database words (fuzzy matches)
+		for (String w : dbWords) {
+			if (w.length() > exactMatchLength) {
+				complementedWords.add(w);
+			}
+		}
+
+		return complementedWords;
+	}
+
+
 	/**
 	 * generatePossibleStemVariations
 	 * Similar to generatePossibleCompletions(), but uses the current filter as a base word. This is
@@ -242,7 +310,7 @@ public class Predictions {
 	 */
 	private ArrayList<String> generatePossibleStemVariations(ArrayList<String> dbWords) {
 		ArrayList<String> variations = new ArrayList<>();
-		if (stem.length() == 0) {
+		if (stem.isEmpty()) {
 			return variations;
 		}
 

src/io/github/sspanak/tt9/languages/Language.java

@@ -18,7 +18,6 @@ public class Language {
 
 	// settings
 	protected boolean hasUpperCase = true;
-	protected boolean isPunctuationPartOfWords; // see the getter for more info
 
 	final public int getId() {
 		if (id == 0) {
@@ -60,24 +59,6 @@ public class Language {
 		return abcString;
 	}
 
-	/**
-	 * isPunctuationPartOfWords
-	 * This plays a role in Predictive mode only.
-	 *
-	 * Return "true", if you need to use the 1-key for typing words, such as:
-	 * "it's" (English), "a'tje" (Dutch), "п'ят" (Ukrainian).
-	 *
-	 * Return "false" also:
-	 * 		- hide words like the above from the suggestions.
-	 *		- 1-key would commit the current word, then display the punctuation list.
-	 * 			For example, pressing 1-key after "it" would accept "it" as a separate word,
-	 * 			then display only: | , | . | ! | ? | ...
-	 *
-	 * "false" is recommended when apostrophes or other punctuation are not part of the words,
-	 * because it would allow faster typing.
-	 */
-	final public boolean isPunctuationPartOfWords() { return isPunctuationPartOfWords; }
-
 
 	public boolean hasUpperCase() {
 		return hasUpperCase;
@@ -124,7 +105,21 @@ public class Language {
 	}
 
 	public String capitalize(String word) {
-		return word != null ? word.substring(0, 1).toUpperCase(locale) + word.substring(1).toLowerCase(locale) : null;
+		if (word == null) {
+			return null;
+		}
+
+		String capitalizedWord = "";
+
+		if (!word.isEmpty()) {
+			capitalizedWord += word.substring(0, 1).toUpperCase(locale);
+		}
+
+		if (word.length() > 1) {
+			capitalizedWord += word.substring(1).toLowerCase(locale);
+		}
+
+		return capitalizedWord;
 	}
 
 	public boolean isMixedCaseWord(String word) {

src/io/github/sspanak/tt9/languages/definitions/BrazilianPortuguese.java

@@ -11,8 +11,6 @@ public class BrazilianPortuguese extends English {
 		locale = new Locale("pt","BR");
 		dictionaryFile = "pt-BR-utf8.csv";
 
-		isPunctuationPartOfWords = true;
-
 		characterMap.get(2).addAll(Arrays.asList("ç", "á", "â", "ã", "à"));
 		characterMap.get(3).addAll(Arrays.asList("é", "ê", "è"));
 		characterMap.get(4).add("í");

src/io/github/sspanak/tt9/languages/definitions/Bulgarian.java

@@ -12,8 +12,6 @@ public class Bulgarian extends Language {
 		locale = new Locale("bg","BG");
 		dictionaryFile = "bg-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap = new ArrayList<>(Arrays.asList(
 			Characters.Special, // 0
 			Characters.Sentence, // 1

src/io/github/sspanak/tt9/languages/definitions/Dutch.java

@@ -10,8 +10,6 @@ public class Dutch extends English {
 		locale = new Locale("nl","NL");
 		dictionaryFile = "nl-utf8.csv";
 
-		isPunctuationPartOfWords = true;
-
 		characterMap.get(2).addAll(Arrays.asList("à", "ä", "ç"));
 		characterMap.get(3).addAll(Arrays.asList("é", "è", "ê", "ë"));
 		characterMap.get(4).addAll(Arrays.asList("î", "ï"));

src/io/github/sspanak/tt9/languages/definitions/English.java

@@ -12,8 +12,6 @@ public class English extends Language {
 		locale = Locale.ENGLISH;
 		dictionaryFile = "en-utf8.csv";
 
-		isPunctuationPartOfWords = true;
-
 		characterMap = new ArrayList<>(Arrays.asList(
 			Characters.Special, // 0
 			Characters.Sentence, // 1

src/io/github/sspanak/tt9/languages/definitions/Finnish.java

@@ -10,8 +10,6 @@ public class Finnish extends English {
 		locale = new Locale("fi","FI");
 		dictionaryFile = "fi-utf8.csv";
 
-		isPunctuationPartOfWords = true;
-
 		characterMap.get(2).addAll(Arrays.asList("ä", "å"));
 		characterMap.get(6).add("ö");
 	}

src/io/github/sspanak/tt9/languages/definitions/French.java

@@ -10,8 +10,6 @@ public class French extends English {
 		locale = Locale.FRENCH;
 		dictionaryFile = "fr-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap.get(2).addAll(Arrays.asList("à", "â", "æ", "ç"));
 		characterMap.get(3).addAll(Arrays.asList("é", "è", "ê", "ë"));
 		characterMap.get(4).addAll(Arrays.asList("î", "ï"));

src/io/github/sspanak/tt9/languages/definitions/German.java

@@ -9,8 +9,6 @@ public class German extends English {
 		locale = Locale.GERMAN;
 		dictionaryFile = "de-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap.get(2).add("ä");
 		characterMap.get(6).add("ö");
 		characterMap.get(7).add("ß");

src/io/github/sspanak/tt9/languages/definitions/Hebrew.java

@@ -14,7 +14,6 @@ public class Hebrew extends Language {
 		abcString = "אבג";
 
 		hasUpperCase = false;
-		isPunctuationPartOfWords = true;
 
 		characterMap = new ArrayList<>(Arrays.asList(
 			Characters.Special, // 0

src/io/github/sspanak/tt9/languages/definitions/Italian.java

@@ -10,8 +10,6 @@ public class Italian extends English {
 		locale = Locale.ITALIAN;
 		dictionaryFile = "it-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap.get(2).add("à");
 		characterMap.get(3).addAll(Arrays.asList("é", "è"));
 		characterMap.get(4).addAll(Arrays.asList("ì", "í", "î"));

src/io/github/sspanak/tt9/languages/definitions/Norwegian.java

@@ -10,8 +10,6 @@ public class Norwegian extends English {
 		locale = new Locale("nb","NO");
 		dictionaryFile = "nb-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap.get(2).addAll(Arrays.asList("æ", "å"));
 		characterMap.get(3).addAll(Arrays.asList("é", "è"));
 		characterMap.get(6).addAll(Arrays.asList("ø", "ó", "ò", "ô"));

src/io/github/sspanak/tt9/languages/definitions/Polish.java

@@ -10,8 +10,6 @@ public class Polish extends English {
 		locale = new Locale("pl","PL");
 		dictionaryFile = "pl-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap.get(2).addAll(Arrays.asList("ą", "ć"));
 		characterMap.get(3).add("ę");
 		characterMap.get(5).add("ł");

src/io/github/sspanak/tt9/languages/definitions/Russian.java

@@ -12,8 +12,6 @@ public class Russian extends Language {
 		locale = new Locale("ru","RU");
 		dictionaryFile = "ru-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap = new ArrayList<>(Arrays.asList(
 			Characters.Special, // 0
 			Characters.Sentence, // 1

src/io/github/sspanak/tt9/languages/definitions/Spanish.java

@@ -13,8 +13,6 @@ public class Spanish extends English {
 		locale = new Locale("es", "ES");
 		dictionaryFile = "es-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap.set(1, new ArrayList<>(Characters.Sentence));
 		characterMap.get(1).addAll(Arrays.asList("¡", "¿"));
 

src/io/github/sspanak/tt9/languages/definitions/Swedish.java

@@ -10,8 +10,6 @@ public class Swedish extends English {
 		locale = new Locale("sv","SE");
 		dictionaryFile = "sv-utf8.csv";
 
-		isPunctuationPartOfWords = false;
-
 		characterMap.get(2).addAll(Arrays.asList("å", "ä"));
 		characterMap.get(3).add("é");
 		characterMap.get(6).add("ö");

src/io/github/sspanak/tt9/languages/definitions/Ukrainian.java

@@ -12,8 +12,6 @@ public class Ukrainian extends Language {
 		locale = new Locale("uk","UA");
 		dictionaryFile = "uk-utf8.csv";
 
-		isPunctuationPartOfWords = true;
-
 		characterMap = new ArrayList<>(Arrays.asList(
 			Characters.Special, // 0
 			Characters.Sentence, // 1

src/io/github/sspanak/tt9/languages/definitions/Yiddish.java

@@ -8,6 +8,5 @@ public class Yiddish extends Hebrew {
 
 		locale = new Locale("ji","JI");
 		dictionaryFile = "ji-utf8.csv";
-		isPunctuationPartOfWords = true;
 	}
 }