Back to SQLite (#420)

* Deleted Objectbox and went back to SQLite. The database structure is entirely new and optimized for fast performance

* Added slow query stats + cache for even faster performance

* automatic language sorting script 

* legacy database management using SQLiteOpener

* simplified access to the constant settings

build.gradle

@@ -9,13 +9,11 @@ buildscript {
 	dependencies {
 		classpath 'com.android.tools.build:gradle:8.0.2'
 		classpath 'gradle.plugin.at.zierler:yaml-validator-plugin:1.5.0'
-		classpath("io.objectbox:objectbox-gradle-plugin:3.7.1")
 	}
 }
 
 apply plugin: 'com.android.application'
 apply plugin: 'at.zierler.yamlvalidator'
-apply plugin: "io.objectbox"
 
 apply from: 'gradle/scripts/constants.gradle'
 apply from: 'gradle/scripts/dictionary-tools.gradle'

objectbox-models/default.json

@@ -1,78 +0,0 @@
-{
-  "_note1": "KEEP THIS FILE! Check it into a version control system (VCS) like git.",
-  "_note2": "ObjectBox manages crucial IDs for your object model. See docs for details.",
-  "_note3": "If you have VCS merge conflicts, you must resolve them according to ObjectBox docs.",
-  "entities": [
-    {
-      "id": "1:8686759156275895115",
-      "lastPropertyId": "9:148562041880145406",
-      "name": "Word",
-      "properties": [
-        {
-          "id": "1:7053654361616810150",
-          "name": "id",
-          "type": 6,
-          "flags": 1
-        },
-        {
-          "id": "2:1100881396300803213",
-          "name": "frequency",
-          "type": 5
-        },
-        {
-          "id": "3:3482550679443532896",
-          "name": "isCustom",
-          "type": 1
-        },
-        {
-          "id": "4:7366572918924354162",
-          "name": "langId",
-          "type": 5
-        },
-        {
-          "id": "5:2610279871343053806",
-          "name": "length",
-          "type": 5
-        },
-        {
-          "id": "6:5269773217039117329",
-          "name": "sequence",
-          "indexId": "1:2971223841434624317",
-          "type": 9,
-          "flags": 2048
-        },
-        {
-          "id": "7:3922044271904033267",
-          "name": "sequenceShort",
-          "indexId": "2:2641086768976362614",
-          "type": 2,
-          "flags": 8
-        },
-        {
-          "id": "8:1684236207225806285",
-          "name": "uniqueId",
-          "indexId": "3:5820769207826940948",
-          "type": 9,
-          "flags": 34848
-        },
-        {
-          "id": "9:148562041880145406",
-          "name": "word",
-          "type": 9
-        }
-      ],
-      "relations": []
-    }
-  ],
-  "lastEntityId": "1:8686759156275895115",
-  "lastIndexId": "3:5820769207826940948",
-  "lastRelationId": "0:0",
-  "lastSequenceId": "0:0",
-  "modelVersion": 5,
-  "modelVersionParserMinimum": 5,
-  "retiredEntityUids": [],
-  "retiredIndexUids": [],
-  "retiredPropertyUids": [],
-  "retiredRelationUids": [],
-  "version": 1
-}

res/values/strings.xml

@@ -25,11 +25,11 @@
 	<string name="pref_category_hacks">Compatibility</string>
 	<string name="pref_category_appearance">Appearance</string>
 	<string name="pref_category_debug_options" translatable="false">Debug Options</string>
-	<string name="pref_category_log_messages" translatable="false">Recent Log Messages</string>
 	<string name="pref_category_predictive_mode">Predictive Mode</string>
 	<string name="pref_category_function_keys">Select Hotkeys</string>
 	<string name="pref_category_keypad">Keypad</string>
 	<string name="pref_category_setup">Initial Setup</string>
+	<string name="pref_category_usage_stats" translatable="false">Usage Stats</string>
 
 	<string name="pref_abc_auto_accept">Automatic Letter Select</string>
 	<string name="pref_abc_auto_accept_summary">Automatically type the selected letter after a short delay.</string>
@@ -44,8 +44,6 @@
 	<string name="pref_dark_theme_yes">Yes</string>
 	<string name="pref_dark_theme_no">No</string>
 	<string name="pref_dark_theme_auto">Auto</string>
-	<string name="pref_debug_logs" translatable="false">Detailed Debug Logs</string>
-	<string name="pref_system_logs" translatable="false">System Logs</string>
 	<string name="pref_double_zero_char">Character for Double 0-key Press</string>
 	<string name="pref_hack_fb_messenger">Send messages with OK in Messenger</string>
 	<string name="pref_hack_fb_messenger_summary">Allows sending messages with the OK key in Facebook Messenger.</string>

res/xml/prefs_screen_debug.xml

@@ -1,20 +1,26 @@
 <?xml version="1.0" encoding="utf-8"?>
 <PreferenceScreen xmlns:app="http://schemas.android.com/apk/res-auto"  app:orderingFromXml="true">
 
+	<SwitchPreferenceCompat
+		app:fragment="io.github.sspanak.tt9.preferences.SlowQueriesScreen"
+		app:key="pref_slow_queries"
+		app:layout="@layout/pref_text"
+		app:title="@string/pref_category_usage_stats" />
+
 	<SwitchPreferenceCompat
 		app:defaultValue="false"
 		app:key="pref_enable_debug_logs"
 		app:layout="@layout/pref_switch"
-		app:title="@string/pref_debug_logs" />
+		app:title="Debug Logs" />
 
 	<SwitchPreferenceCompat
 		app:defaultValue="false"
 		app:key="pref_enable_system_logs"
 		app:layout="@layout/pref_switch"
-		app:title="@string/pref_system_logs" />
+		app:title="System Logs" />
 
 	<PreferenceCategory
-		app:title="@string/pref_category_log_messages"
+		app:title="Recent Log Messages"
 		app:layout="@layout/pref_category"
 		app:singleLineTitle="true">
 		<Preference

res/xml/prefs_screen_usage_stats.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<PreferenceScreen xmlns:app="http://schemas.android.com/apk/res-auto"  app:orderingFromXml="true">
+
+	<SwitchPreferenceCompat
+		app:key="pref_slow_queries_reset_stats"
+		app:layout="@layout/pref_text"
+		app:title="Clear DB Cache" />
+
+	<PreferenceCategory
+		app:title="Summary"
+		app:layout="@layout/pref_category"
+		app:singleLineTitle="true">
+		<Preference
+			app:key="summary_container"
+			app:summary="--"
+			app:layout="@layout/pref_plain_text">
+		</Preference>
+	</PreferenceCategory>
+
+	<PreferenceCategory
+		app:title="Slow Queries"
+		app:layout="@layout/pref_category"
+		app:singleLineTitle="true">
+		<Preference
+			app:key="query_list_container"
+			app:summary="--"
+			app:layout="@layout/pref_plain_text">
+		</Preference>
+	</PreferenceCategory>
+
+</PreferenceScreen>

scripts/sort-dictionary.js

@@ -0,0 +1,123 @@
+const { basename } = require('path');
+const { createReadStream, existsSync } = require('fs');
+const { createInterface } = require('readline');
+
+
+function printHelp() {
+	console.log(`Usage ${basename(process.argv[1])} LOCALE WORD-LIST.txt LANGUAGE-DEFINITION.yml`);
+	console.log('Sorts a dictionary for optimum search speed.');
+}
+
+
+
+function validateInput() {
+	if (process.argv.length < 4) {
+		printHelp();
+		process.exit(1);
+	}
+
+	if (!existsSync(process.argv[3])) {
+		console.error(`Failure! Could not find word list file "${process.argv[3]}."`);
+		process.exit(2);
+	}
+
+	if (!existsSync(process.argv[4])) {
+		console.error(`Failure! Could not find language definition file "${process.argv[3]}."`);
+		process.exit(2);
+	}
+
+	return {
+		definitionFile: process.argv[4],
+		wordsFile: process.argv[3],
+		locale: process.argv[2]
+	};
+}
+
+
+function printWords(wordList) {
+	if (Array.isArray(wordList)) {
+		wordList.forEach(w => console.log(`${w.word}${w.frequency ? '\t' + w.frequency : ''}`));
+	}
+}
+
+
+async function readWords(fileName) {
+	const words = [];
+
+	if (!fileName) {
+		return words;
+	}
+
+	for await (const line of createInterface({ input: createReadStream(fileName) })) {
+		const [word, frequency] = line.split("\t");
+		words.push({ 
+			word, 
+			frequency: Number.isNaN(Number.parseInt(frequency)) ? 0 : Number.parseInt(frequency) 
+		});
+	}
+
+	return words;
+}
+
+
+async function readDefinition(fileName) {
+	if (!fileName) {
+		return new Map();
+	}
+
+	let lettersPattern = /^\s+-\s*\[([^\]]+)/;
+	let letterWeights = new Map([["'", 1], ['-', 1], ['"', 1]]);
+
+	let key = 2;
+	for await (const line of createInterface({ input: createReadStream(fileName) })) {
+		if (line.includes('SPECIAL') || line.includes('PUNCTUATION')) {
+			continue;
+		}
+
+		const matches = line.match(lettersPattern);
+		if (matches && matches[1]) {
+			const letters = matches[1].replace(/\s/g, '').split(',');
+			letters.forEach(l => letterWeights.set(l, key));
+			key++;
+		}
+	}
+
+	return letterWeights;
+}
+
+
+function dictionarySort(a, b, letterWeights, locale) {
+	if (a.word.length !== b.word.length) {
+		return a.word.length - b.word.length;
+	}
+
+	for (let i = 0, end = a.word.length; i < end; i++) {
+		const charA = a.word.toLocaleLowerCase(locale).charAt(i);
+		const charB = b.word.toLocaleLowerCase(locale).charAt(i);
+		const distance = letterWeights.get(charA) - letterWeights.get(charB);
+
+
+		if (distance !== 0) {
+			return distance;
+		}
+	}
+
+	return 0;
+}
+
+
+async function work({ definitionFile, wordsFile, locale }) {
+	return Promise.all([
+		readWords(wordsFile),
+		readDefinition(definitionFile)
+	]).then(([words, letterWeights]) =>
+		words.sort((a, b) => dictionarySort(a, b, letterWeights, locale))
+	);
+}
+
+
+
+/** main **/
+work(validateInput())
+	.then(words => printWords(words))
+	.catch(e => console.error(e));

src/io/github/sspanak/tt9/Logger.java

@@ -7,7 +7,7 @@ public class Logger {
 	public static int LEVEL = BuildConfig.DEBUG ? Log.DEBUG : Log.ERROR;
 
 	public static boolean isDebugLevel() {
-		return LEVEL == Log.DEBUG;
+		return LEVEL <= Log.DEBUG;
 	}
 
 	public static void enableDebugLevel(boolean yes) {
@@ -16,7 +16,7 @@ public class Logger {
 
 	static public void v(String tag, String msg)  {
 		if (LEVEL <= Log.VERBOSE) {
-			Log.v(TAG_PREFIX + tag, msg);
+			Log.d(TAG_PREFIX + tag, msg);
 		}
 	}
 

src/io/github/sspanak/tt9/TextTools.java

@@ -1,5 +1,9 @@
 package io.github.sspanak.tt9;
 
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.TimeZone;
 import java.util.regex.Pattern;
 
 public class TextTools {
@@ -37,7 +41,14 @@ public class TextTools {
 		return str != null && !str.isEmpty() && (str.charAt(0) >= '0' && str.charAt(0) <= '9');
 	}
 
-	public static String removeNonLetters(String str) {
+	public static String unixTimestampToISODate(long timestamp) {
-		return str != null ? str.replaceAll("\\P{L}", "") : null;
+		if (timestamp < 0) {
+			return "--";
+		}
+
+		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US);
+		sdf.setTimeZone(TimeZone.getDefault());
+
+		return sdf.format(new Date(timestamp));
 	}
 }

src/io/github/sspanak/tt9/db/DictionaryDb.java

@@ -1,271 +0,0 @@
-package io.github.sspanak.tt9.db;
-
-import android.content.Context;
-import android.os.Handler;
-
-import androidx.annotation.NonNull;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import io.github.sspanak.tt9.ConsumerCompat;
-import io.github.sspanak.tt9.Logger;
-import io.github.sspanak.tt9.db.exceptions.InsertBlankWordException;
-import io.github.sspanak.tt9.db.objectbox.Word;
-import io.github.sspanak.tt9.db.objectbox.WordList;
-import io.github.sspanak.tt9.db.objectbox.WordStore;
-import io.github.sspanak.tt9.ime.TraditionalT9;
-import io.github.sspanak.tt9.languages.Language;
-import io.github.sspanak.tt9.preferences.SettingsStore;
-import io.objectbox.exception.UniqueViolationException;
-
-public class DictionaryDb {
-	private static WordStore store;
-	private static final Handler asyncHandler = new Handler();
-
-	public static synchronized void init(Context context) {
-		if (store == null) {
-			context = context == null ? TraditionalT9.getMainContext() : context;
-			store = new WordStore(context);
-		}
-	}
-
-
-	public static synchronized void init() {
-		init(null);
-	}
-
-
-	private static WordStore getStore() {
-		init();
-		return store;
-	}
-
-
-	private static void printLoadDebug(String sequence, WordList words, long startTime) {
-		if (!Logger.isDebugLevel()) {
-			return;
-		}
-
-		StringBuilder debugText = new StringBuilder("===== Word Matches =====");
-		debugText
-			.append("\n")
-			.append("Word Count: ").append(words.size())
-			.append(". Time: ").append(System.currentTimeMillis() - startTime).append(" ms.");
-		if (words.size() > 0) {
-			debugText.append("\n").append(words);
-		} else {
-			debugText.append(" Sequence: ").append(sequence);
-		}
-
-		Logger.d("loadWords", debugText.toString());
-	}
-
-
-	public static void runInTransaction(Runnable r) {
-		getStore().runInTransaction(r);
-	}
-
-
-	/**
-	 * normalizeWordFrequencies
-	 * Normalizes the word frequencies for all languages that have reached the maximum, as defined in
-	 * the settings.
-	 * This query will finish immediately, if there is nothing to do. It's safe to run it often.
-	 */
-	public static void normalizeWordFrequencies(SettingsStore settings) {
-		final String LOG_TAG = "db.normalizeWordFrequencies";
-
-		new Thread(() -> {
-			for (int langId : getStore().getLanguages()) {
-				getStore().runInTransactionAsync(() -> {
-					try {
-						long start = System.currentTimeMillis();
-
-						if (getStore().getMaxFrequency(langId) < settings.getWordFrequencyMax()) {
-							return;
-						}
-
-						List<Word> words = getStore().getMany(langId);
-						if (words == null) {
-							return;
-						}
-
-						for (Word w : words) {
-							w.frequency /= settings.getWordFrequencyNormalizationDivider();
-						}
-
-						getStore().put(words);
-
-						Logger.d(
-							LOG_TAG,
-							"Normalized language: " + langId + ", " + words.size() + " words in: " + (System.currentTimeMillis() - start) + " ms"
-						);
-					} catch (Exception e) {
-						Logger.e(LOG_TAG, "Word normalization failed. " + e.getMessage());
-					} finally {
-						getStore().closeThreadResources();
-					}
-				});
-			}
-		}).start();
-	}
-
-
-	public static void areThereWords(ConsumerCompat<Boolean> notification, Language language) {
-		new Thread(() -> {
-			boolean areThere = getStore().count(language != null ? language.getId() : -1) > 0;
-			getStore().closeThreadResources();
-			notification.accept(areThere);
-		}).start();
-	}
-
-
-	public static void deleteWords(Context context, Runnable notification) {
-		new Thread(() -> {
-			getStore().destroy();
-			store = null;
-			init(context);
-			notification.run();
-		}).start();
-	}
-
-
-	public static void deleteWords(Runnable notification, @NonNull ArrayList<Integer> languageIds) {
-		new Thread(() -> {
-			getStore().removeMany(languageIds).closeThreadResources();
-			notification.run();
-		}).start();
-	}
-
-
-	public static void insertWord(ConsumerCompat<Integer> statusHandler, @NonNull Language language, String word) throws Exception {
-		if (word == null || word.length() == 0) {
-			throw new InsertBlankWordException();
-		}
-
-		new Thread(() -> {
-			try {
-				if (getStore().exists(language.getId(), word, language.getDigitSequenceForWord(word))) {
-					throw new UniqueViolationException("Word already exists");
-				}
-				getStore().put(Word.create(language, word, 1, true));
-				statusHandler.accept(0);
-			} catch (UniqueViolationException e) {
-				String msg = "Skipping word: '" + word + "' for language: " + language.getId() + ", because it already exists.";
-				Logger.w("insertWord", msg);
-				statusHandler.accept(1);
-			} catch (Exception e) {
-				String msg = "Failed inserting word: '" + word + "' for language: " + language.getId() + ". " + e.getMessage();
-				Logger.e("insertWord", msg);
-				statusHandler.accept(2);
-			} finally {
-				getStore().closeThreadResources();
-			}
-		}).start();
-	}
-
-
-	public static void upsertWordsSync(List<Word> words) {
-		getStore().put(words);
-		getStore().closeThreadResources();
-	}
-
-
-	public static void incrementWordFrequency(@NonNull Language language, @NonNull String word, @NonNull String sequence) {
-		// If any of these is empty, it is the same as changing the frequency of: "", which is simply a no-op.
-		if (word.length() == 0 || sequence.length() == 0) {
-			return;
-		}
-
-		new Thread(() -> {
-			try {
-				long start = System.currentTimeMillis();
-
-				Word dbWord = getStore().get(language.getId(), word, sequence);
-
-				// In case the user has changed the text case, there would be no match.
-				// Try again with the lowercase equivalent.
-				if (dbWord == null) {
-					dbWord = getStore().get(language.getId(), word.toLowerCase(language.getLocale()), sequence);
-				}
-
-				if (dbWord == null) {
-					throw new Exception("No such word");
-				}
-
-				int max = getStore().getMaxFrequency(dbWord.langId, dbWord.sequence, dbWord.word);
-				if (dbWord.frequency <= max) {
-					dbWord.frequency = max + 1;
-					getStore().put(dbWord);
-					long time = System.currentTimeMillis() - start;
-
-					Logger.d(
-					"incrementWordFrequency",
-					"Incremented frequency of '" + dbWord.word + "' to: " + dbWord.frequency + ". Time: " + time + " ms"
-					);
-				} else {
-					long time = System.currentTimeMillis() - start;
-					Logger.d(
-						"incrementWordFrequency",
-						"'" + dbWord.word + "' is already the top word. Keeping frequency: " + dbWord.frequency + ". Time: " + time + " ms"
-					);
-				}
-			} catch (Exception e) {
-				Logger.e(
-					DictionaryDb.class.getName(),
-					"Failed incrementing word frequency. Word: " + word + ". " + e.getMessage()
-				);
-			} finally {
-				getStore().closeThreadResources();
-			}
-		}).start();
-	}
-
-
-	/**
-	 * loadWords
-	 * Loads words matching and similar to a given digit sequence
-	 * For example: "7655" -> "roll" (exact match), but also: "rolled", "roller", "rolling", ...
-	 * and other similar.
-	 */
-	private static ArrayList<String> loadWords(Language language, String sequence, String filter, int minimumWords, int maximumWords) {
-		long start = System.currentTimeMillis();
-
-		WordList matches = getStore()
-			.getMany(language, sequence, filter, maximumWords)
-			.filter(sequence.length(), minimumWords);
-
-		getStore().closeThreadResources();
-		printLoadDebug(sequence, matches, start);
-		return matches.toStringList();
-	}
-
-
-	public static void getWords(ConsumerCompat<ArrayList<String>> dataHandler, Language language, String sequence, String filter, int minimumWords, int maximumWords) {
-		final int minWords = Math.max(minimumWords, 0);
-		final int maxWords = Math.max(maximumWords, minWords);
-
-		if (sequence == null || sequence.length() == 0) {
-			Logger.w("db.getWords", "Attempting to get words for an empty sequence.");
-			sendWords(dataHandler, new ArrayList<>());
-			return;
-		}
-
-		if (language == null) {
-			Logger.w("db.getWords", "Attempting to get words for NULL language.");
-			sendWords(dataHandler, new ArrayList<>());
-			return;
-		}
-
-		new Thread(() -> sendWords(
-			dataHandler,
-			loadWords(language, sequence, filter, minWords, maxWords))
-		).start();
-	}
-
-
-	private static void sendWords(ConsumerCompat<ArrayList<String>> dataHandler, ArrayList<String> wordList) {
-		asyncHandler.post(() -> dataHandler.accept(wordList));
-	}
-}

src/io/github/sspanak/tt9/db/DictionaryLoader.java

@@ -13,21 +13,25 @@ import java.util.Locale;
 
 import io.github.sspanak.tt9.ConsumerCompat;
 import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.db.entities.WordBatch;
 import io.github.sspanak.tt9.db.exceptions.DictionaryImportAbortedException;
 import io.github.sspanak.tt9.db.exceptions.DictionaryImportAlreadyRunningException;
 import io.github.sspanak.tt9.db.exceptions.DictionaryImportException;
-import io.github.sspanak.tt9.db.objectbox.Word;
+import io.github.sspanak.tt9.db.sqlite.DeleteOps;
+import io.github.sspanak.tt9.db.sqlite.InsertOps;
+import io.github.sspanak.tt9.db.sqlite.SQLiteOpener;
+import io.github.sspanak.tt9.db.sqlite.Tables;
 import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException;
 import io.github.sspanak.tt9.languages.InvalidLanguageException;
 import io.github.sspanak.tt9.languages.Language;
 import io.github.sspanak.tt9.preferences.SettingsStore;
 
 public class DictionaryLoader {
+	private static final String LOG_TAG = "DictionaryLoader";
 	private static DictionaryLoader self;
-	private final String logTag = "DictionaryLoader";
 
 	private final AssetManager assets;
-	private final SettingsStore settings;
+	private final SQLiteOpener sqlite;
 
 	private static final Handler asyncHandler = new Handler();
 	private ConsumerCompat<Bundle> onStatusChange;
@@ -50,7 +54,7 @@ public class DictionaryLoader {
 
 	public DictionaryLoader(Context context) {
 		assets = context.getAssets();
-		settings = new SettingsStore(context);
+		sqlite = SQLiteOpener.getInstance(context);
 	}
 
 
@@ -70,7 +74,7 @@ public class DictionaryLoader {
 		}
 
 		if (languages.size() == 0) {
-			Logger.d(logTag, "Nothing to do");
+			Logger.d(LOG_TAG, "Nothing to do");
 			return;
 		}
 
@@ -109,118 +113,197 @@ public class DictionaryLoader {
 
 	private void importAll(Language language) {
 		if (language == null) {
-			Logger.e(logTag, "Failed loading a dictionary for NULL language.");
+			Logger.e(LOG_TAG, "Failed loading a dictionary for NULL language.");
 			sendError(InvalidLanguageException.class.getSimpleName(), -1);
 			return;
 		}
 
-		DictionaryDb.runInTransaction(() -> {
+		try {
-			try {
+			long start = System.currentTimeMillis();
-				long start = System.currentTimeMillis();
+			float progress = 1;
-				importWords(language, language.getDictionaryFile());
+			final float dictionaryMaxProgress = 90f;
-				Logger.i(
-					logTag,
-					"Dictionary: '" + language.getDictionaryFile() + "'" +
-						" processing time: " + (System.currentTimeMillis() - start) + " ms"
-				);
 
-				start = System.currentTimeMillis();
+			sqlite.beginTransaction();
-				importLetters(language);
-				Logger.i(
-					logTag,
-					"Loaded letters for '" + language.getName() + "' language in: " + (System.currentTimeMillis() - start) + " ms"
-				);
-			} catch (DictionaryImportAbortedException e) {
-				stop();
 
-				Logger.i(
+			Tables.dropIndexes(sqlite.getDb(), language);
-					logTag,
+			sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
-					e.getMessage() + ". File '" + language.getDictionaryFile() + "' not imported."
+			logLoadingStep("Indexes dropped", language, start);
-				);
-			} catch (DictionaryImportException e) {
-				stop();
-				sendImportError(DictionaryImportException.class.getSimpleName(), language.getId(), e.line, e.word);
 
-				Logger.e(
+			start = System.currentTimeMillis();
-					logTag,
+			DeleteOps.delete(sqlite, language.getId());
-					" Invalid word: '" + e.word
+			sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
-					+ "' in dictionary: '" + language.getDictionaryFile() + "'"
+			logLoadingStep("Storage cleared", language, start);
-					+ " on line " + e.line
-					+ " of language '" + language.getName() + "'. "
-					+ e.getMessage()
-				);
-			} catch (Exception | Error e) {
-				stop();
-				sendError(e.getClass().getSimpleName(), language.getId());
 
-				Logger.e(
+			start = System.currentTimeMillis();
-					logTag,
+			int lettersCount = importLetters(language);
-					"Failed loading dictionary: " + language.getDictionaryFile()
+			sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
-					+ " for language '" + language.getName() + "'. "
+			logLoadingStep("Letters imported", language, start);
-					+ e.getMessage()
+
-				);
+			start = System.currentTimeMillis();
-			}
+			InsertOps.restoreCustomWords(sqlite.getDb(), language);
-		});
+			sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
+			logLoadingStep("Custom words restored", language, start);
+
+			start = System.currentTimeMillis();
+			WordBatch words = readWordsFile(language, lettersCount, progress, progress + 25f);
+			progress += 25;
+			sendProgressMessage(language, progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
+			logLoadingStep("Dictionary file loaded in memory", language, start);
+
+			start = System.currentTimeMillis();
+			saveWordBatch(words, progress, dictionaryMaxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
+			progress = dictionaryMaxProgress;
+			sendProgressMessage(language, progress, 0);
+			logLoadingStep("Dictionary words saved in database", language, start);
+
+			start = System.currentTimeMillis();
+			Tables.createPositionIndex(sqlite.getDb(), language);
+			sendProgressMessage(language, progress + (100f - progress) / 2f, 0);
+			Tables.createWordIndex(sqlite.getDb(), language);
+			sendProgressMessage(language, 100, 0);
+			logLoadingStep("Indexes restored", language, start);
+
+			sqlite.finishTransaction();
+			SlowQueryStats.clear();
+		} catch (DictionaryImportAbortedException e) {
+			sqlite.failTransaction();
+			stop();
+			Logger.i(LOG_TAG, e.getMessage() + ". File '" + language.getDictionaryFile() + "' not imported.");
+		} catch (DictionaryImportException e) {
+			stop();
+			sqlite.failTransaction();
+			sendImportError(DictionaryImportException.class.getSimpleName(), language.getId(), e.line, e.word);
+
+			Logger.e(
+				LOG_TAG,
+				" Invalid word: '" + e.word
+				+ "' in dictionary: '" + language.getDictionaryFile() + "'"
+				+ " on line " + e.line
+				+ " of language '" + language.getName() + "'. "
+				+ e.getMessage()
+			);
+		} catch (Exception | Error e) {
+			stop();
+			sqlite.failTransaction();
+			sendError(e.getClass().getSimpleName(), language.getId());
+
+			Logger.e(
+				LOG_TAG,
+				"Failed loading dictionary: " + language.getDictionaryFile()
+				+ " for language '" + language.getName() + "'. "
+				+ e.getMessage()
+			);
+		}
 	}
 
 
-	private void importLetters(Language language) throws InvalidLanguageCharactersException {
+	private int importLetters(Language language) throws InvalidLanguageCharactersException, DictionaryImportAbortedException {
-		ArrayList<Word> letters = new ArrayList<>();
+		int lettersCount = 0;
-
 		boolean isEnglish = language.getLocale().equals(Locale.ENGLISH);
+		WordBatch letters = new WordBatch(language);
 
 		for (int key = 2; key <= 9; key++) {
 			for (String langChar : language.getKeyCharacters(key, false)) {
 				langChar = (isEnglish && langChar.equals("i")) ? langChar.toUpperCase(Locale.ENGLISH) : langChar;
-				letters.add(Word.create(language, langChar, 0));
+				letters.add(langChar, 0, key);
+				lettersCount++;
 			}
 		}
 
-		DictionaryDb.upsertWordsSync(letters);
+		saveWordBatch(letters, -1, -1, -1);
+
+		return lettersCount;
 	}
 
 
-	private void importWords(Language language, String dictionaryFile) throws Exception {
+	private WordBatch readWordsFile(Language language, int positionShift, float minProgress, float maxProgress) throws Exception {
-		sendProgressMessage(language, 1, 0);
+		int currentLine = 1;
+		int totalLines = getFileSize(language.getDictionaryFile());
+		float progressRatio = (maxProgress - minProgress) / totalLines;
 
-		long currentLine = 0;
+		WordBatch batch = new WordBatch(language, totalLines);
-		long totalLines = getFileSize(dictionaryFile);
 
-		BufferedReader br = new BufferedReader(new InputStreamReader(assets.open(dictionaryFile), StandardCharsets.UTF_8));
+		try (BufferedReader br = new BufferedReader(new InputStreamReader(assets.open(language.getDictionaryFile()), StandardCharsets.UTF_8))) {
-		ArrayList<Word> dbWords = new ArrayList<>();
+			for (String line; (line = br.readLine()) != null; currentLine++) {
+				if (loadThread.isInterrupted()) {
+					sendProgressMessage(language, 0, 0);
+					throw new DictionaryImportAbortedException();
+				}
 
-		for (String line; (line = br.readLine()) != null; currentLine++) {
+				String[] parts = splitLine(line);
-			if (loadThread.isInterrupted()) {
+				String word = parts[0];
-				br.close();
+				short frequency = getFrequency(parts);
-				sendProgressMessage(language, 0, 0);
-				throw new DictionaryImportAbortedException();
-			}
 
-			String[] parts = splitLine(line);
+				try {
-			String word = parts[0];
+					batch.add(word, frequency, currentLine + positionShift);
-			int frequency = getFrequency(parts);
+				} catch (InvalidLanguageCharactersException e) {
+					throw new DictionaryImportException(word, currentLine);
+				}
 
-			try {
+				if (totalLines > 0 && currentLine % SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE == 0) {
-				dbWords.add(Word.create(language, word, frequency));
+					sendProgressMessage(language, minProgress + progressRatio * currentLine, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
-			} catch (InvalidLanguageCharactersException e) {
+				}
-				br.close();
-				throw new DictionaryImportException(word, currentLine);
-			}
-
-			if (dbWords.size() >= settings.getDictionaryImportWordChunkSize() || currentLine >= totalLines - 1) {
-				DictionaryDb.upsertWordsSync(dbWords);
-				dbWords.clear();
-			}
-
-			if (totalLines > 0) {
-				int progress = (int) Math.floor(100.0 * currentLine / totalLines);
-				progress = Math.max(1, progress);
-				sendProgressMessage(language, progress, settings.getDictionaryImportProgressUpdateInterval());
 			}
 		}
 
-		br.close();
+		return batch;
-		sendProgressMessage(language, 100, 0);
+	}
+
+
+	public void saveWordBatch(WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
+		float middleProgress = minProgress + (maxProgress - minProgress) / 2;
+
+		InsertOps insertOps = new InsertOps(sqlite.getDb(), batch.getLanguage());
+
+		insertWordsBatch(insertOps, batch, minProgress, middleProgress - 2, sizeUpdateInterval);
+		insertWordPositionsBatch(insertOps, batch, middleProgress - 2, maxProgress - 2, sizeUpdateInterval);
+		InsertOps.insertLanguageMeta(sqlite.getDb(), batch.getLanguage().getId());
+
+		if (sizeUpdateInterval > 0) {
+			sendProgressMessage(batch.getLanguage(), maxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
+		}
+	}
+
+
+	private void insertWordsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
+		if (batch.getWords().size() == 0) {
+			return;
+		}
+
+		float progressRatio = (maxProgress - minProgress) / batch.getWords().size();
+
+		for (int progress = 0, end = batch.getWords().size(); progress < end; progress++) {
+			if (loadThread.isInterrupted()) {
+				sendProgressMessage(batch.getLanguage(), 0, 0);
+				throw new DictionaryImportAbortedException();
+			}
+
+			insertOps.insertWord(batch.getWords().get(progress));
+			if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) {
+				sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
+			}
+		}
+	}
+
+
+	private void insertWordPositionsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
+		if (batch.getPositions().size() == 0) {
+			return;
+		}
+
+		float progressRatio = (maxProgress - minProgress) / batch.getPositions().size();
+
+		for (int progress = 0, end = batch.getPositions().size(); progress < end; progress++) {
+			if (loadThread.isInterrupted()) {
+				sendProgressMessage(batch.getLanguage(), 0, 0);
+				throw new DictionaryImportAbortedException();
+			}
+
+			insertOps.insertWordPosition(batch.getPositions().get(progress));
+			if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) {
+				sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
+			}
+		}
 	}
 
 
@@ -241,21 +324,21 @@ public class DictionaryLoader {
 	}
 
 
-	private long getFileSize(String filename) {
+	private int getFileSize(String filename) {
 		String sizeFilename = filename + ".size";
 
 		try (BufferedReader reader = new BufferedReader(new InputStreamReader(assets.open(sizeFilename), StandardCharsets.UTF_8))) {
 			return Integer.parseInt(reader.readLine());
 		} catch (Exception e) {
-			Logger.w(logTag, "Could not read the size of: " + filename + " from:  " + sizeFilename + ". " + e.getMessage());
+			Logger.w(LOG_TAG, "Could not read the size of: " + filename + " from:  " + sizeFilename + ". " + e.getMessage());
 			return 0;
 		}
 	}
 
 
-	private int getFrequency(String[] lineParts) {
+	private short getFrequency(String[] lineParts) {
 		try {
-			return Integer.parseInt(lineParts[1]);
+			return Short.parseShort(lineParts[1]);
 		} catch (Exception e) {
 			return 0;
 		}
@@ -264,7 +347,7 @@ public class DictionaryLoader {
 
 	private void sendStartMessage(int fileCount) {
 		if (onStatusChange == null) {
-			Logger.w(logTag, "Cannot send file count without a status Handler. Ignoring message.");
+			Logger.w(LOG_TAG, "Cannot send file count without a status Handler. Ignoring message.");
 			return;
 		}
 
@@ -275,9 +358,9 @@ public class DictionaryLoader {
 	}
 
 
-	private void sendProgressMessage(Language language, int progress, int progressUpdateInterval) {
+	private void sendProgressMessage(Language language, float progress, int progressUpdateInterval) {
 		if (onStatusChange == null) {
-			Logger.w(logTag, "Cannot send progress without a status Handler. Ignoring message.");
+			Logger.w(LOG_TAG, "Cannot send progress without a status Handler. Ignoring message.");
 			return;
 		}
 
@@ -291,7 +374,7 @@ public class DictionaryLoader {
 		Bundle progressMsg = new Bundle();
 		progressMsg.putInt("languageId", language.getId());
 		progressMsg.putLong("time", getImportTime());
-		progressMsg.putInt("progress", progress);
+		progressMsg.putInt("progress", (int) Math.round(progress));
 		progressMsg.putInt("currentFile", currentFile);
 		asyncHandler.post(() -> onStatusChange.accept(progressMsg));
 	}
@@ -299,7 +382,7 @@ public class DictionaryLoader {
 
 	private void sendError(String message, int langId) {
 		if (onStatusChange == null) {
-			Logger.w(logTag, "Cannot send an error without a status Handler. Ignoring message.");
+			Logger.w(LOG_TAG, "Cannot send an error without a status Handler. Ignoring message.");
 			return;
 		}
 
@@ -312,7 +395,7 @@ public class DictionaryLoader {
 
 	private void sendImportError(String message, int langId, long fileLine, String word) {
 		if (onStatusChange == null) {
-			Logger.w(logTag, "Cannot send an import error without a status Handler. Ignoring message.");
+			Logger.w(LOG_TAG, "Cannot send an import error without a status Handler. Ignoring message.");
 			return;
 		}
 
@@ -323,4 +406,11 @@ public class DictionaryLoader {
 		errorMsg.putString("word", word);
 		asyncHandler.post(() -> onStatusChange.accept(errorMsg));
 	}
+
+
+	private void logLoadingStep(String message, Language language, long time) {
+		if (Logger.isDebugLevel()) {
+			Logger.d(LOG_TAG, message + " for language '" + language.getName() + "' in: " + (System.currentTimeMillis() - time) + " ms");
+		}
+	}
 }

src/io/github/sspanak/tt9/db/LegacyDb.java

@@ -0,0 +1,39 @@
+package io.github.sspanak.tt9.db;
+
+import android.app.Activity;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteOpenHelper;
+
+import io.github.sspanak.tt9.Logger;
+
+public class LegacyDb extends SQLiteOpenHelper {
+	private final String LOG_TAG = getClass().getSimpleName();
+	private static final String DB_NAME = "t9dict.db";
+	private static final String TABLE_NAME = "words";
+
+	private static boolean isCompleted = false;
+
+	public LegacyDb(Activity activity) {
+		super(activity.getApplicationContext(), DB_NAME, null, 12);
+	}
+
+	public void clear() {
+		if (isCompleted) {
+			return;
+		}
+
+		new Thread(() -> {
+			try (SQLiteDatabase db = getWritableDatabase()) {
+				db.compileStatement("DROP TABLE " + TABLE_NAME).execute();
+				Logger.d(LOG_TAG, "SQL Words cleaned successfully.");
+			} catch (Exception e) {
+				Logger.d(LOG_TAG, "Assuming no words, because of query error. " + e.getMessage());
+			} finally {
+				isCompleted = true;
+			}
+		}).start();
+	}
+
+	@Override public void onCreate(SQLiteDatabase db) {}
+	@Override public void onUpgrade(SQLiteDatabase db, int i, int ii) {}
+}

src/io/github/sspanak/tt9/db/SQLWords.java

@@ -1,78 +0,0 @@
-package io.github.sspanak.tt9.db;
-
-import android.app.Activity;
-import android.database.Cursor;
-import android.database.sqlite.SQLiteDatabase;
-
-import java.io.File;
-
-import io.github.sspanak.tt9.Logger;
-
-public class SQLWords {
-	private static final String DB_NAME = "t9dict.db";
-	private static final String TABLE_NAME = "words";
-
-	private static boolean isCompleted = false;
-
-	private final String LOG_TAG;
-	private final Activity activity;
-	private SQLiteDatabase db;
-
-	public SQLWords(Activity activity) {
-		this.activity = activity;
-
-		LOG_TAG = getClass().getSimpleName();
-	}
-
-	public void clear() {
-		if (isCompleted) {
-			return;
-		}
-
-		new Thread(() -> {
-			openDb();
-			if (areThereWords()) {
-				deleteAll();
-			}
-			closeDb();
-			isCompleted = true;
-		}).start();
-	}
-
-
-	private void openDb() {
-		try {
-			db = null;
-			File dbFile = activity.getDatabasePath(DB_NAME);
-			if (dbFile.exists()) {
-				db = SQLiteDatabase.openDatabase(dbFile.getAbsolutePath(), null, SQLiteDatabase.OPEN_READWRITE);
-			}
-		} catch (Exception e) {
-			Logger.d(LOG_TAG, "Assuming no SQL database, because of error while opening. " + e.getMessage());
-			db = null;
-		}
-	}
-
-
-	private void closeDb() {
-		if (db != null) {
-			db.close();
-		}
-	}
-
-
-	private boolean areThereWords() {
-		String sql = "SELECT COUNT(*) FROM (SELECT id FROM " + TABLE_NAME + " LIMIT 1)";
-		try (Cursor cursor = db.rawQuery(sql, null)) {
-			return cursor.moveToFirst() && cursor.getInt(0) > 0;
-		} catch (Exception e) {
-			Logger.d(LOG_TAG, "Assuming no words, because of query error. " + e.getMessage());
-			return false;
-		}
-	}
-
-	private void deleteAll() {
-		db.execSQL("DROP TABLE words");
-		Logger.d(LOG_TAG, "SQL Words cleaned successfully.");
-	}
-}

src/io/github/sspanak/tt9/db/SlowQueryStats.java

@@ -0,0 +1,87 @@
+package io.github.sspanak.tt9.db;
+
+import java.util.HashMap;
+
+import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.TextTools;
+import io.github.sspanak.tt9.languages.Language;
+import io.github.sspanak.tt9.preferences.SettingsStore;
+
+public class SlowQueryStats {
+	private static final String LOG_TAG = SlowQueryStats.class.getSimpleName();
+	private static long firstQueryTime = -1;
+	private static long maxQueryTime = 0;
+	private static long totalQueries = 0;
+	private static long totalQueryTime = 0;
+	private static final HashMap<String, Integer> slowQueries = new HashMap<>();
+	private static final HashMap<String, String> resultCache = new HashMap<>();
+
+
+	public static String generateKey(Language language, String sequence, String wordFilter, int minimumWords) {
+		return language.getId() + "_" + sequence + "_" + wordFilter + "_" + minimumWords;
+	}
+
+	public static void add(String key, int time, String positionsList) {
+		if (firstQueryTime == -1) {
+			firstQueryTime = System.currentTimeMillis();
+		}
+		maxQueryTime = Math.max(maxQueryTime, time);
+		totalQueries++;
+		totalQueryTime += time;
+		if (time < SettingsStore.SLOW_QUERY_TIME) {
+			return;
+		}
+
+		slowQueries.put(key, time);
+		if (!resultCache.containsKey(key)) {
+			resultCache.put(key, positionsList.replaceAll("-\\d+,", ""));
+		}
+	}
+
+	public static String getCachedIfSlow(String key) {
+		Integer queryTime = slowQueries.get(key);
+		boolean isSlow = queryTime != null && queryTime >= SettingsStore.SLOW_QUERY_TIME;
+
+		if (isSlow) {
+			Logger.d(LOG_TAG, "Loading cached positions for query: " + key);
+			return resultCache.get(key);
+		} else {
+			return null;
+		}
+	}
+
+	public static String getSummary() {
+		long slowQueryTotalTime = 0;
+		for (int time : slowQueries.values()) {
+			slowQueryTotalTime += time;
+		}
+
+		long averageTime = totalQueries == 0 ? 0 : totalQueryTime / totalQueries;
+		long slowAverageTime = slowQueries.size() == 0 ? 0 : slowQueryTotalTime / slowQueries.size();
+
+		return
+			"Queries: " + totalQueries + ". Average time: " + averageTime + " ms." +
+			"\nSlow: " + slowQueries.size() + ". Average time: " + slowAverageTime + " ms." +
+			"\nSlowest: " + maxQueryTime + " ms." +
+			"\nFirst: " + TextTools.unixTimestampToISODate(firstQueryTime);
+	}
+
+	public static String getList() {
+		StringBuilder sb = new StringBuilder();
+		for (String key : slowQueries.keySet()) {
+			sb.append(key).append(": ").append(slowQueries.get(key)).append(" ms\n");
+		}
+
+		return sb.toString();
+	}
+
+
+	public static void clear() {
+		firstQueryTime = -1;
+		maxQueryTime = 0;
+		totalQueries = 0;
+		totalQueryTime = 0;
+		slowQueries.clear();
+		resultCache.clear();
+	}
+}

src/io/github/sspanak/tt9/db/WordStore.java

@@ -0,0 +1,259 @@
+package io.github.sspanak.tt9.db;
+
+import android.content.Context;
+
+import androidx.annotation.NonNull;
+
+import java.util.ArrayList;
+
+import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.db.entities.Word;
+import io.github.sspanak.tt9.db.entities.WordList;
+import io.github.sspanak.tt9.db.sqlite.DeleteOps;
+import io.github.sspanak.tt9.db.sqlite.InsertOps;
+import io.github.sspanak.tt9.db.sqlite.ReadOps;
+import io.github.sspanak.tt9.db.sqlite.SQLiteOpener;
+import io.github.sspanak.tt9.db.sqlite.UpdateOps;
+import io.github.sspanak.tt9.ime.TraditionalT9;
+import io.github.sspanak.tt9.languages.Language;
+import io.github.sspanak.tt9.preferences.SettingsStore;
+import io.github.sspanak.tt9.ui.AddWordAct;
+
+
+public class WordStore {
+	private final String LOG_TAG = "sqlite.WordStore";
+	private static WordStore self;
+
+	private SQLiteOpener sqlite = null;
+	private ReadOps readOps = null;
+
+
+	public WordStore(@NonNull Context context) {
+		try {
+			sqlite = SQLiteOpener.getInstance(context);
+			sqlite.getDb();
+			readOps = new ReadOps();
+		} catch (Exception e) {
+			Logger.w(LOG_TAG, "Database connection failure. All operations will return empty results. " + e.getMessage());
+		}
+		self = this;
+	}
+
+
+	public static synchronized WordStore getInstance(Context context) {
+		if (self == null) {
+			context = context == null ? TraditionalT9.getMainContext() : context;
+			self = new WordStore(context);
+		}
+
+		return self;
+	}
+
+
+	/**
+	 * Loads words matching and similar to a given digit sequence
+	 * For example: "7655" -> "roll" (exact match), but also: "rolled", "roller", "rolling", ...
+	 * and other similar.
+	 */
+	public ArrayList<String> getSimilar(Language language, String sequence, String wordFilter, int minimumWords, int maximumWords) {
+		if (!checkOrNotify()) {
+			return new ArrayList<>();
+		}
+
+		if (sequence == null || sequence.length() == 0) {
+			Logger.w(LOG_TAG, "Attempting to get words for an empty sequence.");
+			return new ArrayList<>();
+		}
+
+		if (language == null) {
+			Logger.w(LOG_TAG, "Attempting to get words for NULL language.");
+			return new ArrayList<>();
+		}
+
+		final int minWords = Math.max(minimumWords, 0);
+		final int maxWords = Math.max(maximumWords, minWords);
+		final String filter = wordFilter == null ? "" : wordFilter;
+
+		long startTime = System.currentTimeMillis();
+		String positions = readOps.getSimilarWordPositions(sqlite.getDb(), language, sequence, filter, minWords);
+		long positionsTime = System.currentTimeMillis() - startTime;
+
+		startTime = System.currentTimeMillis();
+		ArrayList<String> words = readOps.getWords(sqlite.getDb(), language, positions, filter, maxWords, false).toStringList();
+		long wordsTime = System.currentTimeMillis() - startTime;
+
+		printLoadingSummary(sequence, words, positionsTime, wordsTime);
+		SlowQueryStats.add(SlowQueryStats.generateKey(language, sequence, wordFilter, minWords), (int) (positionsTime + wordsTime), positions);
+
+		return words;
+	}
+
+
+	public boolean exists(Language language) {
+		return language != null && checkOrNotify() && readOps.exists(sqlite.getDb(), language.getId());
+	}
+
+
+	public void remove(ArrayList<Integer> languageIds) {
+		if (!checkOrNotify()) {
+			return;
+		}
+
+		long start = System.currentTimeMillis();
+		try {
+			sqlite.beginTransaction();
+			for (int langId : languageIds) {
+				if (readOps.exists(sqlite.getDb(), langId)) {
+					DeleteOps.delete(sqlite, langId);
+				}
+			}
+			sqlite.finishTransaction();
+
+			Logger.d(LOG_TAG, "Deleted " + languageIds.size() + " languages. Time: " + (System.currentTimeMillis() - start) + " ms");
+		} catch (Exception e) {
+			sqlite.failTransaction();
+			Logger.e(LOG_TAG, "Failed deleting languages. " + e.getMessage());
+		}
+	}
+
+
+	public int put(Language language, String word) {
+		if (word == null || word.isEmpty()) {
+			return AddWordAct.CODE_BLANK_WORD;
+		}
+
+		if (language == null) {
+			return AddWordAct.CODE_INVALID_LANGUAGE;
+		}
+
+		if (!checkOrNotify()) {
+			return AddWordAct.CODE_GENERAL_ERROR;
+		}
+
+		try {
+			if (readOps.exists(sqlite.getDb(), language, word)) {
+				return AddWordAct.CODE_WORD_EXISTS;
+			}
+
+			String sequence = language.getDigitSequenceForWord(word);
+
+			if (InsertOps.insertCustomWord(sqlite.getDb(), language, sequence, word)) {
+				makeTopWord(language, word, sequence);
+			} else {
+				throw new Exception("SQLite INSERT failure.");
+			}
+		} catch (Exception e) {
+			String msg = "Failed inserting word: '" + word + "' for language: " + language.getId() + ". " + e.getMessage();
+			Logger.e("insertWord", msg);
+			return AddWordAct.CODE_GENERAL_ERROR;
+		}
+
+		return AddWordAct.CODE_SUCCESS;
+	}
+
+
+	private boolean checkOrNotify() {
+		if (sqlite == null || sqlite.getDb() == null) {
+			Logger.e(LOG_TAG, "No database connection. Cannot query any data.");
+			return false;
+		}
+
+		return true;
+	}
+
+
+	public void makeTopWord(@NonNull Language language, @NonNull String word, @NonNull String sequence) {
+		if (!checkOrNotify() || word.isEmpty() || sequence.isEmpty()) {
+			return;
+		}
+
+		try {
+			long start = System.currentTimeMillis();
+
+			String topWordPositions = readOps.getWordPositions(sqlite.getDb(), language, sequence, 0, 0, "");
+			WordList topWords = readOps.getWords(sqlite.getDb(), language, topWordPositions, "", 9999, true);
+			if (topWords.isEmpty()) {
+				throw new Exception("No such word");
+			}
+
+			Word topWord = topWords.get(0);
+			if (topWord.word.toUpperCase(language.getLocale()).equals(word.toUpperCase(language.getLocale()))) {
+				Logger.d(LOG_TAG, "Word '" + word + "' is already the top word. Time: " + (System.currentTimeMillis() - start) + " ms");
+				return;
+			}
+
+			int wordPosition = 0;
+			for (Word tw : topWords) {
+				if (tw.word.toUpperCase(language.getLocale()).equals(word.toUpperCase(language.getLocale()))) {
+					wordPosition = tw.position;
+					break;
+				}
+			}
+
+			int newTopFrequency = topWord.frequency + 1;
+			String wordFilter = word.length() == 1 ? word.toLowerCase(language.getLocale()) : null;
+			if (!UpdateOps.changeFrequency(sqlite.getDb(), language, wordFilter, wordPosition,  newTopFrequency)) {
+				throw new Exception("No such word");
+			}
+
+			if (newTopFrequency > SettingsStore.WORD_FREQUENCY_MAX) {
+				scheduleNormalization(language);
+			}
+
+			Logger.d(LOG_TAG, "Changed frequency of '" + word + "' to: " + newTopFrequency + ". Time: " + (System.currentTimeMillis() - start) + " ms");
+		} catch (Exception e) {
+			Logger.e(LOG_TAG,"Frequency change failed. Word: '" + word + "'. " + e.getMessage());
+		}
+	}
+
+
+	public void normalizeNext() {
+		if (!checkOrNotify()) {
+			return;
+		}
+
+		long start = System.currentTimeMillis();
+
+		try {
+			sqlite.beginTransaction();
+			int nextLangId = readOps.getNextInNormalizationQueue(sqlite.getDb());
+			UpdateOps.normalize(sqlite.getDb(), nextLangId);
+			sqlite.finishTransaction();
+
+			String message = nextLangId > 0 ? "Normalized language: " + nextLangId : "No languages to normalize";
+			Logger.d(LOG_TAG, message + ". Time: " + (System.currentTimeMillis() - start) + " ms");
+		} catch (Exception e) {
+			sqlite.failTransaction();
+			Logger.e(LOG_TAG, "Normalization failed. " + e.getMessage());
+		}
+	}
+
+
+	public void scheduleNormalization(Language language) {
+		if (language != null && checkOrNotify()) {
+			UpdateOps.scheduleNormalization(sqlite.getDb(), language);
+		}
+	}
+
+
+	private void printLoadingSummary(String sequence, ArrayList<String> words, long positionIndexTime, long wordsTime) {
+		if (!Logger.isDebugLevel()) {
+			return;
+		}
+
+		StringBuilder debugText = new StringBuilder("===== Word Loading Summary =====");
+		debugText
+			.append("\nWord Count: ").append(words.size())
+			.append(".\nTime: ").append(positionIndexTime + wordsTime)
+			.append(" ms (positions: ").append(positionIndexTime)
+			.append(" ms, words: ").append(wordsTime).append(" ms).");
+
+		if (words.isEmpty()) {
+			debugText.append(" Sequence: ").append(sequence);
+		} else {
+			debugText.append("\n").append(words);
+		}
+
+		Logger.d(LOG_TAG, debugText.toString());
+	}
+}

src/io/github/sspanak/tt9/db/WordStoreAsync.java

@@ -0,0 +1,66 @@
+package io.github.sspanak.tt9.db;
+
+import android.content.Context;
+import android.os.Handler;
+
+import androidx.annotation.NonNull;
+
+import java.util.ArrayList;
+
+import io.github.sspanak.tt9.ConsumerCompat;
+import io.github.sspanak.tt9.languages.Language;
+
+public class WordStoreAsync {
+	private static WordStore store;
+	private static final Handler asyncHandler = new Handler();
+
+	public static synchronized void init(Context context) {
+		store = WordStore.getInstance(context);
+	}
+
+
+	public static synchronized void init() {
+		init(null);
+	}
+
+
+	private static WordStore getStore() {
+		init();
+		return store;
+	}
+
+
+	public static void normalizeNext() {
+		new Thread(() -> getStore().normalizeNext()).start();
+	}
+
+
+	public static void areThereWords(ConsumerCompat<Boolean> notification, Language language) {
+		new Thread(() -> notification.accept(getStore().exists(language))).start();
+	}
+
+
+	public static void deleteWords(Runnable notification, @NonNull ArrayList<Integer> languageIds) {
+		new Thread(() -> {
+			getStore().remove(languageIds);
+			notification.run();
+		}).start();
+	}
+
+
+	public static void put(ConsumerCompat<Integer> statusHandler, Language language, String word) {
+		new Thread(() -> statusHandler.accept(getStore().put(language, word))).start();
+	}
+
+
+	public static void makeTopWord(@NonNull Language language, @NonNull String word, @NonNull String sequence) {
+		new Thread(() -> getStore().makeTopWord(language, word, sequence)).start();
+	}
+
+
+	public static void getWords(ConsumerCompat<ArrayList<String>> dataHandler, Language language, String sequence, String filter, int minWords, int maxWords) {
+		new Thread(() -> asyncHandler.post(() -> dataHandler.accept(
+			getStore().getSimilar(language, sequence, filter, minWords, maxWords)))
+		).start();
+	}
+}

src/io/github/sspanak/tt9/db/entities/Word.java

@@ -0,0 +1,18 @@
+package io.github.sspanak.tt9.db.entities;
+
+import androidx.annotation.NonNull;
+
+public class Word {
+	public int frequency;
+	public int position;
+	public String word;
+
+	public static Word create(@NonNull String word, int frequency, int position) {
+		Word w = new Word();
+		w.frequency = frequency;
+		w.position = position;
+		w.word = word;
+
+		return w;
+	}
+}

src/io/github/sspanak/tt9/db/entities/WordBatch.java

@@ -0,0 +1,61 @@
+package io.github.sspanak.tt9.db.entities;
+
+import androidx.annotation.NonNull;
+
+import java.util.ArrayList;
+
+import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException;
+import io.github.sspanak.tt9.languages.Language;
+
+public class WordBatch {
+	@NonNull private final Language language;
+	@NonNull private final ArrayList<Word> words;
+	@NonNull private final ArrayList<WordPosition> positions;
+
+	private WordPosition lastWordPosition;
+
+	public WordBatch(@NonNull Language language, int size) {
+		this.language = language;
+		words = size > 0 ? new ArrayList<>(size) : new ArrayList<>();
+		positions = size > 0 ? new ArrayList<>(size) : new ArrayList<>();
+	}
+
+	public WordBatch(@NonNull Language language) {
+		this(language, 0);
+	}
+
+	public void add(@NonNull String word, int frequency, int position) throws InvalidLanguageCharactersException {
+		words.add(Word.create(word, frequency, position));
+
+		if (position == 0) {
+			return;
+		}
+
+		String sequence = language.getDigitSequenceForWord(word);
+
+		if (position == 1 || lastWordPosition == null) {
+			lastWordPosition = WordPosition.create(sequence, position);
+		} else {
+			lastWordPosition.end = position;
+		}
+
+		if (!sequence.equals(lastWordPosition.sequence)) {
+			lastWordPosition.end--;
+			positions.add(lastWordPosition);
+
+			lastWordPosition = WordPosition.create(sequence, position);
+		}
+	}
+
+	@NonNull public Language getLanguage() {
+		return language;
+	}
+
+	@NonNull public ArrayList<Word> getWords() {
+		return words;
+	}
+
+	@NonNull public ArrayList<WordPosition> getPositions() {
+		return positions;
+	}
+}

src/io/github/sspanak/tt9/db/entities/WordList.java

@@ -0,0 +1,17 @@
+package io.github.sspanak.tt9.db.entities;
+
+import java.util.ArrayList;
+
+public class WordList extends ArrayList<Word> {
+	public void add(String word, int frequency, int position) {
+		add(Word.create(word, frequency, position));
+	}
+
+	public ArrayList<String> toStringList() {
+		ArrayList<String> list = new ArrayList<>(size());
+		for (Word word : this) {
+			list.add(word.word);
+		}
+		return list;
+	}
+}

src/io/github/sspanak/tt9/db/entities/WordPosition.java

@@ -0,0 +1,17 @@
+package io.github.sspanak.tt9.db.entities;
+
+import androidx.annotation.NonNull;
+
+public class WordPosition {
+	public String sequence;
+	public int start;
+	public int end;
+
+	public static WordPosition create(@NonNull String sequence, int start) {
+		WordPosition position = new WordPosition();
+		position.sequence = sequence;
+		position.start = start;
+
+		return position;
+	}
+}

src/io/github/sspanak/tt9/db/entities/WordPositionsStringBuilder.java

@@ -0,0 +1,38 @@
+package io.github.sspanak.tt9.db.entities;
+
+import android.database.Cursor;
+
+import androidx.annotation.NonNull;
+
+
+public class WordPositionsStringBuilder {
+	public int size = 0;
+	private final StringBuilder positions = new StringBuilder();
+
+	public WordPositionsStringBuilder appendFromDbRanges(Cursor cursor) {
+		while (cursor.moveToNext()) {
+			append(cursor.getInt(0), cursor.getInt(1));
+		}
+
+		return this;
+	}
+
+	private void append(int start, int end) {
+		if (size > 0) {
+			positions.append(",");
+		}
+		positions.append(start);
+		size++;
+
+		for (int position = start + 1; position <= end; position++) {
+			positions.append(",").append(position);
+			size++;
+		}
+	}
+
+	@NonNull
+	@Override
+	public String toString() {
+		return positions.toString();
+	}
+}

src/io/github/sspanak/tt9/db/exceptions/InsertBlankWordException.java

@@ -1,7 +0,0 @@
-package io.github.sspanak.tt9.db.exceptions;
-
-public class InsertBlankWordException extends Exception {
-	public InsertBlankWordException() {
-		super("Cannot insert a blank word.");
-	}
-}

src/io/github/sspanak/tt9/db/objectbox/Word.java

@@ -1,48 +0,0 @@
-package io.github.sspanak.tt9.db.objectbox;
-
-import androidx.annotation.NonNull;
-
-import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException;
-import io.github.sspanak.tt9.languages.Language;
-import io.objectbox.annotation.ConflictStrategy;
-import io.objectbox.annotation.Entity;
-import io.objectbox.annotation.Id;
-import io.objectbox.annotation.Index;
-import io.objectbox.annotation.Unique;
-
-@Entity
-public class Word {
-	@Id public long id;
-	public int frequency;
-	public boolean isCustom;
-	public int langId;
-	public int length;
-	@Index public String sequence;
-	@Index public byte sequenceShort; // up to 2 digits
-	@Unique(onConflict = ConflictStrategy.REPLACE) public String uniqueId;
-	public String word;
-
-	public static Word create(@NonNull Language language, @NonNull String word, int frequency) throws InvalidLanguageCharactersException {
-		Word w = new Word();
-		w.frequency = frequency;
-		w.isCustom = false;
-		w.langId = language.getId();
-		w.length = word.length();
-		w.sequence = language.getDigitSequenceForWord(word);
-		w.sequenceShort = shrinkSequence(w.sequence);
-		w.uniqueId = (language.getId() + "-" + word);
-		w.word = word;
-
-		return w;
-	}
-
-	public static Word create(@NonNull Language language, @NonNull String word, int frequency, boolean isCustom) throws InvalidLanguageCharactersException {
-		Word w = create(language, word, frequency);
-		w.isCustom = isCustom;
-		return w;
-	}
-
-	public static Byte shrinkSequence(@NonNull String sequence) {
-		return Byte.parseByte(sequence.substring(0, Math.min(2, sequence.length())));
-	}
-}

src/io/github/sspanak/tt9/db/objectbox/WordList.java

@@ -1,57 +0,0 @@
-package io.github.sspanak.tt9.db.objectbox;
-
-import androidx.annotation.NonNull;
-
-import java.util.ArrayList;
-import java.util.List;
-
-
-public class WordList extends ArrayList<Word> {
-	public WordList() {
-		super();
-	}
-
-
-	public WordList(@NonNull List<Word> words) {
-		addAll(words);
-	}
-
-
-	@NonNull
-	@Override
-	public String toString() {
-		StringBuilder sb = new StringBuilder();
-
-		for (int i = 0; i < size(); i++) {
-			sb
-				.append("word: ").append(get(i).word)
-				.append(" | sequence: ").append(get(i).sequence)
-				.append(" | priority: ").append(get(i).frequency)
-				.append("\n");
-		}
-
-		return sb.toString();
-	}
-
-
-	@NonNull
-	public WordList filter(int minLength, int minWords) {
-		WordList filtered = new WordList();
-		for (int i = 0; i < size(); i++) {
-			if (get(i).length == minLength || filtered.size() < minWords) {
-				filtered.add(get(i));
-			}
-		}
-		return filtered;
-	}
-
-
-	@NonNull
-	public ArrayList<String> toStringList() {
-		ArrayList<String> strings = new ArrayList<>();
-		for (int i = 0; i < size(); i++) {
-			strings.add(get(i).word);
-		}
-		return strings;
-	}
-}

src/io/github/sspanak/tt9/db/objectbox/WordStore.java

@@ -1,208 +0,0 @@
-package io.github.sspanak.tt9.db.objectbox;
-
-import android.content.Context;
-
-import androidx.annotation.NonNull;
-import androidx.annotation.Nullable;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import io.github.sspanak.tt9.Logger;
-import io.github.sspanak.tt9.languages.Language;
-import io.objectbox.Box;
-import io.objectbox.BoxStore;
-import io.objectbox.config.DebugFlags;
-import io.objectbox.query.Query;
-import io.objectbox.query.QueryBuilder;
-import io.objectbox.query.QueryCondition;
-
-public class WordStore {
-	private BoxStore boxStore;
-	private Box<Word> wordBox;
-
-	private Query<Word> longWordQuery;
-	private Query<Word> singleLetterQuery;
-
-
-	public WordStore(Context context) {
-		init(context);
-	}
-
-
-	private void init(Context context) {
-		boxStore = MyObjectBox.builder()
-			.androidContext(context.getApplicationContext())
-			.debugFlags(Logger.isDebugLevel() ? DebugFlags.LOG_QUERY_PARAMETERS : 0)
-			.build();
-
-		wordBox = boxStore.boxFor(Word.class);
-
-		longWordQuery = getLongWordQuery();
-		singleLetterQuery = getSingleLetterQuery();
-	}
-
-
-	private Query<Word> getLongWordQuery() {
-		return boxStore.boxFor(Word.class)
-			.query()
-				.equal(Word_.langId, 0)
-				.startsWith(Word_.sequence, "", QueryBuilder.StringOrder.CASE_SENSITIVE).parameterAlias("seq_start")
-				.lessOrEqual(Word_.sequence, "", QueryBuilder.StringOrder.CASE_SENSITIVE).parameterAlias("seq_end")
-				.equal(Word_.sequenceShort, 0)
-				.startsWith(Word_.word, "", QueryBuilder.StringOrder.CASE_SENSITIVE)
-			.order(Word_.length)
-			.orderDesc(Word_.frequency)
-			.build();
-	}
-
-	private Query<Word> getSingleLetterQuery() {
-		return wordBox
-			.query()
-				.equal(Word_.langId, 0)
-				.equal(Word_.sequenceShort, 0)
-				.startsWith(Word_.word, "", QueryBuilder.StringOrder.CASE_SENSITIVE)
-			.orderDesc(Word_.frequency)
-			.build();
-	}
-
-
-	public long count(int langId) {
-			try (Query<Word> query = wordBox.query(Word_.langId.equal(langId)).build()) {
-				return query.count();
-			} catch (Exception e) {
-				return 0;
-			}
-	}
-
-
-	public boolean exists(int langId, @NonNull String word, @NonNull String sequence) {
-		return get(langId, word, sequence) != null;
-	}
-
-
-	@Nullable
-	public Word get(int langId, @NonNull String word, @NonNull String sequence) {
-		QueryCondition<Word> where = Word_.langId.equal(langId)
-			.and(Word_.sequenceShort.equal(Word.shrinkSequence(sequence)))
-			.and(Word_.word.equal(word, QueryBuilder.StringOrder.CASE_SENSITIVE));
-
-		try (Query<Word> query = wordBox.query(where).build()) {
-			return query.findFirst();
-		}
-	}
-
-	@Nullable
-	public List<Word> getMany(int langId) {
-		try (Query<Word> query = wordBox.query(Word_.langId.equal(langId)).build()) {
-			return query.find();
-		}
-	}
-
-
-	@NonNull
-	public WordList getMany(Language language, @NonNull String sequence, @Nullable String filter, int maxWords) {
-		Query<Word> query;
-		if (sequence.length() < 2) {
-			singleLetterQuery.setParameter(Word_.sequenceShort, Byte.parseByte(sequence));
-			query = singleLetterQuery;
-		} else {
-			longWordQuery.setParameter(Word_.sequenceShort, Word.shrinkSequence(sequence));
-			longWordQuery.setParameter("seq_start", sequence);
-			longWordQuery.setParameter("seq_end", sequence + "99");
-			query = longWordQuery;
-		}
-
-		query.setParameter(Word_.langId, language.getId());
-
-		if (filter != null && !filter.equals("")) {
-			query.setParameter(Word_.word, filter);
-		} else {
-			query.setParameter(Word_.word, "");
-		}
-
-		return new WordList(query.find(0, maxWords));
-	}
-
-
-	public int[] getLanguages() {
-		try (Query<Word> query = wordBox.query().build()) {
-			return query.property(Word_.langId).distinct().findInts();
-		}
-	}
-
-
-	public int getMaxFrequency(int langId) {
-		return getMaxFrequency(langId, null, null);
-	}
-
-
-	public int getMaxFrequency(int langId, String sequence, String word) {
-		QueryCondition<Word> where = Word_.langId.equal(langId);
-
-		if (sequence != null && word != null) {
-			where = where.and(Word_.sequenceShort.equal(Word.shrinkSequence(sequence)))
-				.and(Word_.sequence.equal(sequence))
-				.and(Word_.word.notEqual(word));
-		}
-
-		try (Query<Word> query = wordBox.query(where).build()) {
-			long max = query.property(Word_.frequency).max();
-			return max == Long.MIN_VALUE ? 0 : (int)max;
-		}
-	}
-
-
-	public void put(@NonNull List<Word> words) {
-		wordBox.put(words);
-	}
-
-
-	public void put(@NonNull Word word) {
-		wordBox.put(word);
-	}
-
-
-	public WordStore removeMany(@NonNull ArrayList<Integer> languageIds) {
-		if (languageIds.size() > 0) {
-			try (Query<Word> query = wordBox.query(Word_.langId.oneOf(IntegerListToIntArray(languageIds))).build()) {
-				query.remove();
-			}
-		}
-
-		return this;
-	}
-
-
-	public void destroy() {
-		boxStore.closeThreadResources();
-		boxStore.close();
-		boxStore.deleteAllFiles();
-	}
-
-
-	public void runInTransaction(Runnable r) {
-		boxStore.runInTx(r);
-	}
-
-
-	public void runInTransactionAsync(Runnable action) {
-		boxStore.runInTxAsync(action, null);
-	}
-
-
-	public void closeThreadResources() {
-		boxStore.closeThreadResources();
-	}
-
-
-	private int[] IntegerListToIntArray(ArrayList<Integer> in) {
-		int[] out = new int[in.size()];
-		Iterator<Integer> iterator = in.iterator();
-		for (int i = 0; i < out.length; i++) {
-			out[i] = iterator.next();
-		}
-		return out;
-	}
-}

src/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java

@@ -0,0 +1,63 @@
+package io.github.sspanak.tt9.db.sqlite;
+
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteDoneException;
+import android.database.sqlite.SQLiteStatement;
+
+import androidx.annotation.NonNull;
+
+import java.util.HashMap;
+
+class CompiledQueryCache {
+	private static CompiledQueryCache self;
+	private final SQLiteDatabase db;
+	private final HashMap<Integer, SQLiteStatement> statements = new HashMap<>();
+
+	private CompiledQueryCache(@NonNull SQLiteDatabase db) {
+		this.db = db;
+	}
+
+	CompiledQueryCache execute(String sql) {
+		get(sql).execute();
+		return this;
+	}
+
+	SQLiteStatement get(@NonNull String sql) {
+		SQLiteStatement statement = statements.get(sql.hashCode());
+		if (statement == null) {
+			statement = db.compileStatement(sql);
+			statements.put(sql.hashCode(), statement);
+		}
+
+		return statement;
+	}
+
+	long simpleQueryForLong(String sql, long defaultValue) {
+		try {
+			return get(sql).simpleQueryForLong();
+		} catch (SQLiteDoneException e) {
+			return defaultValue;
+		}
+	}
+
+
+	static CompiledQueryCache getInstance(SQLiteDatabase db) {
+		if (self == null) {
+			self = new CompiledQueryCache(db);
+		}
+
+		return self;
+	}
+
+	static CompiledQueryCache execute(SQLiteDatabase db, String sql) {
+		return getInstance(db).execute(sql);
+	}
+
+	static SQLiteStatement get(SQLiteDatabase db, String sql) {
+		return getInstance(db).get(sql);
+	}
+
+	static long simpleQueryForLong(SQLiteDatabase db, String sql, long defaultValue) {
+		return getInstance(db).simpleQueryForLong(sql, defaultValue);
+	}
+}

src/io/github/sspanak/tt9/db/sqlite/DeleteOps.java

@@ -0,0 +1,10 @@
+package io.github.sspanak.tt9.db.sqlite;
+
+import androidx.annotation.NonNull;
+
+public class DeleteOps {
+	public static void delete(@NonNull SQLiteOpener sqlite, int languageId) {
+		sqlite.getDb().delete(Tables.getWords(languageId), null, null);
+		sqlite.getDb().delete(Tables.getWordPositions(languageId), null, null);
+	}
+}

src/io/github/sspanak/tt9/db/sqlite/InsertOps.java

@@ -0,0 +1,79 @@
+package io.github.sspanak.tt9.db.sqlite;
+
+import android.content.ContentValues;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteStatement;
+
+import androidx.annotation.NonNull;
+
+import io.github.sspanak.tt9.db.entities.Word;
+import io.github.sspanak.tt9.db.entities.WordPosition;
+import io.github.sspanak.tt9.languages.Language;
+
+
+public class InsertOps {
+	private final SQLiteStatement insertWordsQuery;
+	private final SQLiteStatement insertPositionsQuery;
+
+
+	public InsertOps(SQLiteDatabase db, @NonNull Language language) {
+		// super cache to avoid String concatenation in the dictionary loading loop
+		insertWordsQuery = CompiledQueryCache.get(db, "INSERT INTO " + Tables.getWords(language.getId()) + " (frequency, position, word) VALUES (?, ?, ?)");
+		insertPositionsQuery = CompiledQueryCache.get(db, "INSERT INTO " + Tables.getWordPositions(language.getId()) + " (sequence, `start`, `end`) VALUES (?, ?, ?)");
+	}
+
+
+	public void insertWord(Word word) {
+		insertWordsQuery.bindLong(1, word.frequency);
+		insertWordsQuery.bindLong(2, word.position);
+		insertWordsQuery.bindString(3, word.word);
+		insertWordsQuery.execute();
+	}
+
+
+	public void insertWordPosition(WordPosition position) {
+		insertPositionsQuery.bindString(1, position.sequence);
+		insertPositionsQuery.bindLong(2, position.start);
+		insertPositionsQuery.bindLong(3, position.end);
+		insertPositionsQuery.execute();
+	}
+
+
+	public static void insertLanguageMeta(@NonNull SQLiteDatabase db, int langId) {
+		SQLiteStatement query = CompiledQueryCache.get(db, "REPLACE INTO " + Tables.LANGUAGES_META + " (langId) VALUES (?)");
+		query.bindLong(1, langId);
+		query.execute();
+	}
+
+
+	public static boolean insertCustomWord(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String sequence, @NonNull String word) {
+		ContentValues values = new ContentValues();
+		values.put("langId", language.getId());
+		values.put("sequence", sequence);
+		values.put("word", word);
+
+		long insertId = db.insert(Tables.CUSTOM_WORDS, null, values);
+		if (insertId == -1) {
+			return false;
+		}
+
+		// If the user inserts more than 2^31 custom words, the "position" will overflow and will mess up
+		// the words table, but realistically it will never happen, so we don't bother preventing it.
+
+		values = new ContentValues();
+		values.put("position", (int)-insertId);
+		values.put("word", word);
+		insertId = db.insert(Tables.getWords(language.getId()), null, values);
+
+		return insertId != -1;
+	}
+
+
+	public static void restoreCustomWords(@NonNull SQLiteDatabase db, @NonNull Language language) {
+		CompiledQueryCache.execute(
+			db,
+			"INSERT INTO " + Tables.getWords(language.getId()) + " (position, word) " +
+				"SELECT -id, word FROM " + Tables.CUSTOM_WORDS + " WHERE langId = " + language.getId()
+		);
+	}
+}

src/io/github/sspanak/tt9/db/sqlite/ReadOps.java

@@ -0,0 +1,223 @@
+package io.github.sspanak.tt9.db.sqlite;
+
+import android.database.Cursor;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteDoneException;
+import android.database.sqlite.SQLiteStatement;
+
+import androidx.annotation.NonNull;
+
+import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.db.SlowQueryStats;
+import io.github.sspanak.tt9.db.entities.WordList;
+import io.github.sspanak.tt9.db.entities.WordPositionsStringBuilder;
+import io.github.sspanak.tt9.languages.Language;
+
+public class ReadOps {
+	private final String LOG_TAG = "ReadOperations";
+
+
+	/**
+	 * Checks if a word exists in the database for the given language (case-insensitive).
+	 */
+	public boolean exists(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String word) {
+		String lowercaseWord = word.toLowerCase(language.getLocale());
+		String uppercaseWord = word.toUpperCase(language.getLocale());
+
+		SQLiteStatement query = CompiledQueryCache.get(db, "SELECT COUNT(*) FROM " + Tables.getWords(language.getId()) + " WHERE word IN(?, ?, ?)");
+		query.bindString(1, word);
+		query.bindString(2, lowercaseWord);
+		query.bindString(3, uppercaseWord);
+		try {
+			return query.simpleQueryForLong() > 0;
+		} catch (SQLiteDoneException e) {
+			return false;
+		}
+	}
+
+
+	/**
+	 * Checks if language exists (has words) in the database.
+	 */
+	public boolean exists(@NonNull SQLiteDatabase db, int langId) {
+		return CompiledQueryCache.simpleQueryForLong(
+			db,
+			"SELECT COUNT(*) FROM " + Tables.getWords(langId),
+			0
+		) > 0;
+	}
+
+
+	@NonNull
+	public WordList getWords(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String positions, String filter, int maximumWords, boolean fullOutput) {
+		if (positions.isEmpty()) {
+			Logger.d(LOG_TAG, "No word positions. Not searching words.");
+			return new WordList();
+		}
+
+
+		String wordsQuery = getWordsQuery(language, positions, filter, maximumWords, fullOutput);
+		if (wordsQuery.isEmpty()) {
+			return new WordList();
+		}
+
+		WordList words = new WordList();
+		try (Cursor cursor = db.rawQuery(wordsQuery, null)) {
+			while (cursor.moveToNext()) {
+					words.add(
+						cursor.getString(0),
+						fullOutput ? cursor.getInt(1) : 0,
+						fullOutput ? cursor.getInt(2) : 0
+					);
+			}
+		}
+
+		return words;
+	}
+
+
+	public String getSimilarWordPositions(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String sequence, String wordFilter, int minPositions) {
+		int generations;
+
+		switch (sequence.length()) {
+			case 2:
+				generations = wordFilter.isEmpty() ? 1 : Integer.MAX_VALUE;
+				break;
+			case 3:
+			case 4:
+				generations = wordFilter.isEmpty() ? 2 : Integer.MAX_VALUE;
+				break;
+			default:
+				generations = Integer.MAX_VALUE;
+				break;
+		}
+
+		return getWordPositions(db, language, sequence, generations, minPositions, wordFilter);
+	}
+
+
+	@NonNull
+	public String getWordPositions(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String sequence, int generations, int minPositions, String wordFilter) {
+		if (sequence.length() == 1) {
+			return sequence;
+		}
+
+		WordPositionsStringBuilder positions = new WordPositionsStringBuilder();
+
+		String cachedFactoryPositions = SlowQueryStats.getCachedIfSlow(SlowQueryStats.generateKey(language, sequence, wordFilter, minPositions));
+		if (cachedFactoryPositions != null) {
+			String customWordPositions = getCustomWordPositions(db, language, sequence, generations);
+			return customWordPositions.isEmpty() ? cachedFactoryPositions : customWordPositions + "," + cachedFactoryPositions;
+		}
+
+		try (Cursor cursor = db.rawQuery(getPositionsQuery(language, sequence, generations), null)) {
+			positions.appendFromDbRanges(cursor);
+		}
+
+		if (positions.size < minPositions) {
+			Logger.d(LOG_TAG, "Not enough positions: " + positions.size + " < " + minPositions + ". Searching for more.");
+			try (Cursor cursor = db.rawQuery(getFactoryWordPositionsQuery(language, sequence, Integer.MAX_VALUE), null)) {
+				positions.appendFromDbRanges(cursor);
+			}
+		}
+
+		return positions.toString();
+	}
+
+
+
+	@NonNull private String getCustomWordPositions(@NonNull SQLiteDatabase db, Language language, String sequence, int generations) {
+		try (Cursor cursor = db.rawQuery(getCustomWordPositionsQuery(language, sequence, generations), null)) {
+			return new WordPositionsStringBuilder().appendFromDbRanges(cursor).toString();
+		}
+	}
+
+
+	private String getPositionsQuery(@NonNull Language language, @NonNull String sequence, int generations) {
+		return
+			"SELECT `start`, `end` FROM ( " +
+				getFactoryWordPositionsQuery(language, sequence, generations) +
+				") UNION " +
+				getCustomWordPositionsQuery(language, sequence, generations);
+	}
+
+
+
+	@NonNull private String getFactoryWordPositionsQuery(@NonNull Language language, @NonNull String sequence, int generations) {
+		StringBuilder sql = new StringBuilder("SELECT `start`, `end` FROM ")
+			.append(Tables.getWordPositions(language.getId()))
+			.append(" WHERE ");
+
+		if (generations >= 0 && generations < 10) {
+			sql.append(" sequence IN(").append(sequence);
+
+			int lastChild = (int)Math.pow(10, generations) - 1;
+
+			for (int seqEnd = 1; seqEnd <= lastChild; seqEnd++) {
+				if (seqEnd % 10 != 0) {
+					sql.append(",").append(sequence).append(seqEnd);
+				}
+			}
+
+			sql.append(")");
+		} else {
+			sql.append(" sequence = ").append(sequence).append(" OR sequence BETWEEN ").append(sequence).append("1 AND ").append(sequence).append("9");
+			sql.append(" ORDER BY `start` ");
+			sql.append(" LIMIT 100");
+		}
+
+		String positionsSql = sql.toString();
+		Logger.v(LOG_TAG, "Index SQL: " + positionsSql);
+		return positionsSql;
+	}
+
+
+	@NonNull private String getCustomWordPositionsQuery(@NonNull Language language, @NonNull String sequence, int generations) {
+		String sql = "SELECT -id as `start`, -id as `end` FROM " + Tables.CUSTOM_WORDS +
+			" WHERE langId = " + language.getId() +
+			" AND (sequence = " + sequence;
+
+		if (generations > 0) {
+			sql += " OR sequence BETWEEN " + sequence + "1 AND " + sequence + "9)";
+		} else {
+			sql += ")";
+		}
+
+		Logger.v(LOG_TAG, "Custom words SQL: " + sql);
+		return sql;
+	}
+
+
+	@NonNull private String getWordsQuery(@NonNull Language language, @NonNull String positions, @NonNull String filter, int maxWords, boolean fullOutput) {
+		StringBuilder sql = new StringBuilder();
+		sql
+			.append("SELECT word");
+		if (fullOutput) {
+			sql.append(",frequency,position");
+		}
+
+		sql.append(" FROM ").append(Tables.getWords(language.getId()))
+			.append(" WHERE position IN(").append(positions).append(")");
+
+		if (!filter.isEmpty()) {
+			sql.append(" AND word LIKE '").append(filter.replaceAll("'", "''")).append("%'");
+		}
+
+		sql
+			.append(" ORDER BY LENGTH(word), frequency DESC")
+			.append(" LIMIT ").append(maxWords);
+
+		String wordsSql = sql.toString();
+		Logger.v(LOG_TAG, "Words SQL: " + wordsSql);
+		return wordsSql;
+	}
+
+
+	public int getNextInNormalizationQueue(@NonNull SQLiteDatabase db) {
+		return (int) CompiledQueryCache.simpleQueryForLong(
+			db,
+			"SELECT langId FROM " + Tables.LANGUAGES_META + " WHERE normalizationPending = 1 LIMIT 1",
+			-1
+		);
+	}
+}

src/io/github/sspanak/tt9/db/sqlite/SQLiteOpener.java

@@ -0,0 +1,85 @@
+package io.github.sspanak.tt9.db.sqlite;
+
+import android.content.Context;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteOpenHelper;
+
+import java.util.ArrayList;
+
+import io.github.sspanak.tt9.languages.Language;
+import io.github.sspanak.tt9.languages.LanguageCollection;
+
+public class SQLiteOpener extends SQLiteOpenHelper {
+	private static final String DATABASE_NAME = "tt9.db";
+	private static final int DATABASE_VERSION = 1;
+	private static SQLiteOpener self;
+
+	private final ArrayList<Language> allLanguages;
+	private SQLiteDatabase db;
+
+
+	public SQLiteOpener(Context context) {
+		super(context, DATABASE_NAME, null, DATABASE_VERSION);
+		allLanguages = LanguageCollection.getAll(context);
+	}
+
+
+	public static SQLiteOpener getInstance(Context context) {
+		if (self == null) {
+			self = new SQLiteOpener(context);
+		}
+
+		return self;
+	}
+
+
+	@Override
+	public void onCreate(SQLiteDatabase db) {
+		for (String query : Tables.getCreateQueries(allLanguages)) {
+			db.execSQL(query);
+		}
+	}
+
+
+	@Override
+	public void onConfigure(SQLiteDatabase db) {
+		super.onConfigure(db);
+		setWriteAheadLoggingEnabled(true);
+	}
+
+
+	@Override
+	public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+		// No migrations as of now
+	}
+
+
+	public SQLiteDatabase getDb() {
+		if (db == null) {
+			db = getWritableDatabase();
+		}
+		return db;
+	}
+
+
+	public void beginTransaction() {
+		if (db != null) {
+			db.beginTransactionNonExclusive();
+		}
+	}
+
+
+	public void failTransaction() {
+		if (db != null) {
+			db.endTransaction();
+		}
+	}
+
+
+	public void finishTransaction() {
+		if (db != null) {
+			db.setTransactionSuccessful();
+			db.endTransaction();
+		}
+	}
+}

src/io/github/sspanak/tt9/db/sqlite/Tables.java

@@ -0,0 +1,111 @@
+package io.github.sspanak.tt9.db.sqlite;
+
+import android.database.sqlite.SQLiteDatabase;
+
+import androidx.annotation.NonNull;
+
+import java.util.ArrayList;
+
+import io.github.sspanak.tt9.languages.Language;
+
+public class Tables {
+
+	static final String LANGUAGES_META = "languages_meta";
+	static final String CUSTOM_WORDS = "custom_words";
+	private static final String POSITIONS_TABLE_BASE_NAME = "word_positions_";
+	private static final String WORDS_TABLE_BASE_NAME = "words_";
+
+	static String getWords(int langId) { return WORDS_TABLE_BASE_NAME + langId; }
+	static String getWordPositions(int langId) { return POSITIONS_TABLE_BASE_NAME + langId; }
+
+
+	static String[] getCreateQueries(ArrayList<Language> languages) {
+		int languageCount = languages.size();
+		String[] queries = new String[languageCount * 4 + 3];
+
+		queries[0] = createCustomWords();
+		queries[1] = createCustomWordsIndex();
+		queries[2] = createLanguagesMeta();
+
+		int queryId = 3;
+		for (Language language : languages) {
+			queries[queryId++] = createWordsTable(language.getId());
+			queries[queryId++] = createWordsIndex(language.getId());
+			queries[queryId++] = createWordPositions(language.getId());
+			queries[queryId++] = createWordsPositionsIndex(language.getId());
+		}
+
+		return queries;
+	}
+
+
+	public static void createWordIndex(@NonNull SQLiteDatabase db, @NonNull Language language) {
+		CompiledQueryCache.execute(db, createWordsIndex(language.getId()));
+	}
+
+	public static void createPositionIndex(@NonNull SQLiteDatabase db, @NonNull Language language) {
+		CompiledQueryCache.execute(db, createWordsPositionsIndex(language.getId()));
+	}
+
+
+	public static void dropIndexes(@NonNull SQLiteDatabase db, @NonNull Language language) {
+		CompiledQueryCache
+			.execute(db, dropWordsIndex(language.getId()))
+			.execute(dropWordPositionsIndex(language.getId()));
+	}
+
+
+	private static String createWordsTable(int langId) {
+		return
+			"CREATE TABLE IF NOT EXISTS " + getWords(langId) + " (" +
+				"frequency INTEGER NOT NULL DEFAULT 0, " +
+				"position INTEGER NOT NULL, " +
+				"word TEXT NOT NULL" +
+			")";
+	}
+
+	private static String createWordsIndex(int langId) {
+		return "CREATE INDEX IF NOT EXISTS idx_position_" + langId + " ON " + getWords(langId) + " (position, word)";
+	}
+
+	private static String dropWordsIndex(int langId) {
+		return "DROP INDEX IF EXISTS idx_position_" + langId;
+	}
+
+	private static String createWordPositions(int langId) {
+		return
+			"CREATE TABLE IF NOT EXISTS " + getWordPositions(langId) + " (" +
+				"sequence TEXT NOT NULL, " +
+				"start INTEGER NOT NULL, " +
+				"end INTEGER NOT NULL" +
+			")";
+	}
+
+	private static String createWordsPositionsIndex(int langId) {
+		return "CREATE INDEX IF NOT EXISTS idx_sequence_start_" + langId + " ON " + getWordPositions(langId) + " (sequence, `start`)";
+	}
+
+	private static String dropWordPositionsIndex(int langId) {
+		return "DROP INDEX IF EXISTS idx_sequence_start_" + langId;
+	}
+
+	private static String createCustomWords() {
+		return "CREATE TABLE IF NOT EXISTS " + CUSTOM_WORDS + " (" +
+			"id INTEGER PRIMARY KEY, " +
+			"langId INTEGER NOT NULL, " +
+			"sequence TEXT NOT NULL, " +
+			"word INTEGER NOT NULL " +
+		")";
+	}
+
+	private static String createCustomWordsIndex() {
+		return "CREATE INDEX IF NOT EXISTS idx_langId_sequence ON " + CUSTOM_WORDS + " (langId, sequence)";
+	}
+
+	private static String createLanguagesMeta() {
+		return "CREATE TABLE IF NOT EXISTS " + LANGUAGES_META + " (" +
+			"langId INTEGER UNIQUE NOT NULL, " +
+			"normalizationPending INT2 NOT NULL DEFAULT 0 " +
+		")";
+	}
+}

src/io/github/sspanak/tt9/db/sqlite/UpdateOps.java

@@ -0,0 +1,59 @@
+package io.github.sspanak.tt9.db.sqlite;
+
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteStatement;
+
+import androidx.annotation.NonNull;
+
+import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.languages.Language;
+import io.github.sspanak.tt9.preferences.SettingsStore;
+
+
+public class UpdateOps {
+	private static final String LOG_TAG = UpdateOps.class.getSimpleName();
+
+
+	public static boolean changeFrequency(@NonNull SQLiteDatabase db, @NonNull Language language, String word, int position, int frequency) {
+		String sql = "UPDATE " + Tables.getWords(language.getId()) + " SET frequency = ? WHERE position = ?";
+
+		if (word != null && !word.isEmpty()) {
+			sql += " AND word = ?";
+		}
+
+		SQLiteStatement query = CompiledQueryCache.get(db, sql);
+		query.bindLong(1, frequency);
+		query.bindLong(2, position);
+		if (word != null && !word.isEmpty()) {
+			query.bindString(3, word);
+		}
+
+		Logger.v(LOG_TAG, "Change frequency SQL: " + query + "; (" + frequency + ", " + position + ", " + word + ")");
+
+		return query.executeUpdateDelete() > 0;
+	}
+
+
+	public static void normalize(@NonNull SQLiteDatabase db, int langId) {
+		if (langId <= 0) {
+			return;
+		}
+
+		SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.getWords(langId) + " SET frequency = frequency / ?");
+		query.bindLong(1, SettingsStore.WORD_FREQUENCY_NORMALIZATION_DIVIDER);
+		query.execute();
+
+		query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET normalizationPending = ? WHERE langId = ?");
+		query.bindLong(1, 0);
+		query.bindLong(2, langId);
+		query.execute();
+	}
+
+
+	public static void scheduleNormalization(@NonNull SQLiteDatabase db, @NonNull Language language) {
+		SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET normalizationPending = ? WHERE langId = ?");
+		query.bindLong(1, 1);
+		query.bindLong(2, language.getId());
+		query.execute();
+	}
+}

src/io/github/sspanak/tt9/ime/EmptyDatabaseWarning.java

@@ -5,22 +5,19 @@ import android.content.Context;
 import java.util.HashMap;
 
 import io.github.sspanak.tt9.R;
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
 import io.github.sspanak.tt9.languages.Language;
 import io.github.sspanak.tt9.languages.LanguageCollection;
 import io.github.sspanak.tt9.preferences.SettingsStore;
 import io.github.sspanak.tt9.ui.UI;
 
 public class EmptyDatabaseWarning {
-	final int WARNING_INTERVAL;
 	private static final HashMap<Integer, Long> warningDisplayedTime = new HashMap<>();
 
 	private Context context;
 	private Language language;
 
-	public EmptyDatabaseWarning(SettingsStore settings) {
+	public EmptyDatabaseWarning() {
-		WARNING_INTERVAL = settings.getDictionaryMissingWarningInterval();
-
 		for (Language lang : LanguageCollection.getAll(context)) {
 			if (!warningDisplayedTime.containsKey(lang.getId())) {
 				warningDisplayedTime.put(lang.getId(), 0L);
@@ -33,7 +30,7 @@ public class EmptyDatabaseWarning {
 		this.language = language;
 
 		if (isItTimeAgain(TraditionalT9.getMainContext())) {
-			DictionaryDb.areThereWords(this::show, language);
+			WordStoreAsync.areThereWords(this::show, language);
 		}
 	}
 
@@ -44,7 +41,7 @@ public class EmptyDatabaseWarning {
 
 		long now = System.currentTimeMillis();
 		Long lastWarningTime = warningDisplayedTime.get(language.getId());
-		return lastWarningTime != null && now - lastWarningTime > WARNING_INTERVAL;
+		return lastWarningTime != null && now - lastWarningTime > SettingsStore.DICTIONARY_MISSING_WARNING_INTERVAL;
 	}
 
 	private void show(boolean areThereWords) {

src/io/github/sspanak/tt9/ime/TraditionalT9.java

@@ -18,7 +18,7 @@ import java.util.List;
 
 import io.github.sspanak.tt9.Logger;
 import io.github.sspanak.tt9.R;
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
 import io.github.sspanak.tt9.ime.helpers.AppHacks;
 import io.github.sspanak.tt9.ime.helpers.InputModeValidator;
 import io.github.sspanak.tt9.ime.helpers.InputType;
@@ -41,6 +41,7 @@ public class TraditionalT9 extends KeyPadHandler {
 	@NonNull private TextField textField = new TextField(null, null);
 	@NonNull private InputType inputType = new InputType(null, null);
 	@NonNull private final Handler autoAcceptHandler = new Handler(Looper.getMainLooper());
+	@NonNull private final Handler normalizationHandler = new Handler(Looper.getMainLooper());
 
 	// input mode
 	private ArrayList<Integer> allowedInputModes = new ArrayList<>();
@@ -149,8 +150,7 @@ public class TraditionalT9 extends KeyPadHandler {
 		self = this;
 		Logger.enableDebugLevel(settings.getDebugLogsEnabled());
 
-		DictionaryDb.init(this);
+		WordStoreAsync.init(this);
-		DictionaryDb.normalizeWordFrequencies(settings);
 
 		if (mainView == null) {
 			mainView = new MainView(this);
@@ -219,6 +219,7 @@ public class TraditionalT9 extends KeyPadHandler {
 			return;
 		}
 
+		normalizationHandler.removeCallbacksAndMessages(null);
 		initUi();
 		updateInputViewShown();
 	}
@@ -234,6 +235,9 @@ public class TraditionalT9 extends KeyPadHandler {
 		onFinishTyping();
 		clearSuggestions();
 		statusBar.setText("--");
+
+		normalizationHandler.removeCallbacksAndMessages(null);
+		normalizationHandler.postDelayed(WordStoreAsync::normalizeNext, SettingsStore.WORD_NORMALIZATION_DELAY);
 	}
 
 

src/io/github/sspanak/tt9/ime/modes/ModePredictive.java

@@ -6,7 +6,7 @@ import java.util.ArrayList;
 
 import io.github.sspanak.tt9.Logger;
 import io.github.sspanak.tt9.TextTools;
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
 import io.github.sspanak.tt9.ime.helpers.InputType;
 import io.github.sspanak.tt9.ime.helpers.TextField;
 import io.github.sspanak.tt9.ime.modes.helpers.AutoSpace;
@@ -271,7 +271,7 @@ public class ModePredictive extends InputMode {
 			// emoji and punctuation are not in the database, so there is no point in
 			// running queries that would update nothing
 			if (!sequence.startsWith("11") && !sequence.equals("1") && !sequence.startsWith("0")) {
-				DictionaryDb.incrementWordFrequency(language, currentWord, sequence);
+				WordStoreAsync.makeTopWord(language, currentWord, sequence);
 			}
 		} catch (Exception e) {
 			Logger.e(LOG_TAG, "Failed incrementing priority of word: '" + currentWord + "'. " + e.getMessage());

src/io/github/sspanak/tt9/ime/modes/helpers/Predictions.java

@@ -3,7 +3,7 @@ package io.github.sspanak.tt9.ime.modes.helpers;
 import java.util.ArrayList;
 import java.util.regex.Pattern;
 
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
 import io.github.sspanak.tt9.ime.EmptyDatabaseWarning;
 import io.github.sspanak.tt9.languages.Characters;
 import io.github.sspanak.tt9.languages.Language;
@@ -32,7 +32,7 @@ public class Predictions {
 
 
 	public Predictions(SettingsStore settingsStore) {
-		emptyDbWarning = new EmptyDatabaseWarning(settingsStore);
+		emptyDbWarning = new EmptyDatabaseWarning();
 		settings = settingsStore;
 
 		// digitSequence limiter when selecting emoji
@@ -128,13 +128,13 @@ public class Predictions {
 		if (loadStatic()) {
 			onWordsChanged.run();
 		} else {
-			DictionaryDb.getWords(
+			WordStoreAsync.getWords(
 				(words) -> onDbWords(words, true),
 				language,
 				digitSequence,
 				stem,
-				settings.getSuggestionsMin(),
+				SettingsStore.SUGGESTIONS_MIN,
-				settings.getSuggestionsMax()
+				SettingsStore.SUGGESTIONS_MAX
 			);
 		}
 	}
@@ -176,7 +176,7 @@ public class Predictions {
 	}
 
 	private void loadWithoutLeadingPunctuation() {
-		DictionaryDb.getWords(
+		WordStoreAsync.getWords(
 			(dbWords) -> {
 				char firstChar = inputWord.charAt(0);
 				for (int i = 0; i < dbWords.size(); i++) {
@@ -187,8 +187,8 @@ public class Predictions {
 			language,
 			digitSequence.substring(1),
 			stem.length() > 1 ? stem.substring(1) : "",
-			settings.getSuggestionsMin(),
+			SettingsStore.SUGGESTIONS_MIN,
-			settings.getSuggestionsMax()
+			SettingsStore.SUGGESTIONS_MAX
 		);
 	}
 

src/io/github/sspanak/tt9/languages/Language.java

@@ -101,7 +101,6 @@ public class Language {
 		return keyChars;
 	}
 
-
 	final public int getId() {
 		if (id == 0) {
 			id = generateId();
@@ -172,8 +171,8 @@ public class Language {
 	/**
 	 * generateId
 	 * Uses the letters of the Locale to generate an ID for the language.
-	 * Each letter is converted to uppercase and used as n 5-bit integer. Then the the 5-bits
+	 * Each letter is converted to uppercase and used as a 5-bit integer. Then the 5-bits
-	 * are packed to form a 10-bit or a 20-bit integer, depending on the Locale.
+	 * are packed to form a 10-bit or a 20-bit integer, depending on the Locale length.
 	 *
 	 * Example (2-letter Locale)
 	 * 	"en"
@@ -186,12 +185,14 @@ public class Language {
 	 * 	-> "B" | "G" | "B" | "G"
 	 * 	-> 2 | 224 | 2048 | 229376 (shift each 5-bit number, not overlap with the previous ones)
 	 *	-> 231650
+	 *
+	 * Maximum ID is: "zz-ZZ" -> 879450
 	 */
 	private int generateId() {
 		String idString = (locale.getLanguage() + locale.getCountry()).toUpperCase();
 		int idInt = 0;
 		for (int i = 0; i < idString.length(); i++) {
-			idInt |= ((idString.charAt(i) & 31) << (i * 5));
+			idInt |= ((idString.codePointAt(i) & 31) << (i * 5));
 		}
 
 		return idInt;

src/io/github/sspanak/tt9/languages/LanguageCollection.java

@@ -95,7 +95,6 @@ public class LanguageCollection {
 		return getAll(context,false);
 	}
 
-
 	public static String toString(ArrayList<Language> list) {
 		StringBuilder stringList = new StringBuilder();
 		int listSize = list.size();

src/io/github/sspanak/tt9/preferences/PreferencesActivity.java

@@ -15,9 +15,9 @@ import androidx.preference.PreferenceFragmentCompat;
 
 import io.github.sspanak.tt9.Logger;
 import io.github.sspanak.tt9.R;
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
 import io.github.sspanak.tt9.db.DictionaryLoader;
-import io.github.sspanak.tt9.db.SQLWords;
+import io.github.sspanak.tt9.db.LegacyDb;
 import io.github.sspanak.tt9.ime.helpers.GlobalKeyboardSettings;
 import io.github.sspanak.tt9.ime.helpers.InputModeValidator;
 import io.github.sspanak.tt9.preferences.helpers.Hotkeys;
@@ -28,6 +28,7 @@ import io.github.sspanak.tt9.preferences.screens.HotkeysScreen;
 import io.github.sspanak.tt9.preferences.screens.KeyPadScreen;
 import io.github.sspanak.tt9.preferences.screens.MainSettingsScreen;
 import io.github.sspanak.tt9.preferences.screens.SetupScreen;
+import io.github.sspanak.tt9.preferences.screens.UsageStatsScreen;
 import io.github.sspanak.tt9.ui.DictionaryLoadingBar;
 
 public class PreferencesActivity extends AppCompatActivity implements PreferenceFragmentCompat.OnPreferenceStartFragmentCallback {
@@ -42,9 +43,8 @@ public class PreferencesActivity extends AppCompatActivity implements Preference
 		applyTheme();
 		Logger.enableDebugLevel(settings.getDebugLogsEnabled());
 
-		new SQLWords(this).clear();
+		try (LegacyDb db = new LegacyDb(this)) { db.clear(); }
-		DictionaryDb.init(this);
+		WordStoreAsync.init(this);
-		DictionaryDb.normalizeWordFrequencies(settings);
 
 		InputModeValidator.validateEnabledLanguages(this, settings.getEnabledLanguageIds());
 		validateFunctionKeys();
@@ -99,6 +99,8 @@ public class PreferencesActivity extends AppCompatActivity implements Preference
 				return new KeyPadScreen(this);
 			case "Setup":
 				return new SetupScreen(this);
+			case "SlowQueries":
+				return new UsageStatsScreen(this);
 			default:
 				return new MainSettingsScreen(this);
 		}

src/io/github/sspanak/tt9/preferences/SettingsStore.java

@@ -272,21 +272,19 @@ public class SettingsStore {
 
 	public boolean getDebugLogsEnabled() { return prefs.getBoolean("pref_enable_debug_logs", Logger.isDebugLevel()); }
 
-	public int getDictionaryImportProgressUpdateInterval() { return 250; /* ms */ }
+	public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE = 1000; // items
-	public int getDictionaryImportWordChunkSize() { return 1000; /* words */ }
+	public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME = 250; // ms
-
+	public final static int DICTIONARY_MISSING_WARNING_INTERVAL = 30000; // ms
-	public int getDictionaryMissingWarningInterval() { return 30000; /* ms */ }
+	public final static int PREFERENCES_CLICK_DEBOUNCE_TIME = 250; // ms
-
+	public final static byte SLOW_QUERY_TIME = 50; // ms
-	public int getSuggestionsMax() { return 20; }
+	public final static int SOFT_KEY_REPEAT_DELAY = 40; // ms
-	public int getSuggestionsMin() { return 8; }
+	public final static int SUGGESTIONS_MAX = 20;
-
+	public final static int SUGGESTIONS_MIN = 8;
-	public int getSuggestionSelectAnimationDuration() { return 66; }
+	public final static int SUGGESTIONS_SELECT_ANIMATION_DURATION = 66;
-	public int getSuggestionTranslateAnimationDuration() { return 0; }
+	public final static int SUGGESTIONS_TRANSLATE_ANIMATION_DURATION = 0;
-
+	public final static int WORD_FREQUENCY_MAX = 25500;
-	public int getSoftKeyRepeatDelay() { return 40; /* ms */ }
+	public final static int WORD_FREQUENCY_NORMALIZATION_DIVIDER = 100; // normalized frequency = WORD_FREQUENCY_MAX / WORD_FREQUENCY_NORMALIZATION_DIVIDER
-
+	public final static int WORD_NORMALIZATION_DELAY = 120000; // ms
-	public int getWordFrequencyMax() { return 25500; }
-	public int getWordFrequencyNormalizationDivider() { return 100; } // normalized frequency = getWordFrequencyMax() / getWordFrequencyNormalizationDivider()
 
 
 	/************* hack settings *************/

src/io/github/sspanak/tt9/preferences/items/ItemClickable.java

@@ -6,9 +6,9 @@ import java.util.ArrayList;
 import java.util.List;
 
 import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.preferences.SettingsStore;
 
 abstract public class ItemClickable {
-	private final int CLICK_DEBOUNCE_TIME = 250;
 	private long lastClickTime = 0;
 
 	protected final Preference item;
@@ -69,7 +69,7 @@ abstract public class ItemClickable {
 	 */
 	protected boolean debounceClick(Preference p) {
 		long now = System.currentTimeMillis();
-		if (now - lastClickTime < CLICK_DEBOUNCE_TIME) {
+		if (now - lastClickTime < SettingsStore.PREFERENCES_CLICK_DEBOUNCE_TIME) {
 			Logger.d("debounceClick", "Preference click debounced.");
 			return true;
 		}

src/io/github/sspanak/tt9/preferences/items/ItemDropDown.java

@@ -60,13 +60,11 @@ public class ItemDropDown {
 		}
 	}
 
-	public ItemDropDown preview() {
+	public void preview() {
 		try {
 			setPreview(values.get(Integer.parseInt(item.getValue())));
 		} catch (NumberFormatException e) {
 			setPreview("");
 		}
-
-		return this;
 	}
 }

src/io/github/sspanak/tt9/preferences/items/ItemTruncateAll.java

@@ -2,9 +2,13 @@ package io.github.sspanak.tt9.preferences.items;
 
 import androidx.preference.Preference;
 
+import java.util.ArrayList;
+
 import io.github.sspanak.tt9.R;
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
 import io.github.sspanak.tt9.db.DictionaryLoader;
+import io.github.sspanak.tt9.languages.Language;
+import io.github.sspanak.tt9.languages.LanguageCollection;
 import io.github.sspanak.tt9.preferences.PreferencesActivity;
 import io.github.sspanak.tt9.ui.UI;
 
@@ -30,7 +34,11 @@ public class ItemTruncateAll extends ItemClickable {
 		}
 
 		onStartDeleting();
-		DictionaryDb.deleteWords(activity.getApplicationContext(), this::onFinishDeleting);
+		ArrayList<Integer> languageIds = new ArrayList<>();
+		for (Language lang : LanguageCollection.getAll(activity, false)) {
+			languageIds.add(lang.getId());
+		}
+		WordStoreAsync.deleteWords(this::onFinishDeleting, languageIds);
 
 		return true;
 	}

src/io/github/sspanak/tt9/preferences/items/ItemTruncateUnselected.java

@@ -4,7 +4,7 @@ import androidx.preference.Preference;
 
 import java.util.ArrayList;
 
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
 import io.github.sspanak.tt9.db.DictionaryLoader;
 import io.github.sspanak.tt9.languages.Language;
 import io.github.sspanak.tt9.languages.LanguageCollection;
@@ -40,7 +40,7 @@ public class ItemTruncateUnselected extends ItemTruncateAll {
 		}
 
 		onStartDeleting();
-		DictionaryDb.deleteWords(this::onFinishDeleting, unselectedLanguageIds);
+		WordStoreAsync.deleteWords(this::onFinishDeleting, unselectedLanguageIds);
 
 		return true;
 	}

src/io/github/sspanak/tt9/preferences/screens/DictionariesScreen.java

@@ -11,8 +11,6 @@ import io.github.sspanak.tt9.preferences.items.ItemTruncateUnselected;
 
 public class DictionariesScreen extends BaseScreenFragment {
 	private ItemLoadDictionary loadItem;
-	private ItemTruncateUnselected deleteItem;
-	private ItemTruncateAll truncateItem;
 
 	public DictionariesScreen() { init(); }
 	public DictionariesScreen(PreferencesActivity activity) { init(activity); }
@@ -37,14 +35,14 @@ public class DictionariesScreen extends BaseScreenFragment {
 			activity.getDictionaryProgressBar()
 		);
 
-		deleteItem = new ItemTruncateUnselected(
+		ItemTruncateUnselected deleteItem = new ItemTruncateUnselected(
 			findPreference(ItemTruncateUnselected.NAME),
 			activity,
 			activity.settings,
 			activity.getDictionaryLoader()
 		);
 
-		truncateItem = new ItemTruncateAll(
+		ItemTruncateAll truncateItem = new ItemTruncateAll(
 			findPreference(ItemTruncateAll.NAME),
 			activity,
 			activity.getDictionaryLoader()

src/io/github/sspanak/tt9/preferences/screens/UsageStatsScreen.java

@@ -0,0 +1,74 @@
+package io.github.sspanak.tt9.preferences.screens;
+
+import android.content.ClipData;
+import android.content.ClipboardManager;
+import android.content.Context;
+import android.os.Build;
+
+import androidx.preference.Preference;
+
+import io.github.sspanak.tt9.R;
+import io.github.sspanak.tt9.db.SlowQueryStats;
+import io.github.sspanak.tt9.preferences.PreferencesActivity;
+import io.github.sspanak.tt9.ui.UI;
+
+public class UsageStatsScreen extends BaseScreenFragment {
+	private final static String RESET_BUTTON = "pref_slow_queries_reset_stats";
+	private final static String SUMMARY_CONTAINER = "summary_container";
+	private final static String QUERY_LIST_CONTAINER = "query_list_container";
+
+	public UsageStatsScreen() { init(); }
+	public UsageStatsScreen(PreferencesActivity activity) { init(activity); }
+
+	@Override protected int getTitle() { return R.string.pref_category_usage_stats; }
+	@Override protected int getXml() { return R.xml.prefs_screen_usage_stats; }
+
+	@Override
+	protected void onCreate() {
+		printSummary();
+		printSlowQueries();
+		enableLogsCopy();
+
+		Preference resetButton = findPreference(RESET_BUTTON);
+		if (resetButton != null) {
+			resetButton.setOnPreferenceClickListener((Preference p) -> {
+				SlowQueryStats.clear();
+				printSummary();
+				printSlowQueries();
+				return true;
+			});
+		}
+	}
+
+	private void printSummary() {
+		Preference logsContainer = findPreference(SUMMARY_CONTAINER);
+		if (logsContainer != null) {
+			logsContainer.setSummary(SlowQueryStats.getSummary());
+		}
+	}
+
+	private void printSlowQueries() {
+		Preference queryListContainer = findPreference(QUERY_LIST_CONTAINER);
+		if (queryListContainer != null) {
+			String slowQueries = SlowQueryStats.getList();
+			queryListContainer.setSummary(slowQueries.isEmpty() ? "No slow queries." : slowQueries);
+		}
+	}
+
+
+	private void enableLogsCopy() {
+		Preference queryListContainer = findPreference(QUERY_LIST_CONTAINER);
+		if (activity == null || queryListContainer == null) {
+			return;
+		}
+
+		ClipboardManager clipboard = (ClipboardManager) activity.getSystemService(Context.CLIPBOARD_SERVICE);
+		queryListContainer.setOnPreferenceClickListener((Preference p) -> {
+			clipboard.setPrimaryClip(ClipData.newPlainText("TT9 debug log", p.getSummary()));
+			if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.S_V2) {
+				UI.toast(activity, "Logs copied.");
+			}
+			return true;
+		});
+	}
+}

src/io/github/sspanak/tt9/ui/AddWordAct.java

@@ -9,14 +9,18 @@ import androidx.appcompat.app.AppCompatActivity;
 
 import io.github.sspanak.tt9.Logger;
 import io.github.sspanak.tt9.R;
-import io.github.sspanak.tt9.db.DictionaryDb;
+import io.github.sspanak.tt9.db.WordStoreAsync;
-import io.github.sspanak.tt9.db.exceptions.InsertBlankWordException;
 import io.github.sspanak.tt9.ime.TraditionalT9;
-import io.github.sspanak.tt9.languages.InvalidLanguageException;
 import io.github.sspanak.tt9.languages.Language;
 import io.github.sspanak.tt9.languages.LanguageCollection;
 
 public class AddWordAct extends AppCompatActivity {
+	public static final int CODE_SUCCESS = 0;
+	public static final int CODE_BLANK_WORD = 1;
+	public static final int CODE_INVALID_LANGUAGE = 2;
+	public static final int CODE_WORD_EXISTS = 3;
+	public static final int CODE_GENERAL_ERROR = 666;
+
 	public static final String INTENT_FILTER = "tt9.add_word";
 
 	private Language language;
@@ -61,14 +65,22 @@ public class AddWordAct extends AppCompatActivity {
 	private void onAddedWord(int statusCode) {
 		String message;
 		switch (statusCode) {
-			case 0:
+			case CODE_SUCCESS:
 				message = getString(R.string.add_word_success, word);
 				break;
 
-			case 1:
+			case CODE_WORD_EXISTS:
 				message = getResources().getString(R.string.add_word_exist, word);
 				break;
 
+			case CODE_BLANK_WORD:
+				message = getString(R.string.add_word_blank);
+				break;
+
+			case CODE_INVALID_LANGUAGE:
+				message = getResources().getString(R.string.add_word_invalid_language);
+				break;
+
 			default:
 				message = getString(R.string.error_unexpected);
 				break;
@@ -80,22 +92,7 @@ public class AddWordAct extends AppCompatActivity {
 
 
 	public void addWord(View v) {
-		try {
+		WordStoreAsync.put(this::onAddedWord, language, word);
-			Logger.d("addWord", "Attempting to add word: '" + word + "'...");
-			DictionaryDb.insertWord(this::onAddedWord, language, word);
-		} catch (InsertBlankWordException e) {
-			Logger.e("AddWordAct.addWord", e.getMessage());
-			finish();
-			sendMessageToMain(getString(R.string.add_word_blank));
-		} catch (InvalidLanguageException e) {
-			Logger.e("AddWordAct.addWord", "Cannot insert a word for language: '" + language.getName() + "'. " + e.getMessage());
-			finish();
-			sendMessageToMain(getString(R.string.add_word_invalid_language));
-		} catch (Exception e) {
-			Logger.e("AddWordAct.addWord", e.getMessage());
-			finish();
-			sendMessageToMain(e.getMessage());
-		}
 	}
 
 

src/io/github/sspanak/tt9/ui/main/BaseMainLayout.java

@@ -15,7 +15,7 @@ abstract class BaseMainLayout {
 	protected View view = null;
 	protected ArrayList<SoftKey> keys = new ArrayList<>();
 
-	public BaseMainLayout(TraditionalT9 tt9, int xml) {
+	BaseMainLayout(TraditionalT9 tt9, int xml) {
 		this.tt9 = tt9;
 		this.xml = xml;
 	}

src/io/github/sspanak/tt9/ui/main/MainLayoutNumpad.java

@@ -13,7 +13,7 @@ import io.github.sspanak.tt9.ime.TraditionalT9;
 import io.github.sspanak.tt9.ui.main.keys.SoftKey;
 
 class MainLayoutNumpad extends BaseMainLayout {
-	public MainLayoutNumpad(TraditionalT9 tt9) {
+	MainLayoutNumpad(TraditionalT9 tt9) {
 		super(tt9, R.layout.main_numpad);
 	}
 

src/io/github/sspanak/tt9/ui/main/MainLayoutSmall.java

@@ -12,7 +12,7 @@ import io.github.sspanak.tt9.ime.TraditionalT9;
 import io.github.sspanak.tt9.ui.main.keys.SoftKey;
 
 class MainLayoutSmall extends BaseMainLayout {
-	public MainLayoutSmall(TraditionalT9 tt9) {
+	MainLayoutSmall(TraditionalT9 tt9) {
 		super(tt9, R.layout.main_small);
 	}
 

src/io/github/sspanak/tt9/ui/main/keys/SoftKey.java

@@ -17,6 +17,7 @@ import androidx.core.content.ContextCompat;
 import io.github.sspanak.tt9.Logger;
 import io.github.sspanak.tt9.R;
 import io.github.sspanak.tt9.ime.TraditionalT9;
+import io.github.sspanak.tt9.preferences.SettingsStore;
 
 public class SoftKey extends androidx.appcompat.widget.AppCompatButton implements View.OnTouchListener, View.OnLongClickListener {
 	protected TraditionalT9 tt9;
@@ -109,7 +110,7 @@ public class SoftKey extends androidx.appcompat.widget.AppCompatButton implement
 			handleHold();
 			lastPressedKey = getId();
 			repeatHandler.removeCallbacks(this::repeatOnLongPress);
-			repeatHandler.postDelayed(this::repeatOnLongPress, tt9.getSettings().getSoftKeyRepeatDelay());
+			repeatHandler.postDelayed(this::repeatOnLongPress, SettingsStore.SOFT_KEY_REPEAT_DELAY);
 		}
 	}
 

src/io/github/sspanak/tt9/ui/tray/SuggestionsBar.java

@@ -19,6 +19,7 @@ import java.util.List;
 
 import io.github.sspanak.tt9.R;
 import io.github.sspanak.tt9.ime.TraditionalT9;
+import io.github.sspanak.tt9.preferences.SettingsStore;
 
 public class SuggestionsBar {
 	private final List<String> suggestions = new ArrayList<>();
@@ -52,13 +53,10 @@ public class SuggestionsBar {
 	private void configureAnimation() {
 		DefaultItemAnimator animator = new DefaultItemAnimator();
 
-		int translateDuration = tt9.getSettings().getSuggestionTranslateAnimationDuration();
+		animator.setMoveDuration(SettingsStore.SUGGESTIONS_SELECT_ANIMATION_DURATION);
-		int selectDuration = tt9.getSettings().getSuggestionSelectAnimationDuration();
+		animator.setChangeDuration(SettingsStore.SUGGESTIONS_TRANSLATE_ANIMATION_DURATION);
-
+		animator.setAddDuration(SettingsStore.SUGGESTIONS_TRANSLATE_ANIMATION_DURATION);
-		animator.setMoveDuration(selectDuration);
+		animator.setRemoveDuration(SettingsStore.SUGGESTIONS_TRANSLATE_ANIMATION_DURATION);
-		animator.setChangeDuration(translateDuration);
-		animator.setAddDuration(translateDuration);
-		animator.setRemoveDuration(translateDuration);
 
 		mView.setItemAnimator(animator);
 	}